diff options
author | Chocobozzz <me@florianbigard.com> | 2019-02-11 11:52:34 +0100 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2019-02-11 11:52:34 +0100 |
commit | 88108880bbdba473cfe36ecbebc1c3c4f972e102 (patch) | |
tree | b242efb3b4f0d7e49d88f2d1f2063b5b3b0489c0 /server/lib | |
parent | 53a94c7cfa8368da4cd248d65df8346905938f0c (diff) | |
parent | 9b712a2017e4ab3cf12cd6bd58278905520159d0 (diff) | |
download | PeerTube-88108880bbdba473cfe36ecbebc1c3c4f972e102.tar.gz PeerTube-88108880bbdba473cfe36ecbebc1c3c4f972e102.tar.zst PeerTube-88108880bbdba473cfe36ecbebc1c3c4f972e102.zip |
Merge branch 'develop' into pr/1217
Diffstat (limited to 'server/lib')
58 files changed, 2287 insertions, 643 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 45dd4443d..a3f379b76 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -1,19 +1,18 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { join } from 'path' | ||
3 | import { Transaction } from 'sequelize' | 2 | import { Transaction } from 'sequelize' |
4 | import * as url from 'url' | 3 | import * as url from 'url' |
5 | import * as uuidv4 from 'uuid/v4' | 4 | import * as uuidv4 from 'uuid/v4' |
6 | import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' | 5 | import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' |
7 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' | 6 | import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' |
8 | import { getActorUrl } from '../../helpers/activitypub' | 7 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
9 | import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' | 8 | import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' |
10 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 9 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
11 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' | 10 | import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' |
12 | import { logger } from '../../helpers/logger' | 11 | import { logger } from '../../helpers/logger' |
13 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' | 12 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' |
14 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | 13 | import { doRequest, downloadImage } from '../../helpers/requests' |
15 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | 14 | import { getUrlFromWebfinger } from '../../helpers/webfinger' |
16 | import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' | 15 | import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers' |
17 | import { AccountModel } from '../../models/account/account' | 16 | import { AccountModel } from '../../models/account/account' |
18 | import { ActorModel } from '../../models/activitypub/actor' | 17 | import { ActorModel } from '../../models/activitypub/actor' |
19 | import { AvatarModel } from '../../models/avatar/avatar' | 18 | import { AvatarModel } from '../../models/avatar/avatar' |
@@ -43,7 +42,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
43 | recurseIfNeeded = true, | 42 | recurseIfNeeded = true, |
44 | updateCollections = false | 43 | updateCollections = false |
45 | ) { | 44 | ) { |
46 | const actorUrl = getActorUrl(activityActor) | 45 | const actorUrl = getAPId(activityActor) |
47 | let created = false | 46 | let created = false |
48 | 47 | ||
49 | let actor = await fetchActorByUrl(actorUrl, fetchType) | 48 | let actor = await fetchActorByUrl(actorUrl, fetchType) |
@@ -65,8 +64,12 @@ async function getOrCreateActorAndServerAndModel ( | |||
65 | const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') | 64 | const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') |
66 | if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) | 65 | if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) |
67 | 66 | ||
67 | if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { | ||
68 | throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) | ||
69 | } | ||
70 | |||
68 | try { | 71 | try { |
69 | // Assert we don't recurse another time | 72 | // Don't recurse another time |
70 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) | 73 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) |
71 | } catch (err) { | 74 | } catch (err) { |
72 | logger.error('Cannot get or create account attributed to video channel ' + actor.url) | 75 | logger.error('Cannot get or create account attributed to video channel ' + actor.url) |
@@ -168,18 +171,13 @@ async function fetchActorTotalItems (url: string) { | |||
168 | 171 | ||
169 | async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { | 172 | async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { |
170 | if ( | 173 | if ( |
171 | actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && | 174 | actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && |
172 | isActivityPubUrlValid(actorJSON.icon.url) | 175 | isActivityPubUrlValid(actorJSON.icon.url) |
173 | ) { | 176 | ) { |
174 | const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] | 177 | const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] |
175 | 178 | ||
176 | const avatarName = uuidv4() + extension | 179 | const avatarName = uuidv4() + extension |
177 | const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) | 180 | await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) |
178 | |||
179 | await doRequestAndSaveToFile({ | ||
180 | method: 'GET', | ||
181 | uri: actorJSON.icon.url | ||
182 | }, destPath) | ||
183 | 181 | ||
184 | return avatarName | 182 | return avatarName |
185 | } | 183 | } |
@@ -203,6 +201,69 @@ async function addFetchOutboxJob (actor: ActorModel) { | |||
203 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | 201 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) |
204 | } | 202 | } |
205 | 203 | ||
204 | async function refreshActorIfNeeded ( | ||
205 | actorArg: ActorModel, | ||
206 | fetchedType: ActorFetchByUrlType | ||
207 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
208 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
209 | |||
210 | // We need more attributes | ||
211 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
212 | |||
213 | try { | ||
214 | let actorUrl: string | ||
215 | try { | ||
216 | actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
217 | } catch (err) { | ||
218 | logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) | ||
219 | actorUrl = actor.url | ||
220 | } | ||
221 | |||
222 | const { result, statusCode } = await fetchRemoteActor(actorUrl) | ||
223 | |||
224 | if (statusCode === 404) { | ||
225 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
226 | actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() | ||
227 | return { actor: undefined, refreshed: false } | ||
228 | } | ||
229 | |||
230 | if (result === undefined) { | ||
231 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
232 | return { actor, refreshed: false } | ||
233 | } | ||
234 | |||
235 | return sequelizeTypescript.transaction(async t => { | ||
236 | updateInstanceWithAnother(actor, result.actor) | ||
237 | |||
238 | if (result.avatarName !== undefined) { | ||
239 | await updateActorAvatarInstance(actor, result.avatarName, t) | ||
240 | } | ||
241 | |||
242 | // Force update | ||
243 | actor.setDataValue('updatedAt', new Date()) | ||
244 | await actor.save({ transaction: t }) | ||
245 | |||
246 | if (actor.Account) { | ||
247 | actor.Account.set('name', result.name) | ||
248 | actor.Account.set('description', result.summary) | ||
249 | |||
250 | await actor.Account.save({ transaction: t }) | ||
251 | } else if (actor.VideoChannel) { | ||
252 | actor.VideoChannel.set('name', result.name) | ||
253 | actor.VideoChannel.set('description', result.summary) | ||
254 | actor.VideoChannel.set('support', result.support) | ||
255 | |||
256 | await actor.VideoChannel.save({ transaction: t }) | ||
257 | } | ||
258 | |||
259 | return { refreshed: true, actor } | ||
260 | }) | ||
261 | } catch (err) { | ||
262 | logger.warn('Cannot refresh actor.', { err }) | ||
263 | return { actor, refreshed: false } | ||
264 | } | ||
265 | } | ||
266 | |||
206 | export { | 267 | export { |
207 | getOrCreateActorAndServerAndModel, | 268 | getOrCreateActorAndServerAndModel, |
208 | buildActorInstance, | 269 | buildActorInstance, |
@@ -210,6 +271,7 @@ export { | |||
210 | fetchActorTotalItems, | 271 | fetchActorTotalItems, |
211 | fetchAvatarIfExists, | 272 | fetchAvatarIfExists, |
212 | updateActorInstance, | 273 | updateActorInstance, |
274 | refreshActorIfNeeded, | ||
213 | updateActorAvatarInstance, | 275 | updateActorAvatarInstance, |
214 | addFetchOutboxJob | 276 | addFetchOutboxJob |
215 | } | 277 | } |
@@ -293,16 +355,19 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
293 | 355 | ||
294 | logger.info('Fetching remote actor %s.', actorUrl) | 356 | logger.info('Fetching remote actor %s.', actorUrl) |
295 | 357 | ||
296 | const requestResult = await doRequest(options) | 358 | const requestResult = await doRequest<ActivityPubActor>(options) |
297 | normalizeActor(requestResult.body) | 359 | normalizeActor(requestResult.body) |
298 | 360 | ||
299 | const actorJSON: ActivityPubActor = requestResult.body | 361 | const actorJSON = requestResult.body |
300 | |||
301 | if (isActorObjectValid(actorJSON) === false) { | 362 | if (isActorObjectValid(actorJSON) === false) { |
302 | logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) | 363 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) |
303 | return { result: undefined, statusCode: requestResult.response.statusCode } | 364 | return { result: undefined, statusCode: requestResult.response.statusCode } |
304 | } | 365 | } |
305 | 366 | ||
367 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { | ||
368 | throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) | ||
369 | } | ||
370 | |||
306 | const followersCount = await fetchActorTotalItems(actorJSON.followers) | 371 | const followersCount = await fetchActorTotalItems(actorJSON.followers) |
307 | const followingCount = await fetchActorTotalItems(actorJSON.following) | 372 | const followingCount = await fetchActorTotalItems(actorJSON.following) |
308 | 373 | ||
@@ -371,59 +436,3 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu | |||
371 | 436 | ||
372 | return videoChannelCreated | 437 | return videoChannelCreated |
373 | } | 438 | } |
374 | |||
375 | async function refreshActorIfNeeded ( | ||
376 | actorArg: ActorModel, | ||
377 | fetchedType: ActorFetchByUrlType | ||
378 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
379 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
380 | |||
381 | // We need more attributes | ||
382 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
383 | |||
384 | try { | ||
385 | const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
386 | const { result, statusCode } = await fetchRemoteActor(actorUrl) | ||
387 | |||
388 | if (statusCode === 404) { | ||
389 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
390 | actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() | ||
391 | return { actor: undefined, refreshed: false } | ||
392 | } | ||
393 | |||
394 | if (result === undefined) { | ||
395 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
396 | return { actor, refreshed: false } | ||
397 | } | ||
398 | |||
399 | return sequelizeTypescript.transaction(async t => { | ||
400 | updateInstanceWithAnother(actor, result.actor) | ||
401 | |||
402 | if (result.avatarName !== undefined) { | ||
403 | await updateActorAvatarInstance(actor, result.avatarName, t) | ||
404 | } | ||
405 | |||
406 | // Force update | ||
407 | actor.setDataValue('updatedAt', new Date()) | ||
408 | await actor.save({ transaction: t }) | ||
409 | |||
410 | if (actor.Account) { | ||
411 | actor.Account.set('name', result.name) | ||
412 | actor.Account.set('description', result.summary) | ||
413 | |||
414 | await actor.Account.save({ transaction: t }) | ||
415 | } else if (actor.VideoChannel) { | ||
416 | actor.VideoChannel.set('name', result.name) | ||
417 | actor.VideoChannel.set('description', result.summary) | ||
418 | actor.VideoChannel.set('support', result.support) | ||
419 | |||
420 | await actor.VideoChannel.save({ transaction: t }) | ||
421 | } | ||
422 | |||
423 | return { refreshed: true, actor } | ||
424 | }) | ||
425 | } catch (err) { | ||
426 | logger.warn('Cannot refresh actor.', { err }) | ||
427 | return { actor, refreshed: false } | ||
428 | } | ||
429 | } | ||
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..9a40414bb 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,11 +1,28 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | 1 | import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index' |
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
4 | import { Transaction } from 'sequelize' | 4 | import { Transaction } from 'sequelize' |
5 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | 6 | ||
6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
7 | const url = cacheFileObject.url | ||
8 | 8 | ||
9 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
10 | const url = cacheFileObject.url | ||
11 | |||
12 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
14 | |||
15 | return { | ||
16 | expiresOn: new Date(cacheFileObject.expires), | ||
17 | url: cacheFileObject.id, | ||
18 | fileUrl: url.href, | ||
19 | strategy: null, | ||
20 | videoStreamingPlaylistId: playlist.id, | ||
21 | actorId: byActor.id | ||
22 | } | ||
23 | } | ||
24 | |||
25 | const url = cacheFileObject.url | ||
9 | const videoFile = video.VideoFiles.find(f => { | 26 | const videoFile = video.VideoFiles.find(f => { |
10 | return f.resolution === url.height && f.fps === url.fps | 27 | return f.resolution === url.height && f.fps === url.fps |
11 | }) | 28 | }) |
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
15 | return { | 32 | return { |
16 | expiresOn: new Date(cacheFileObject.expires), | 33 | expiresOn: new Date(cacheFileObject.expires), |
17 | url: cacheFileObject.id, | 34 | url: cacheFileObject.id, |
18 | fileUrl: cacheFileObject.url.href, | 35 | fileUrl: url.href, |
19 | strategy: null, | 36 | strategy: null, |
20 | videoFileId: videoFile.id, | 37 | videoFileId: videoFile.id, |
21 | actorId: byActor.id | 38 | actorId: byActor.id |
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 55912341c..1b9b14c2e 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts | |||
@@ -1,7 +1,8 @@ | |||
1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' | 1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' |
2 | import { doRequest } from '../../helpers/requests' | 2 | import { doRequest } from '../../helpers/requests' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import Bluebird = require('bluebird') | 4 | import * as Bluebird from 'bluebird' |
5 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | ||
5 | 6 | ||
6 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { | 7 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { |
7 | logger.info('Crawling ActivityPub data on %s.', uri) | 8 | logger.info('Crawling ActivityPub data on %s.', uri) |
@@ -14,7 +15,7 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr | |||
14 | timeout: JOB_REQUEST_TIMEOUT | 15 | timeout: JOB_REQUEST_TIMEOUT |
15 | } | 16 | } |
16 | 17 | ||
17 | const response = await doRequest(options) | 18 | const response = await doRequest<ActivityPubOrderedCollection<T>>(options) |
18 | const firstBody = response.body | 19 | const firstBody = response.body |
19 | 20 | ||
20 | let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | 21 | let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT |
@@ -23,7 +24,7 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr | |||
23 | while (nextLink && i < limit) { | 24 | while (nextLink && i < limit) { |
24 | options.uri = nextLink | 25 | options.uri = nextLink |
25 | 26 | ||
26 | const { body } = await doRequest(options) | 27 | const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options) |
27 | nextLink = body.next | 28 | nextLink = body.next |
28 | i++ | 29 | i++ |
29 | 30 | ||
diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts index db4980a72..5466739c1 100644 --- a/server/lib/activitypub/process/index.ts +++ b/server/lib/activitypub/process/index.ts | |||
@@ -1,9 +1 @@ | |||
1 | export * from './process' | export * from './process' | |
2 | export * from './process-accept' | ||
3 | export * from './process-announce' | ||
4 | export * from './process-create' | ||
5 | export * from './process-delete' | ||
6 | export * from './process-follow' | ||
7 | export * from './process-like' | ||
8 | export * from './process-undo' | ||
9 | export * from './process-update' | ||
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 89bda9c32..ebb275e34 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts | |||
@@ -24,6 +24,7 @@ async function processAccept (actor: ActorModel, targetActor: ActorModel) { | |||
24 | if (follow.state !== 'accepted') { | 24 | if (follow.state !== 'accepted') { |
25 | follow.set('state', 'accepted') | 25 | follow.set('state', 'accepted') |
26 | await follow.save() | 26 | await follow.save() |
27 | |||
27 | await addFetchOutboxJob(targetActor) | 28 | await addFetchOutboxJob(targetActor) |
28 | } | 29 | } |
29 | } | 30 | } |
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index cc88b5423..23310b41e 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts | |||
@@ -5,6 +5,8 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
5 | import { VideoShareModel } from '../../../models/video/video-share' | 5 | import { VideoShareModel } from '../../../models/video/video-share' |
6 | import { forwardVideoRelatedActivity } from '../send/utils' | 6 | import { forwardVideoRelatedActivity } from '../send/utils' |
7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
8 | import { VideoPrivacy } from '../../../../shared/models/videos' | ||
9 | import { Notifier } from '../../notifier' | ||
8 | 10 | ||
9 | async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) { | 11 | async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) { |
10 | return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) | 12 | return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) |
@@ -21,9 +23,9 @@ export { | |||
21 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { | 23 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { |
22 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id | 24 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id |
23 | 25 | ||
24 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) | 26 | const { video, created: videoCreated } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) |
25 | 27 | ||
26 | return sequelizeTypescript.transaction(async t => { | 28 | await sequelizeTypescript.transaction(async t => { |
27 | // Add share entry | 29 | // Add share entry |
28 | 30 | ||
29 | const share = { | 31 | const share = { |
@@ -49,4 +51,6 @@ async function processVideoShare (actorAnnouncer: ActorModel, activity: Activity | |||
49 | 51 | ||
50 | return undefined | 52 | return undefined |
51 | }) | 53 | }) |
54 | |||
55 | if (videoCreated) Notifier.Instance.notifyOnNewVideo(video) | ||
52 | } | 56 | } |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index cefe89db0..5f4d793a5 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -1,34 +1,44 @@ | |||
1 | import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' | 1 | import { ActivityCreate, CacheFileObject, VideoTorrentObject } from '../../../../shared' |
2 | import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' | ||
3 | import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' | 2 | import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' |
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
5 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
6 | import { sequelizeTypescript } from '../../../initializers' | 5 | import { sequelizeTypescript } from '../../../initializers' |
7 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
8 | import { ActorModel } from '../../../models/activitypub/actor' | 6 | import { ActorModel } from '../../../models/activitypub/actor' |
9 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
10 | import { addVideoComment, resolveThread } from '../video-comments' | 7 | import { addVideoComment, resolveThread } from '../video-comments' |
11 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 8 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
12 | import { forwardVideoRelatedActivity } from '../send/utils' | 9 | import { forwardVideoRelatedActivity } from '../send/utils' |
13 | import { Redis } from '../../redis' | ||
14 | import { createOrUpdateCacheFile } from '../cache-file' | 10 | import { createOrUpdateCacheFile } from '../cache-file' |
11 | import { Notifier } from '../../notifier' | ||
12 | import { processViewActivity } from './process-view' | ||
13 | import { processDislikeActivity } from './process-dislike' | ||
14 | import { processFlagActivity } from './process-flag' | ||
15 | 15 | ||
16 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { | 16 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { |
17 | const activityObject = activity.object | 17 | const activityObject = activity.object |
18 | const activityType = activityObject.type | 18 | const activityType = activityObject.type |
19 | 19 | ||
20 | if (activityType === 'View') { | 20 | if (activityType === 'View') { |
21 | return processCreateView(byActor, activity) | 21 | return processViewActivity(activity, byActor) |
22 | } else if (activityType === 'Dislike') { | 22 | } |
23 | return retryTransactionWrapper(processCreateDislike, byActor, activity) | 23 | |
24 | } else if (activityType === 'Video') { | 24 | if (activityType === 'Dislike') { |
25 | return retryTransactionWrapper(processDislikeActivity, activity, byActor) | ||
26 | } | ||
27 | |||
28 | if (activityType === 'Flag') { | ||
29 | return retryTransactionWrapper(processFlagActivity, activity, byActor) | ||
30 | } | ||
31 | |||
32 | if (activityType === 'Video') { | ||
25 | return processCreateVideo(activity) | 33 | return processCreateVideo(activity) |
26 | } else if (activityType === 'Flag') { | 34 | } |
27 | return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) | 35 | |
28 | } else if (activityType === 'Note') { | 36 | if (activityType === 'Note') { |
29 | return retryTransactionWrapper(processCreateVideoComment, byActor, activity) | 37 | return retryTransactionWrapper(processCreateVideoComment, activity, byActor) |
30 | } else if (activityType === 'CacheFile') { | 38 | } |
31 | return retryTransactionWrapper(processCacheFile, byActor, activity) | 39 | |
40 | if (activityType === 'CacheFile') { | ||
41 | return retryTransactionWrapper(processCacheFile, activity, byActor) | ||
32 | } | 42 | } |
33 | 43 | ||
34 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | 44 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) |
@@ -46,60 +56,14 @@ export { | |||
46 | async function processCreateVideo (activity: ActivityCreate) { | 56 | async function processCreateVideo (activity: ActivityCreate) { |
47 | const videoToCreateData = activity.object as VideoTorrentObject | 57 | const videoToCreateData = activity.object as VideoTorrentObject |
48 | 58 | ||
49 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) | 59 | const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) |
50 | |||
51 | return video | ||
52 | } | ||
53 | |||
54 | async function processCreateDislike (byActor: ActorModel, activity: ActivityCreate) { | ||
55 | const dislike = activity.object as DislikeObject | ||
56 | const byAccount = byActor.Account | ||
57 | |||
58 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
59 | |||
60 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) | ||
61 | |||
62 | return sequelizeTypescript.transaction(async t => { | ||
63 | const rate = { | ||
64 | type: 'dislike' as 'dislike', | ||
65 | videoId: video.id, | ||
66 | accountId: byAccount.id | ||
67 | } | ||
68 | const [ , created ] = await AccountVideoRateModel.findOrCreate({ | ||
69 | where: rate, | ||
70 | defaults: rate, | ||
71 | transaction: t | ||
72 | }) | ||
73 | if (created === true) await video.increment('dislikes', { transaction: t }) | ||
74 | |||
75 | if (video.isOwned() && created === true) { | ||
76 | // Don't resend the activity to the sender | ||
77 | const exceptions = [ byActor ] | ||
78 | |||
79 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
80 | } | ||
81 | }) | ||
82 | } | ||
83 | |||
84 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { | ||
85 | const view = activity.object as ViewObject | ||
86 | |||
87 | const options = { | ||
88 | videoObject: view.object, | ||
89 | fetchType: 'only-video' as 'only-video' | ||
90 | } | ||
91 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | ||
92 | 60 | ||
93 | await Redis.Instance.addVideoView(video.id) | 61 | if (created) Notifier.Instance.notifyOnNewVideo(video) |
94 | 62 | ||
95 | if (video.isOwned()) { | 63 | return video |
96 | // Don't resend the activity to the sender | ||
97 | const exceptions = [ byActor ] | ||
98 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
99 | } | ||
100 | } | 64 | } |
101 | 65 | ||
102 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { | 66 | async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) { |
103 | const cacheFile = activity.object as CacheFileObject | 67 | const cacheFile = activity.object as CacheFileObject |
104 | 68 | ||
105 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) | 69 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) |
@@ -115,29 +79,7 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) | |||
115 | } | 79 | } |
116 | } | 80 | } |
117 | 81 | ||
118 | async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { | 82 | async function processCreateVideoComment (activity: ActivityCreate, byActor: ActorModel) { |
119 | logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) | ||
120 | |||
121 | const account = byActor.Account | ||
122 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
123 | |||
124 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) | ||
125 | |||
126 | return sequelizeTypescript.transaction(async t => { | ||
127 | const videoAbuseData = { | ||
128 | reporterAccountId: account.id, | ||
129 | reason: videoAbuseToCreateData.content, | ||
130 | videoId: video.id, | ||
131 | state: VideoAbuseState.PENDING | ||
132 | } | ||
133 | |||
134 | await VideoAbuseModel.create(videoAbuseData, { transaction: t }) | ||
135 | |||
136 | logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) | ||
137 | }) | ||
138 | } | ||
139 | |||
140 | async function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreate) { | ||
141 | const commentObject = activity.object as VideoCommentObject | 83 | const commentObject = activity.object as VideoCommentObject |
142 | const byAccount = byActor.Account | 84 | const byAccount = byActor.Account |
143 | 85 | ||
@@ -145,7 +87,7 @@ async function processCreateVideoComment (byActor: ActorModel, activity: Activit | |||
145 | 87 | ||
146 | const { video } = await resolveThread(commentObject.inReplyTo) | 88 | const { video } = await resolveThread(commentObject.inReplyTo) |
147 | 89 | ||
148 | const { created } = await addVideoComment(video, commentObject.id) | 90 | const { comment, created } = await addVideoComment(video, commentObject.id) |
149 | 91 | ||
150 | if (video.isOwned() && created === true) { | 92 | if (video.isOwned() && created === true) { |
151 | // Don't resend the activity to the sender | 93 | // Don't resend the activity to the sender |
@@ -153,4 +95,6 @@ async function processCreateVideoComment (byActor: ActorModel, activity: Activit | |||
153 | 95 | ||
154 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | 96 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) |
155 | } | 97 | } |
98 | |||
99 | if (created === true) Notifier.Instance.notifyOnNewComment(comment) | ||
156 | } | 100 | } |
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts new file mode 100644 index 000000000..bfd69e07a --- /dev/null +++ b/server/lib/activitypub/process/process-dislike.ts | |||
@@ -0,0 +1,52 @@ | |||
1 | import { ActivityCreate, ActivityDislike } from '../../../../shared' | ||
2 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' | ||
3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
4 | import { sequelizeTypescript } from '../../../initializers' | ||
5 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
6 | import { ActorModel } from '../../../models/activitypub/actor' | ||
7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | ||
8 | import { forwardVideoRelatedActivity } from '../send/utils' | ||
9 | import { getVideoDislikeActivityPubUrl } from '../url' | ||
10 | |||
11 | async function processDislikeActivity (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { | ||
12 | return retryTransactionWrapper(processDislike, activity, byActor) | ||
13 | } | ||
14 | |||
15 | // --------------------------------------------------------------------------- | ||
16 | |||
17 | export { | ||
18 | processDislikeActivity | ||
19 | } | ||
20 | |||
21 | // --------------------------------------------------------------------------- | ||
22 | |||
23 | async function processDislike (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) { | ||
24 | const dislikeObject = activity.type === 'Dislike' ? activity.object : (activity.object as DislikeObject).object | ||
25 | const byAccount = byActor.Account | ||
26 | |||
27 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
28 | |||
29 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) | ||
30 | |||
31 | return sequelizeTypescript.transaction(async t => { | ||
32 | const rate = { | ||
33 | type: 'dislike' as 'dislike', | ||
34 | videoId: video.id, | ||
35 | accountId: byAccount.id | ||
36 | } | ||
37 | |||
38 | const [ , created ] = await AccountVideoRateModel.findOrCreate({ | ||
39 | where: rate, | ||
40 | defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }), | ||
41 | transaction: t | ||
42 | }) | ||
43 | if (created === true) await video.increment('dislikes', { transaction: t }) | ||
44 | |||
45 | if (video.isOwned() && created === true) { | ||
46 | // Don't resend the activity to the sender | ||
47 | const exceptions = [ byActor ] | ||
48 | |||
49 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | ||
50 | } | ||
51 | }) | ||
52 | } | ||
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts new file mode 100644 index 000000000..79ce6fb41 --- /dev/null +++ b/server/lib/activitypub/process/process-flag.ts | |||
@@ -0,0 +1,49 @@ | |||
1 | import { ActivityCreate, ActivityFlag, VideoAbuseState } from '../../../../shared' | ||
2 | import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects' | ||
3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
4 | import { logger } from '../../../helpers/logger' | ||
5 | import { sequelizeTypescript } from '../../../initializers' | ||
6 | import { ActorModel } from '../../../models/activitypub/actor' | ||
7 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
8 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | ||
9 | import { Notifier } from '../../notifier' | ||
10 | import { getAPId } from '../../../helpers/activitypub' | ||
11 | |||
12 | async function processFlagActivity (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { | ||
13 | return retryTransactionWrapper(processCreateVideoAbuse, activity, byActor) | ||
14 | } | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | export { | ||
19 | processFlagActivity | ||
20 | } | ||
21 | |||
22 | // --------------------------------------------------------------------------- | ||
23 | |||
24 | async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) { | ||
25 | const flag = activity.type === 'Flag' ? activity : (activity.object as VideoAbuseObject) | ||
26 | |||
27 | logger.debug('Reporting remote abuse for video %s.', getAPId(flag.object)) | ||
28 | |||
29 | const account = byActor.Account | ||
30 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | ||
31 | |||
32 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object }) | ||
33 | |||
34 | return sequelizeTypescript.transaction(async t => { | ||
35 | const videoAbuseData = { | ||
36 | reporterAccountId: account.id, | ||
37 | reason: flag.content, | ||
38 | videoId: video.id, | ||
39 | state: VideoAbuseState.PENDING | ||
40 | } | ||
41 | |||
42 | const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t }) | ||
43 | videoAbuseInstance.Video = video | ||
44 | |||
45 | Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance) | ||
46 | |||
47 | logger.info('Remote abuse for video uuid %s created', flag.object) | ||
48 | }) | ||
49 | } | ||
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index 24c9085f7..0cd537187 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts | |||
@@ -5,9 +5,11 @@ import { sequelizeTypescript } from '../../../initializers' | |||
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
7 | import { sendAccept } from '../send' | 7 | import { sendAccept } from '../send' |
8 | import { Notifier } from '../../notifier' | ||
9 | import { getAPId } from '../../../helpers/activitypub' | ||
8 | 10 | ||
9 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { | 11 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { |
10 | const activityObject = activity.object | 12 | const activityObject = getAPId(activity.object) |
11 | 13 | ||
12 | return retryTransactionWrapper(processFollow, byActor, activityObject) | 14 | return retryTransactionWrapper(processFollow, byActor, activityObject) |
13 | } | 15 | } |
@@ -21,13 +23,13 @@ export { | |||
21 | // --------------------------------------------------------------------------- | 23 | // --------------------------------------------------------------------------- |
22 | 24 | ||
23 | async function processFollow (actor: ActorModel, targetActorURL: string) { | 25 | async function processFollow (actor: ActorModel, targetActorURL: string) { |
24 | await sequelizeTypescript.transaction(async t => { | 26 | const { actorFollow, created } = await sequelizeTypescript.transaction(async t => { |
25 | const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) | 27 | const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) |
26 | 28 | ||
27 | if (!targetActor) throw new Error('Unknown actor') | 29 | if (!targetActor) throw new Error('Unknown actor') |
28 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') | 30 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') |
29 | 31 | ||
30 | const [ actorFollow ] = await ActorFollowModel.findOrCreate({ | 32 | const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({ |
31 | where: { | 33 | where: { |
32 | actorId: actor.id, | 34 | actorId: actor.id, |
33 | targetActorId: targetActor.id | 35 | targetActorId: targetActor.id |
@@ -52,8 +54,12 @@ async function processFollow (actor: ActorModel, targetActorURL: string) { | |||
52 | actorFollow.ActorFollowing = targetActor | 54 | actorFollow.ActorFollowing = targetActor |
53 | 55 | ||
54 | // Target sends to actor he accepted the follow request | 56 | // Target sends to actor he accepted the follow request |
55 | return sendAccept(actorFollow) | 57 | await sendAccept(actorFollow) |
58 | |||
59 | return { actorFollow, created } | ||
56 | }) | 60 | }) |
57 | 61 | ||
62 | if (created) Notifier.Instance.notifyOfNewFollow(actorFollow) | ||
63 | |||
58 | logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) | 64 | logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) |
59 | } | 65 | } |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index f7200db61..2a04167d7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -5,6 +5,8 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { forwardVideoRelatedActivity } from '../send/utils' | 6 | import { forwardVideoRelatedActivity } from '../send/utils' |
7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
8 | import { getVideoLikeActivityPubUrl } from '../url' | ||
9 | import { getAPId } from '../../../helpers/activitypub' | ||
8 | 10 | ||
9 | async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { | 11 | async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { |
10 | return retryTransactionWrapper(processLikeVideo, byActor, activity) | 12 | return retryTransactionWrapper(processLikeVideo, byActor, activity) |
@@ -19,7 +21,7 @@ export { | |||
19 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
20 | 22 | ||
21 | async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { | 23 | async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { |
22 | const videoUrl = activity.object | 24 | const videoUrl = getAPId(activity.object) |
23 | 25 | ||
24 | const byAccount = byActor.Account | 26 | const byAccount = byActor.Account |
25 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 27 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
@@ -34,7 +36,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { | |||
34 | } | 36 | } |
35 | const [ , created ] = await AccountVideoRateModel.findOrCreate({ | 37 | const [ , created ] = await AccountVideoRateModel.findOrCreate({ |
36 | where: rate, | 38 | where: rate, |
37 | defaults: rate, | 39 | defaults: Object.assign({}, rate, { url: getVideoLikeActivityPubUrl(byActor, video) }), |
38 | transaction: t | 40 | transaction: t |
39 | }) | 41 | }) |
40 | if (created === true) await video.increment('likes', { transaction: t }) | 42 | if (created === true) await video.increment('likes', { transaction: t }) |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index ff019cd8c..ed0177a67 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -26,6 +26,10 @@ async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) | |||
26 | } | 26 | } |
27 | } | 27 | } |
28 | 28 | ||
29 | if (activityToUndo.type === 'Dislike') { | ||
30 | return retryTransactionWrapper(processUndoDislike, byActor, activity) | ||
31 | } | ||
32 | |||
29 | if (activityToUndo.type === 'Follow') { | 33 | if (activityToUndo.type === 'Follow') { |
30 | return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) | 34 | return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) |
31 | } | 35 | } |
@@ -55,7 +59,8 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { | |||
55 | return sequelizeTypescript.transaction(async t => { | 59 | return sequelizeTypescript.transaction(async t => { |
56 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 60 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
57 | 61 | ||
58 | const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) | 62 | let rate = await AccountVideoRateModel.loadByUrl(likeActivity.id, t) |
63 | if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) | ||
59 | if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) | 64 | if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) |
60 | 65 | ||
61 | await rate.destroy({ transaction: t }) | 66 | await rate.destroy({ transaction: t }) |
@@ -71,14 +76,17 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { | |||
71 | } | 76 | } |
72 | 77 | ||
73 | async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { | 78 | async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { |
74 | const dislike = activity.object.object as DislikeObject | 79 | const dislike = activity.object.type === 'Dislike' |
80 | ? activity.object | ||
81 | : activity.object.object as DislikeObject | ||
75 | 82 | ||
76 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) | 83 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) |
77 | 84 | ||
78 | return sequelizeTypescript.transaction(async t => { | 85 | return sequelizeTypescript.transaction(async t => { |
79 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 86 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
80 | 87 | ||
81 | const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) | 88 | let rate = await AccountVideoRateModel.loadByUrl(dislike.id, t) |
89 | if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) | ||
82 | if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) | 90 | if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) |
83 | 91 | ||
84 | await rate.destroy({ transaction: t }) | 92 | await rate.destroy({ transaction: t }) |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index bd4013555..c6b42d846 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -51,7 +51,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
51 | return undefined | 51 | return undefined |
52 | } | 52 | } |
53 | 53 | ||
54 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) | 54 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id, allowRefresh: false }) |
55 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 55 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) |
56 | 56 | ||
57 | const updateOptions = { | 57 | const updateOptions = { |
@@ -59,7 +59,6 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
59 | videoObject, | 59 | videoObject, |
60 | account: actor.Account, | 60 | account: actor.Account, |
61 | channel: channelActor.VideoChannel, | 61 | channel: channelActor.VideoChannel, |
62 | updateViews: true, | ||
63 | overrideTo: activity.to | 62 | overrideTo: activity.to |
64 | } | 63 | } |
65 | return updateVideoFromAP(updateOptions) | 64 | return updateVideoFromAP(updateOptions) |
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts new file mode 100644 index 000000000..8f66d3630 --- /dev/null +++ b/server/lib/activitypub/process/process-view.ts | |||
@@ -0,0 +1,35 @@ | |||
1 | import { ActorModel } from '../../../models/activitypub/actor' | ||
2 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | ||
3 | import { forwardVideoRelatedActivity } from '../send/utils' | ||
4 | import { Redis } from '../../redis' | ||
5 | import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' | ||
6 | |||
7 | async function processViewActivity (activity: ActivityView | ActivityCreate, byActor: ActorModel) { | ||
8 | return processCreateView(activity, byActor) | ||
9 | } | ||
10 | |||
11 | // --------------------------------------------------------------------------- | ||
12 | |||
13 | export { | ||
14 | processViewActivity | ||
15 | } | ||
16 | |||
17 | // --------------------------------------------------------------------------- | ||
18 | |||
19 | async function processCreateView (activity: ActivityView | ActivityCreate, byActor: ActorModel) { | ||
20 | const videoObject = activity.type === 'View' ? activity.object : (activity.object as ViewObject).object | ||
21 | |||
22 | const options = { | ||
23 | videoObject: videoObject, | ||
24 | fetchType: 'only-video' as 'only-video' | ||
25 | } | ||
26 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | ||
27 | |||
28 | await Redis.Instance.addVideoView(video.id) | ||
29 | |||
30 | if (video.isOwned()) { | ||
31 | // Don't resend the activity to the sender | ||
32 | const exceptions = [ byActor ] | ||
33 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) | ||
34 | } | ||
35 | } | ||
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index b263f1ea2..9dd241402 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | 1 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' |
2 | import { getActorUrl } from '../../../helpers/activitypub' | 2 | import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { processAcceptActivity } from './process-accept' | 5 | import { processAcceptActivity } from './process-accept' |
@@ -12,6 +12,9 @@ import { processRejectActivity } from './process-reject' | |||
12 | import { processUndoActivity } from './process-undo' | 12 | import { processUndoActivity } from './process-undo' |
13 | import { processUpdateActivity } from './process-update' | 13 | import { processUpdateActivity } from './process-update' |
14 | import { getOrCreateActorAndServerAndModel } from '../actor' | 14 | import { getOrCreateActorAndServerAndModel } from '../actor' |
15 | import { processDislikeActivity } from './process-dislike' | ||
16 | import { processFlagActivity } from './process-flag' | ||
17 | import { processViewActivity } from './process-view' | ||
15 | 18 | ||
16 | const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = { | 19 | const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = { |
17 | Create: processCreateActivity, | 20 | Create: processCreateActivity, |
@@ -22,27 +25,41 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac | |||
22 | Reject: processRejectActivity, | 25 | Reject: processRejectActivity, |
23 | Announce: processAnnounceActivity, | 26 | Announce: processAnnounceActivity, |
24 | Undo: processUndoActivity, | 27 | Undo: processUndoActivity, |
25 | Like: processLikeActivity | 28 | Like: processLikeActivity, |
29 | Dislike: processDislikeActivity, | ||
30 | Flag: processFlagActivity, | ||
31 | View: processViewActivity | ||
26 | } | 32 | } |
27 | 33 | ||
28 | async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { | 34 | async function processActivities ( |
35 | activities: Activity[], | ||
36 | options: { | ||
37 | signatureActor?: ActorModel | ||
38 | inboxActor?: ActorModel | ||
39 | outboxUrl?: string | ||
40 | } = {}) { | ||
29 | const actorsCache: { [ url: string ]: ActorModel } = {} | 41 | const actorsCache: { [ url: string ]: ActorModel } = {} |
30 | 42 | ||
31 | for (const activity of activities) { | 43 | for (const activity of activities) { |
32 | if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { | 44 | if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { |
33 | logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) | 45 | logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) |
34 | continue | 46 | continue |
35 | } | 47 | } |
36 | 48 | ||
37 | const actorUrl = getActorUrl(activity.actor) | 49 | const actorUrl = getAPId(activity.actor) |
38 | 50 | ||
39 | // When we fetch remote data, we don't have signature | 51 | // When we fetch remote data, we don't have signature |
40 | if (signatureActor && actorUrl !== signatureActor.url) { | 52 | if (options.signatureActor && actorUrl !== options.signatureActor.url) { |
41 | logger.warn('Signature mismatch between %s and %s.', actorUrl, signatureActor.url) | 53 | logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, options.signatureActor.url) |
42 | continue | 54 | continue |
43 | } | 55 | } |
44 | 56 | ||
45 | const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) | 57 | if (options.outboxUrl && checkUrlsSameHost(options.outboxUrl, actorUrl) !== true) { |
58 | logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', options.outboxUrl, actorUrl) | ||
59 | continue | ||
60 | } | ||
61 | |||
62 | const byActor = options.signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) | ||
46 | actorsCache[actorUrl] = byActor | 63 | actorsCache[actorUrl] = byActor |
47 | 64 | ||
48 | const activityProcessor = processActivity[activity.type] | 65 | const activityProcessor = processActivity[activity.type] |
@@ -52,7 +69,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor | |||
52 | } | 69 | } |
53 | 70 | ||
54 | try { | 71 | try { |
55 | await activityProcessor(activity, byActor, inboxActor) | 72 | await activityProcessor(activity, byActor, options.inboxActor) |
56 | } catch (err) { | 73 | } catch (err) { |
57 | logger.warn('Cannot process activity %s.', activity.type, { err }) | 74 | logger.warn('Cannot process activity %s.', activity.type, { err }) |
58 | } | 75 | } |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 285edba3b..ef20e404c 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -3,9 +3,7 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti | |||
3 | import { VideoPrivacy } from '../../../../shared/models/videos' | 3 | import { VideoPrivacy } from '../../../../shared/models/videos' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { VideoModel } from '../../../models/video/video' | 5 | import { VideoModel } from '../../../models/video/video' |
6 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
7 | import { VideoCommentModel } from '../../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../../models/video/video-comment' |
8 | import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' | ||
9 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 7 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
10 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' | 8 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' |
11 | import { logger } from '../../../helpers/logger' | 9 | import { logger } from '../../../helpers/logger' |
@@ -25,31 +23,14 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { | |||
25 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) | 23 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) |
26 | } | 24 | } |
27 | 25 | ||
28 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { | 26 | async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { |
29 | if (!video.VideoChannel.Account.Actor.serverId) return // Local | ||
30 | |||
31 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | ||
32 | |||
33 | logger.info('Creating job to send video abuse %s.', url) | ||
34 | |||
35 | // Custom audience, we only send the abuse to the origin instance | ||
36 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | ||
37 | const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) | ||
38 | |||
39 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
40 | } | ||
41 | |||
42 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | ||
43 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | 27 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) |
44 | 28 | ||
45 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | ||
46 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
47 | |||
48 | return sendVideoRelatedCreateActivity({ | 29 | return sendVideoRelatedCreateActivity({ |
49 | byActor, | 30 | byActor, |
50 | video, | 31 | video, |
51 | url: fileRedundancy.url, | 32 | url: fileRedundancy.url, |
52 | object: redundancyObject | 33 | object: fileRedundancy.toActivityPubObject() |
53 | }) | 34 | }) |
54 | } | 35 | } |
55 | 36 | ||
@@ -91,37 +72,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio | |||
91 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) | 72 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) |
92 | } | 73 | } |
93 | 74 | ||
94 | async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
95 | logger.info('Creating job to send view of %s.', video.url) | ||
96 | |||
97 | const url = getVideoViewActivityPubUrl(byActor, video) | ||
98 | const viewActivity = buildViewActivity(byActor, video) | ||
99 | |||
100 | return sendVideoRelatedCreateActivity({ | ||
101 | // Use the server actor to send the view | ||
102 | byActor, | ||
103 | video, | ||
104 | url, | ||
105 | object: viewActivity, | ||
106 | transaction: t | ||
107 | }) | ||
108 | } | ||
109 | |||
110 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
111 | logger.info('Creating job to dislike %s.', video.url) | ||
112 | |||
113 | const url = getVideoDislikeActivityPubUrl(byActor, video) | ||
114 | const dislikeActivity = buildDislikeActivity(byActor, video) | ||
115 | |||
116 | return sendVideoRelatedCreateActivity({ | ||
117 | byActor, | ||
118 | video, | ||
119 | url, | ||
120 | object: dislikeActivity, | ||
121 | transaction: t | ||
122 | }) | ||
123 | } | ||
124 | |||
125 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { | 75 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { |
126 | if (!audience) audience = getAudience(byActor) | 76 | if (!audience) audience = getAudience(byActor) |
127 | 77 | ||
@@ -136,31 +86,11 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud | |||
136 | ) | 86 | ) |
137 | } | 87 | } |
138 | 88 | ||
139 | function buildDislikeActivity (byActor: ActorModel, video: VideoModel) { | ||
140 | return { | ||
141 | type: 'Dislike', | ||
142 | actor: byActor.url, | ||
143 | object: video.url | ||
144 | } | ||
145 | } | ||
146 | |||
147 | function buildViewActivity (byActor: ActorModel, video: VideoModel) { | ||
148 | return { | ||
149 | type: 'View', | ||
150 | actor: byActor.url, | ||
151 | object: video.url | ||
152 | } | ||
153 | } | ||
154 | |||
155 | // --------------------------------------------------------------------------- | 89 | // --------------------------------------------------------------------------- |
156 | 90 | ||
157 | export { | 91 | export { |
158 | sendCreateVideo, | 92 | sendCreateVideo, |
159 | sendVideoAbuse, | ||
160 | buildCreateActivity, | 93 | buildCreateActivity, |
161 | sendCreateView, | ||
162 | sendCreateDislike, | ||
163 | buildDislikeActivity, | ||
164 | sendCreateVideoComment, | 94 | sendCreateVideoComment, |
165 | sendCreateCacheFile | 95 | sendCreateCacheFile |
166 | } | 96 | } |
diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts new file mode 100644 index 000000000..a88436f2c --- /dev/null +++ b/server/lib/activitypub/send/send-dislike.ts | |||
@@ -0,0 +1,41 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { ActorModel } from '../../../models/activitypub/actor' | ||
3 | import { VideoModel } from '../../../models/video/video' | ||
4 | import { getVideoDislikeActivityPubUrl } from '../url' | ||
5 | import { logger } from '../../../helpers/logger' | ||
6 | import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub' | ||
7 | import { sendVideoRelatedActivity } from './utils' | ||
8 | import { audiencify, getAudience } from '../audience' | ||
9 | |||
10 | async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
11 | logger.info('Creating job to dislike %s.', video.url) | ||
12 | |||
13 | const activityBuilder = (audience: ActivityAudience) => { | ||
14 | const url = getVideoDislikeActivityPubUrl(byActor, video) | ||
15 | |||
16 | return buildDislikeActivity(url, byActor, video, audience) | ||
17 | } | ||
18 | |||
19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) | ||
20 | } | ||
21 | |||
22 | function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike { | ||
23 | if (!audience) audience = getAudience(byActor) | ||
24 | |||
25 | return audiencify( | ||
26 | { | ||
27 | id: url, | ||
28 | type: 'Dislike' as 'Dislike', | ||
29 | actor: byActor.url, | ||
30 | object: video.url | ||
31 | }, | ||
32 | audience | ||
33 | ) | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | sendDislike, | ||
40 | buildDislikeActivity | ||
41 | } | ||
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts new file mode 100644 index 000000000..96a7311b9 --- /dev/null +++ b/server/lib/activitypub/send/send-flag.ts | |||
@@ -0,0 +1,39 @@ | |||
1 | import { ActorModel } from '../../../models/activitypub/actor' | ||
2 | import { VideoModel } from '../../../models/video/video' | ||
3 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
4 | import { getVideoAbuseActivityPubUrl } from '../url' | ||
5 | import { unicastTo } from './utils' | ||
6 | import { logger } from '../../../helpers/logger' | ||
7 | import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub' | ||
8 | import { audiencify, getAudience } from '../audience' | ||
9 | |||
10 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { | ||
11 | if (!video.VideoChannel.Account.Actor.serverId) return // Local user | ||
12 | |||
13 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | ||
14 | |||
15 | logger.info('Creating job to send video abuse %s.', url) | ||
16 | |||
17 | // Custom audience, we only send the abuse to the origin instance | ||
18 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | ||
19 | const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience) | ||
20 | |||
21 | return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
22 | } | ||
23 | |||
24 | function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag { | ||
25 | if (!audience) audience = getAudience(byActor) | ||
26 | |||
27 | const activity = Object.assign( | ||
28 | { id: url, actor: byActor.url }, | ||
29 | videoAbuse.toActivityPubObject() | ||
30 | ) | ||
31 | |||
32 | return audiencify(activity, audience) | ||
33 | } | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | export { | ||
38 | sendVideoAbuse | ||
39 | } | ||
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index 89307acc6..35227887a 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts | |||
@@ -24,8 +24,8 @@ function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, | |||
24 | 24 | ||
25 | return audiencify( | 25 | return audiencify( |
26 | { | 26 | { |
27 | type: 'Like' as 'Like', | ||
28 | id: url, | 27 | id: url, |
28 | type: 'Like' as 'Like', | ||
29 | actor: byActor.url, | 29 | actor: byActor.url, |
30 | object: video.url | 30 | object: video.url |
31 | }, | 31 | }, |
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 5236d2cb3..ecbf605d6 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' | |||
2 | import { | 2 | import { |
3 | ActivityAnnounce, | 3 | ActivityAnnounce, |
4 | ActivityAudience, | 4 | ActivityAudience, |
5 | ActivityCreate, | 5 | ActivityCreate, ActivityDislike, |
6 | ActivityFollow, | 6 | ActivityFollow, |
7 | ActivityLike, | 7 | ActivityLike, |
8 | ActivityUndo | 8 | ActivityUndo |
@@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video' | |||
13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' | 13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' |
14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
15 | import { audiencify, getAudience } from '../audience' | 15 | import { audiencify, getAudience } from '../audience' |
16 | import { buildCreateActivity, buildDislikeActivity } from './send-create' | 16 | import { buildCreateActivity } from './send-create' |
17 | import { buildFollowActivity } from './send-follow' | 17 | import { buildFollowActivity } from './send-follow' |
18 | import { buildLikeActivity } from './send-like' | 18 | import { buildLikeActivity } from './send-like' |
19 | import { VideoShareModel } from '../../../models/video/video-share' | 19 | import { VideoShareModel } from '../../../models/video/video-share' |
20 | import { buildAnnounceWithVideoAudience } from './send-announce' | 20 | import { buildAnnounceWithVideoAudience } from './send-announce' |
21 | import { logger } from '../../../helpers/logger' | 21 | import { logger } from '../../../helpers/logger' |
22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
23 | import { buildDislikeActivity } from './send-dislike' | ||
23 | 24 | ||
24 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | 25 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { |
25 | const me = actorFollow.ActorFollower | 26 | const me = actorFollow.ActorFollower |
@@ -64,16 +65,16 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans | |||
64 | logger.info('Creating job to undo a dislike of video %s.', video.url) | 65 | logger.info('Creating job to undo a dislike of video %s.', video.url) |
65 | 66 | ||
66 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) | 67 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) |
67 | const dislikeActivity = buildDislikeActivity(byActor, video) | 68 | const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) |
68 | const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) | ||
69 | 69 | ||
70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) | 70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) |
71 | } | 71 | } |
72 | 72 | ||
73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | 73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { |
74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | 74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) |
75 | 75 | ||
76 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 76 | const videoId = redundancyModel.getVideo().id |
77 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
77 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | 78 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) |
78 | 79 | ||
79 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) | 80 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) |
@@ -94,7 +95,7 @@ export { | |||
94 | function undoActivityData ( | 95 | function undoActivityData ( |
95 | url: string, | 96 | url: string, |
96 | byActor: ActorModel, | 97 | byActor: ActorModel, |
97 | object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | 98 | object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, |
98 | audience?: ActivityAudience | 99 | audience?: ActivityAudience |
99 | ): ActivityUndo { | 100 | ): ActivityUndo { |
100 | if (!audience) audience = getAudience(byActor) | 101 | if (!audience) audience = getAudience(byActor) |
@@ -114,7 +115,7 @@ async function sendUndoVideoRelatedActivity (options: { | |||
114 | byActor: ActorModel, | 115 | byActor: ActorModel, |
115 | video: VideoModel, | 116 | video: VideoModel, |
116 | url: string, | 117 | url: string, |
117 | activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | 118 | activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, |
118 | transaction: Transaction | 119 | transaction: Transaction |
119 | }) { | 120 | }) { |
120 | const activityBuilder = (audience: ActivityAudience) => { | 121 | const activityBuilder = (audience: ActivityAudience) => { |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..839f66470 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | 61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { |
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | 62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) |
63 | 63 | ||
64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) |
65 | 65 | ||
66 | const activityBuilder = (audience: ActivityAudience) => { | 66 | const activityBuilder = (audience: ActivityAudience) => { |
67 | const redundancyObject = redundancyModel.toActivityPubObject() | 67 | const redundancyObject = redundancyModel.toActivityPubObject() |
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts new file mode 100644 index 000000000..8ad126be0 --- /dev/null +++ b/server/lib/activitypub/send/send-view.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' | ||
3 | import { ActorModel } from '../../../models/activitypub/actor' | ||
4 | import { VideoModel } from '../../../models/video/video' | ||
5 | import { getVideoLikeActivityPubUrl } from '../url' | ||
6 | import { sendVideoRelatedActivity } from './utils' | ||
7 | import { audiencify, getAudience } from '../audience' | ||
8 | import { logger } from '../../../helpers/logger' | ||
9 | |||
10 | async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
11 | logger.info('Creating job to send view of %s.', video.url) | ||
12 | |||
13 | const activityBuilder = (audience: ActivityAudience) => { | ||
14 | const url = getVideoLikeActivityPubUrl(byActor, video) | ||
15 | |||
16 | return buildViewActivity(url, byActor, video, audience) | ||
17 | } | ||
18 | |||
19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) | ||
20 | } | ||
21 | |||
22 | function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView { | ||
23 | if (!audience) audience = getAudience(byActor) | ||
24 | |||
25 | return audiencify( | ||
26 | { | ||
27 | id: url, | ||
28 | type: 'View' as 'View', | ||
29 | actor: byActor.url, | ||
30 | object: video.url | ||
31 | }, | ||
32 | audience | ||
33 | ) | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | sendView | ||
40 | } | ||
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 3ff60a97c..1767df0ae 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts | |||
@@ -4,13 +4,14 @@ import { getServerActor } from '../../helpers/utils' | |||
4 | import { VideoModel } from '../../models/video/video' | 4 | import { VideoModel } from '../../models/video/video' |
5 | import { VideoShareModel } from '../../models/video/video-share' | 5 | import { VideoShareModel } from '../../models/video/video-share' |
6 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' | 6 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' |
7 | import { getAnnounceActivityPubUrl } from './url' | 7 | import { getVideoAnnounceActivityPubUrl } from './url' |
8 | import { VideoChannelModel } from '../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../models/video/video-channel' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { doRequest } from '../../helpers/requests' | 10 | import { doRequest } from '../../helpers/requests' |
11 | import { getOrCreateActorAndServerAndModel } from './actor' | 11 | import { getOrCreateActorAndServerAndModel } from './actor' |
12 | import { logger } from '../../helpers/logger' | 12 | import { logger } from '../../helpers/logger' |
13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' |
14 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | ||
14 | 15 | ||
15 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { | 16 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { |
16 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | 17 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined |
@@ -38,9 +39,13 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { | |||
38 | json: true, | 39 | json: true, |
39 | activityPub: true | 40 | activityPub: true |
40 | }) | 41 | }) |
41 | if (!body || !body.actor) throw new Error('Body of body actor is invalid') | 42 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') |
43 | |||
44 | const actorUrl = getAPId(body.actor) | ||
45 | if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { | ||
46 | throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) | ||
47 | } | ||
42 | 48 | ||
43 | const actorUrl = body.actor | ||
44 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 49 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) |
45 | 50 | ||
46 | const entry = { | 51 | const entry = { |
@@ -72,8 +77,8 @@ export { | |||
72 | async function shareByServer (video: VideoModel, t: Transaction) { | 77 | async function shareByServer (video: VideoModel, t: Transaction) { |
73 | const serverActor = await getServerActor() | 78 | const serverActor = await getServerActor() |
74 | 79 | ||
75 | const serverShareUrl = getAnnounceActivityPubUrl(video.url, serverActor) | 80 | const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video) |
76 | return VideoShareModel.findOrCreate({ | 81 | const [ serverShare ] = await VideoShareModel.findOrCreate({ |
77 | defaults: { | 82 | defaults: { |
78 | actorId: serverActor.id, | 83 | actorId: serverActor.id, |
79 | videoId: video.id, | 84 | videoId: video.id, |
@@ -83,16 +88,14 @@ async function shareByServer (video: VideoModel, t: Transaction) { | |||
83 | url: serverShareUrl | 88 | url: serverShareUrl |
84 | }, | 89 | }, |
85 | transaction: t | 90 | transaction: t |
86 | }).then(([ serverShare, created ]) => { | ||
87 | if (created) return sendVideoAnnounce(serverActor, serverShare, video, t) | ||
88 | |||
89 | return undefined | ||
90 | }) | 91 | }) |
92 | |||
93 | return sendVideoAnnounce(serverActor, serverShare, video, t) | ||
91 | } | 94 | } |
92 | 95 | ||
93 | async function shareByVideoChannel (video: VideoModel, t: Transaction) { | 96 | async function shareByVideoChannel (video: VideoModel, t: Transaction) { |
94 | const videoChannelShareUrl = getAnnounceActivityPubUrl(video.url, video.VideoChannel.Actor) | 97 | const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) |
95 | return VideoShareModel.findOrCreate({ | 98 | const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ |
96 | defaults: { | 99 | defaults: { |
97 | actorId: video.VideoChannel.actorId, | 100 | actorId: video.VideoChannel.actorId, |
98 | videoId: video.id, | 101 | videoId: video.id, |
@@ -102,11 +105,9 @@ async function shareByVideoChannel (video: VideoModel, t: Transaction) { | |||
102 | url: videoChannelShareUrl | 105 | url: videoChannelShareUrl |
103 | }, | 106 | }, |
104 | transaction: t | 107 | transaction: t |
105 | }).then(([ videoChannelShare, created ]) => { | ||
106 | if (created) return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) | ||
107 | |||
108 | return undefined | ||
109 | }) | 108 | }) |
109 | |||
110 | return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) | ||
110 | } | 111 | } |
111 | 112 | ||
112 | async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { | 113 | async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index e792be698..4229fe094 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video' | |||
5 | import { VideoAbuseModel } from '../../models/video/video-abuse' | 5 | import { VideoAbuseModel } from '../../models/video/video-abuse' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoFileModel } from '../../models/video/video-file' | 7 | import { VideoFileModel } from '../../models/video/video-file' |
8 | import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' | ||
9 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
8 | 10 | ||
9 | function getVideoActivityPubUrl (video: VideoModel) { | 11 | function getVideoActivityPubUrl (video: VideoModel) { |
10 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 12 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid |
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { | |||
16 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | 18 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` |
17 | } | 19 | } |
18 | 20 | ||
21 | function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
22 | return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}` | ||
23 | } | ||
24 | |||
19 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { | 25 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { |
20 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | 26 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id |
21 | } | 27 | } |
@@ -33,14 +39,14 @@ function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) { | |||
33 | } | 39 | } |
34 | 40 | ||
35 | function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { | 41 | function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { |
36 | return video.url + '/views/' + byActor.uuid + '/' + new Date().toISOString() | 42 | return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString() |
37 | } | 43 | } |
38 | 44 | ||
39 | function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { | 45 | function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) { |
40 | return byActor.url + '/likes/' + video.id | 46 | return byActor.url + '/likes/' + video.id |
41 | } | 47 | } |
42 | 48 | ||
43 | function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { | 49 | function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) { |
44 | return byActor.url + '/dislikes/' + video.id | 50 | return byActor.url + '/dislikes/' + video.id |
45 | } | 51 | } |
46 | 52 | ||
@@ -74,8 +80,8 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) { | |||
74 | return follower.url + '/accepts/follows/' + me.id | 80 | return follower.url + '/accepts/follows/' + me.id |
75 | } | 81 | } |
76 | 82 | ||
77 | function getAnnounceActivityPubUrl (originalUrl: string, byActor: ActorModel) { | 83 | function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) { |
78 | return originalUrl + '/announces/' + byActor.id | 84 | return video.url + '/announces/' + byActor.id |
79 | } | 85 | } |
80 | 86 | ||
81 | function getDeleteActivityPubUrl (originalUrl: string) { | 87 | function getDeleteActivityPubUrl (originalUrl: string) { |
@@ -92,12 +98,13 @@ function getUndoActivityPubUrl (originalUrl: string) { | |||
92 | 98 | ||
93 | export { | 99 | export { |
94 | getVideoActivityPubUrl, | 100 | getVideoActivityPubUrl, |
101 | getVideoCacheStreamingPlaylistActivityPubUrl, | ||
95 | getVideoChannelActivityPubUrl, | 102 | getVideoChannelActivityPubUrl, |
96 | getAccountActivityPubUrl, | 103 | getAccountActivityPubUrl, |
97 | getVideoAbuseActivityPubUrl, | 104 | getVideoAbuseActivityPubUrl, |
98 | getActorFollowActivityPubUrl, | 105 | getActorFollowActivityPubUrl, |
99 | getActorFollowAcceptActivityPubUrl, | 106 | getActorFollowAcceptActivityPubUrl, |
100 | getAnnounceActivityPubUrl, | 107 | getVideoAnnounceActivityPubUrl, |
101 | getUpdateActivityPubUrl, | 108 | getUpdateActivityPubUrl, |
102 | getUndoActivityPubUrl, | 109 | getUndoActivityPubUrl, |
103 | getVideoViewActivityPubUrl, | 110 | getVideoViewActivityPubUrl, |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index c8c17f4c4..e87301fe7 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -9,6 +9,7 @@ import { VideoCommentModel } from '../../models/video/video-comment' | |||
9 | import { getOrCreateActorAndServerAndModel } from './actor' | 9 | import { getOrCreateActorAndServerAndModel } from './actor' |
10 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | 10 | import { getOrCreateVideoAndAccountAndChannel } from './videos' |
11 | import * as Bluebird from 'bluebird' | 11 | import * as Bluebird from 'bluebird' |
12 | import { checkUrlsSameHost } from '../../helpers/activitypub' | ||
12 | 13 | ||
13 | async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { | 14 | async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { |
14 | let originCommentId: number = null | 15 | let originCommentId: number = null |
@@ -61,7 +62,15 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { | |||
61 | const actorUrl = body.attributedTo | 62 | const actorUrl = body.attributedTo |
62 | if (!actorUrl) return { created: false } | 63 | if (!actorUrl) return { created: false } |
63 | 64 | ||
64 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 65 | if (checkUrlsSameHost(commentUrl, actorUrl) !== true) { |
66 | throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${commentUrl}`) | ||
67 | } | ||
68 | |||
69 | if (checkUrlsSameHost(body.id, commentUrl) !== true) { | ||
70 | throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`) | ||
71 | } | ||
72 | |||
73 | const actor = await getOrCreateActorAndServerAndModel(actorUrl, 'all') | ||
65 | const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) | 74 | const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) |
66 | if (!entry) return { created: false } | 75 | if (!entry) return { created: false } |
67 | 76 | ||
@@ -71,6 +80,8 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { | |||
71 | }, | 80 | }, |
72 | defaults: entry | 81 | defaults: entry |
73 | }) | 82 | }) |
83 | comment.Account = actor.Account | ||
84 | comment.Video = videoInstance | ||
74 | 85 | ||
75 | return { comment, created } | 86 | return { comment, created } |
76 | } | 87 | } |
@@ -134,6 +145,14 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { | |||
134 | const actorUrl = body.attributedTo | 145 | const actorUrl = body.attributedTo |
135 | if (!actorUrl) throw new Error('Miss attributed to in comment') | 146 | if (!actorUrl) throw new Error('Miss attributed to in comment') |
136 | 147 | ||
148 | if (checkUrlsSameHost(url, actorUrl) !== true) { | ||
149 | throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) | ||
150 | } | ||
151 | |||
152 | if (checkUrlsSameHost(body.id, url) !== true) { | ||
153 | throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) | ||
154 | } | ||
155 | |||
137 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 156 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) |
138 | const comment = new VideoCommentModel({ | 157 | const comment = new VideoCommentModel({ |
139 | url: body.id, | 158 | url: body.id, |
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 1619251c3..7aac79118 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts | |||
@@ -1,20 +1,43 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { AccountModel } from '../../models/account/account' | 2 | import { AccountModel } from '../../models/account/account' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' | 4 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' |
5 | import { VideoRateType } from '../../../shared/models/videos' | 5 | import { VideoRateType } from '../../../shared/models/videos' |
6 | import * as Bluebird from 'bluebird' | 6 | import * as Bluebird from 'bluebird' |
7 | import { getOrCreateActorAndServerAndModel } from './actor' | 7 | import { getOrCreateActorAndServerAndModel } from './actor' |
8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | 8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' |
9 | import { logger } from '../../helpers/logger' | 9 | import { logger } from '../../helpers/logger' |
10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' |
11 | import { doRequest } from '../../helpers/requests' | ||
12 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | ||
13 | import { ActorModel } from '../../models/activitypub/actor' | ||
14 | import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' | ||
15 | import { sendDislike } from './send/send-dislike' | ||
11 | 16 | ||
12 | async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { | 17 | async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { |
13 | let rateCounts = 0 | 18 | let rateCounts = 0 |
14 | 19 | ||
15 | await Bluebird.map(actorUrls, async actorUrl => { | 20 | await Bluebird.map(ratesUrl, async rateUrl => { |
16 | try { | 21 | try { |
22 | // Fetch url | ||
23 | const { body } = await doRequest({ | ||
24 | uri: rateUrl, | ||
25 | json: true, | ||
26 | activityPub: true | ||
27 | }) | ||
28 | if (!body || !body.actor) throw new Error('Body or body actor is invalid') | ||
29 | |||
30 | const actorUrl = getAPId(body.actor) | ||
31 | if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { | ||
32 | throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) | ||
33 | } | ||
34 | |||
35 | if (checkUrlsSameHost(body.id, rateUrl) !== true) { | ||
36 | throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) | ||
37 | } | ||
38 | |||
17 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 39 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) |
40 | |||
18 | const [ , created ] = await AccountVideoRateModel | 41 | const [ , created ] = await AccountVideoRateModel |
19 | .findOrCreate({ | 42 | .findOrCreate({ |
20 | where: { | 43 | where: { |
@@ -24,13 +47,14 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR | |||
24 | defaults: { | 47 | defaults: { |
25 | videoId: video.id, | 48 | videoId: video.id, |
26 | accountId: actor.Account.id, | 49 | accountId: actor.Account.id, |
27 | type: rate | 50 | type: rate, |
51 | url: body.id | ||
28 | } | 52 | } |
29 | }) | 53 | }) |
30 | 54 | ||
31 | if (created) rateCounts += 1 | 55 | if (created) rateCounts += 1 |
32 | } catch (err) { | 56 | } catch (err) { |
33 | logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) | 57 | logger.warn('Cannot add rate %s.', rateUrl, { err }) |
34 | } | 58 | } |
35 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | 59 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) |
36 | 60 | ||
@@ -59,10 +83,15 @@ async function sendVideoRateChange (account: AccountModel, | |||
59 | // Like | 83 | // Like |
60 | if (likes > 0) await sendLike(actor, video, t) | 84 | if (likes > 0) await sendLike(actor, video, t) |
61 | // Dislike | 85 | // Dislike |
62 | if (dislikes > 0) await sendCreateDislike(actor, video, t) | 86 | if (dislikes > 0) await sendDislike(actor, video, t) |
87 | } | ||
88 | |||
89 | function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { | ||
90 | return rateType === 'like' ? getVideoLikeActivityPubUrl(actor, video) : getVideoDislikeActivityPubUrl(actor, video) | ||
63 | } | 91 | } |
64 | 92 | ||
65 | export { | 93 | export { |
94 | getRateUrl, | ||
66 | createRates, | 95 | createRates, |
67 | sendVideoRateChange | 96 | sendVideoRateChange |
68 | } | 97 | } |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 8521572a1..710929aac 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -1,17 +1,23 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import * as sequelize from 'sequelize' | 2 | import * as sequelize from 'sequelize' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import { join } from 'path' | ||
5 | import * as request from 'request' | 4 | import * as request from 'request' |
6 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' | 5 | import { |
6 | ActivityIconObject, | ||
7 | ActivityPlaylistSegmentHashesObject, | ||
8 | ActivityPlaylistUrlObject, | ||
9 | ActivityUrlObject, | ||
10 | ActivityVideoUrlObject, | ||
11 | VideoState | ||
12 | } from '../../../shared/index' | ||
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 13 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy } from '../../../shared/models/videos' | 14 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 15 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | 16 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' |
11 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' | 17 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' |
12 | import { logger } from '../../helpers/logger' | 18 | import { logger } from '../../helpers/logger' |
13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | 19 | import { doRequest, downloadImage } from '../../helpers/requests' |
14 | import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' | 20 | import { ACTIVITY_PUB, CONFIG, MIMETYPES, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers' |
15 | import { ActorModel } from '../../models/activitypub/actor' | 21 | import { ActorModel } from '../../models/activitypub/actor' |
16 | import { TagModel } from '../../models/video/tag' | 22 | import { TagModel } from '../../models/video/tag' |
17 | import { VideoModel } from '../../models/video/video' | 23 | import { VideoModel } from '../../models/video/video' |
@@ -29,6 +35,11 @@ import { createRates } from './video-rates' | |||
29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | 35 | import { addVideoShares, shareVideoByServerAndChannel } from './share' |
30 | import { AccountModel } from '../../models/account/account' | 36 | import { AccountModel } from '../../models/account/account' |
31 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | 37 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' |
38 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | ||
39 | import { Notifier } from '../notifier' | ||
40 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
42 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | ||
32 | 43 | ||
33 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 44 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
34 | // If the video is not private and published, we federate it | 45 | // If the video is not private and published, we federate it |
@@ -63,7 +74,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request. | |||
63 | 74 | ||
64 | const { response, body } = await doRequest(options) | 75 | const { response, body } = await doRequest(options) |
65 | 76 | ||
66 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | 77 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { |
67 | logger.debug('Remote video JSON is not valid.', { body }) | 78 | logger.debug('Remote video JSON is not valid.', { body }) |
68 | return { response, videoObject: undefined } | 79 | return { response, videoObject: undefined } |
69 | } | 80 | } |
@@ -94,19 +105,18 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu | |||
94 | 105 | ||
95 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { | 106 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { |
96 | const thumbnailName = video.getThumbnailName() | 107 | const thumbnailName = video.getThumbnailName() |
97 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) | ||
98 | 108 | ||
99 | const options = { | 109 | return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) |
100 | method: 'GET', | ||
101 | uri: icon.url | ||
102 | } | ||
103 | return doRequestAndSaveToFile(options, thumbnailPath) | ||
104 | } | 110 | } |
105 | 111 | ||
106 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { | 112 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { |
107 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | 113 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') |
108 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | 114 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) |
109 | 115 | ||
116 | if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { | ||
117 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) | ||
118 | } | ||
119 | |||
110 | return getOrCreateActorAndServerAndModel(channel.id, 'all') | 120 | return getOrCreateActorAndServerAndModel(channel.id, 'all') |
111 | } | 121 | } |
112 | 122 | ||
@@ -116,7 +126,7 @@ type SyncParam = { | |||
116 | shares: boolean | 126 | shares: boolean |
117 | comments: boolean | 127 | comments: boolean |
118 | thumbnail: boolean | 128 | thumbnail: boolean |
119 | refreshVideo: boolean | 129 | refreshVideo?: boolean |
120 | } | 130 | } |
121 | async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { | 131 | async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { |
122 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | 132 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) |
@@ -155,31 +165,34 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid | |||
155 | } | 165 | } |
156 | 166 | ||
157 | async function getOrCreateVideoAndAccountAndChannel (options: { | 167 | async function getOrCreateVideoAndAccountAndChannel (options: { |
158 | videoObject: VideoTorrentObject | string, | 168 | videoObject: { id: string } | string, |
159 | syncParam?: SyncParam, | 169 | syncParam?: SyncParam, |
160 | fetchType?: VideoFetchByUrlType, | 170 | fetchType?: VideoFetchByUrlType, |
161 | refreshViews?: boolean | 171 | allowRefresh?: boolean // true by default |
162 | }) { | 172 | }) { |
163 | // Default params | 173 | // Default params |
164 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | 174 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } |
165 | const fetchType = options.fetchType || 'all' | 175 | const fetchType = options.fetchType || 'all' |
166 | const refreshViews = options.refreshViews || false | 176 | const allowRefresh = options.allowRefresh !== false |
167 | 177 | ||
168 | // Get video url | 178 | // Get video url |
169 | const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id | 179 | const videoUrl = getAPId(options.videoObject) |
170 | 180 | ||
171 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | 181 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) |
172 | if (videoFromDatabase) { | 182 | if (videoFromDatabase) { |
173 | const refreshOptions = { | 183 | |
174 | video: videoFromDatabase, | 184 | if (allowRefresh === true) { |
175 | fetchedType: fetchType, | 185 | const refreshOptions = { |
176 | syncParam, | 186 | video: videoFromDatabase, |
177 | refreshViews | 187 | fetchedType: fetchType, |
188 | syncParam | ||
189 | } | ||
190 | |||
191 | if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | ||
192 | else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) | ||
178 | } | 193 | } |
179 | const p = refreshVideoIfNeeded(refreshOptions) | ||
180 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
181 | 194 | ||
182 | return { video: videoFromDatabase } | 195 | return { video: videoFromDatabase, created: false } |
183 | } | 196 | } |
184 | 197 | ||
185 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | 198 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) |
@@ -190,7 +203,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { | |||
190 | 203 | ||
191 | await syncVideoExternalAttributes(video, fetchedVideo, syncParam) | 204 | await syncVideoExternalAttributes(video, fetchedVideo, syncParam) |
192 | 205 | ||
193 | return { video } | 206 | return { video, created: true } |
194 | } | 207 | } |
195 | 208 | ||
196 | async function updateVideoFromAP (options: { | 209 | async function updateVideoFromAP (options: { |
@@ -198,17 +211,17 @@ async function updateVideoFromAP (options: { | |||
198 | videoObject: VideoTorrentObject, | 211 | videoObject: VideoTorrentObject, |
199 | account: AccountModel, | 212 | account: AccountModel, |
200 | channel: VideoChannelModel, | 213 | channel: VideoChannelModel, |
201 | updateViews: boolean, | ||
202 | overrideTo?: string[] | 214 | overrideTo?: string[] |
203 | }) { | 215 | }) { |
204 | logger.debug('Updating remote video "%s".', options.videoObject.uuid) | 216 | logger.debug('Updating remote video "%s".', options.videoObject.uuid) |
217 | |||
205 | let videoFieldsSave: any | 218 | let videoFieldsSave: any |
219 | const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE | ||
220 | const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED | ||
206 | 221 | ||
207 | try { | 222 | try { |
208 | await sequelizeTypescript.transaction(async t => { | 223 | await sequelizeTypescript.transaction(async t => { |
209 | const sequelizeOptions = { | 224 | const sequelizeOptions = { transaction: t } |
210 | transaction: t | ||
211 | } | ||
212 | 225 | ||
213 | videoFieldsSave = options.video.toJSON() | 226 | videoFieldsSave = options.video.toJSON() |
214 | 227 | ||
@@ -238,14 +251,10 @@ async function updateVideoFromAP (options: { | |||
238 | options.video.set('publishedAt', videoData.publishedAt) | 251 | options.video.set('publishedAt', videoData.publishedAt) |
239 | options.video.set('privacy', videoData.privacy) | 252 | options.video.set('privacy', videoData.privacy) |
240 | options.video.set('channelId', videoData.channelId) | 253 | options.video.set('channelId', videoData.channelId) |
254 | options.video.set('views', videoData.views) | ||
241 | 255 | ||
242 | if (options.updateViews === true) options.video.set('views', videoData.views) | ||
243 | await options.video.save(sequelizeOptions) | 256 | await options.video.save(sequelizeOptions) |
244 | 257 | ||
245 | // Don't block on request | ||
246 | generateThumbnailFromUrl(options.video, options.videoObject.icon) | ||
247 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) | ||
248 | |||
249 | { | 258 | { |
250 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) | 259 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) |
251 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) | 260 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) |
@@ -266,6 +275,25 @@ async function updateVideoFromAP (options: { | |||
266 | } | 275 | } |
267 | 276 | ||
268 | { | 277 | { |
278 | const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) | ||
279 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
280 | |||
281 | // Remove video files that do not exist anymore | ||
282 | const destroyTasks = options.video.VideoStreamingPlaylists | ||
283 | .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) | ||
284 | .map(f => f.destroy(sequelizeOptions)) | ||
285 | await Promise.all(destroyTasks) | ||
286 | |||
287 | // Update or add other one | ||
288 | const upsertTasks = streamingPlaylistAttributes.map(a => { | ||
289 | return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) | ||
290 | .then(([ streamingPlaylist ]) => streamingPlaylist) | ||
291 | }) | ||
292 | |||
293 | options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) | ||
294 | } | ||
295 | |||
296 | { | ||
269 | // Update Tags | 297 | // Update Tags |
270 | const tags = options.videoObject.tag.map(tag => tag.name) | 298 | const tags = options.videoObject.tag.map(tag => tag.name) |
271 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 299 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
@@ -283,6 +311,11 @@ async function updateVideoFromAP (options: { | |||
283 | } | 311 | } |
284 | }) | 312 | }) |
285 | 313 | ||
314 | // Notify our users? | ||
315 | if (wasPrivateVideo || wasUnlistedVideo) { | ||
316 | Notifier.Instance.notifyOnNewVideo(options.video) | ||
317 | } | ||
318 | |||
286 | logger.info('Remote video with uuid %s updated', options.videoObject.uuid) | 319 | logger.info('Remote video with uuid %s updated', options.videoObject.uuid) |
287 | } catch (err) { | 320 | } catch (err) { |
288 | if (options.video !== undefined && videoFieldsSave !== undefined) { | 321 | if (options.video !== undefined && videoFieldsSave !== undefined) { |
@@ -293,10 +326,66 @@ async function updateVideoFromAP (options: { | |||
293 | logger.debug('Cannot update the remote video.', { err }) | 326 | logger.debug('Cannot update the remote video.', { err }) |
294 | throw err | 327 | throw err |
295 | } | 328 | } |
329 | |||
330 | try { | ||
331 | await generateThumbnailFromUrl(options.video, options.videoObject.icon) | ||
332 | } catch (err) { | ||
333 | logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }) | ||
334 | } | ||
335 | } | ||
336 | |||
337 | async function refreshVideoIfNeeded (options: { | ||
338 | video: VideoModel, | ||
339 | fetchedType: VideoFetchByUrlType, | ||
340 | syncParam: SyncParam | ||
341 | }): Promise<VideoModel> { | ||
342 | if (!options.video.isOutdated()) return options.video | ||
343 | |||
344 | // We need more attributes if the argument video was fetched with not enough joints | ||
345 | const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
346 | |||
347 | try { | ||
348 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
349 | if (response.statusCode === 404) { | ||
350 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
351 | |||
352 | // Video does not exist anymore | ||
353 | await video.destroy() | ||
354 | return undefined | ||
355 | } | ||
356 | |||
357 | if (videoObject === undefined) { | ||
358 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
359 | |||
360 | await video.setAsRefreshed() | ||
361 | return video | ||
362 | } | ||
363 | |||
364 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
365 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
366 | |||
367 | const updateOptions = { | ||
368 | video, | ||
369 | videoObject, | ||
370 | account, | ||
371 | channel: channelActor.VideoChannel | ||
372 | } | ||
373 | await retryTransactionWrapper(updateVideoFromAP, updateOptions) | ||
374 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
375 | |||
376 | return video | ||
377 | } catch (err) { | ||
378 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
379 | |||
380 | // Don't refresh in loop | ||
381 | await video.setAsRefreshed() | ||
382 | return video | ||
383 | } | ||
296 | } | 384 | } |
297 | 385 | ||
298 | export { | 386 | export { |
299 | updateVideoFromAP, | 387 | updateVideoFromAP, |
388 | refreshVideoIfNeeded, | ||
300 | federateVideoIfNeeded, | 389 | federateVideoIfNeeded, |
301 | fetchRemoteVideo, | 390 | fetchRemoteVideo, |
302 | getOrCreateVideoAndAccountAndChannel, | 391 | getOrCreateVideoAndAccountAndChannel, |
@@ -308,10 +397,23 @@ export { | |||
308 | 397 | ||
309 | // --------------------------------------------------------------------------- | 398 | // --------------------------------------------------------------------------- |
310 | 399 | ||
311 | function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { | 400 | function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { |
312 | const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) | 401 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) |
313 | 402 | ||
314 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') | 403 | const urlMediaType = url.mediaType || url.mimeType |
404 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') | ||
405 | } | ||
406 | |||
407 | function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { | ||
408 | const urlMediaType = url.mediaType || url.mimeType | ||
409 | |||
410 | return urlMediaType === 'application/x-mpegURL' | ||
411 | } | ||
412 | |||
413 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
414 | const urlMediaType = tag.mediaType || tag.mimeType | ||
415 | |||
416 | return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' | ||
315 | } | 417 | } |
316 | 418 | ||
317 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 419 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
@@ -334,8 +436,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
334 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | 436 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) |
335 | await Promise.all(videoFilePromises) | 437 | await Promise.all(videoFilePromises) |
336 | 438 | ||
439 | const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) | ||
440 | const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) | ||
441 | await Promise.all(playlistPromises) | ||
442 | |||
337 | // Process tags | 443 | // Process tags |
338 | const tags = videoObject.tag.map(t => t.name) | 444 | const tags = videoObject.tag |
445 | .filter(t => t.type === 'Hashtag') | ||
446 | .map(t => t.name) | ||
339 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 447 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
340 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | 448 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) |
341 | 449 | ||
@@ -359,52 +467,6 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
359 | return videoCreated | 467 | return videoCreated |
360 | } | 468 | } |
361 | 469 | ||
362 | async function refreshVideoIfNeeded (options: { | ||
363 | video: VideoModel, | ||
364 | fetchedType: VideoFetchByUrlType, | ||
365 | syncParam: SyncParam, | ||
366 | refreshViews: boolean | ||
367 | }): Promise<VideoModel> { | ||
368 | if (!options.video.isOutdated()) return options.video | ||
369 | |||
370 | // We need more attributes if the argument video was fetched with not enough joints | ||
371 | const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
372 | |||
373 | try { | ||
374 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
375 | if (response.statusCode === 404) { | ||
376 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | ||
377 | |||
378 | // Video does not exist anymore | ||
379 | await video.destroy() | ||
380 | return undefined | ||
381 | } | ||
382 | |||
383 | if (videoObject === undefined) { | ||
384 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | ||
385 | return video | ||
386 | } | ||
387 | |||
388 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
389 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
390 | |||
391 | const updateOptions = { | ||
392 | video, | ||
393 | videoObject, | ||
394 | account, | ||
395 | channel: channelActor.VideoChannel, | ||
396 | updateViews: options.refreshViews | ||
397 | } | ||
398 | await retryTransactionWrapper(updateVideoFromAP, updateOptions) | ||
399 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
400 | |||
401 | return video | ||
402 | } catch (err) { | ||
403 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | ||
404 | return video | ||
405 | } | ||
406 | } | ||
407 | |||
408 | async function videoActivityObjectToDBAttributes ( | 470 | async function videoActivityObjectToDBAttributes ( |
409 | videoChannel: VideoChannelModel, | 471 | videoChannel: VideoChannelModel, |
410 | videoObject: VideoTorrentObject, | 472 | videoObject: VideoTorrentObject, |
@@ -460,17 +522,18 @@ async function videoActivityObjectToDBAttributes ( | |||
460 | } | 522 | } |
461 | 523 | ||
462 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | 524 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { |
463 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | 525 | const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] |
464 | 526 | ||
465 | if (fileUrls.length === 0) { | 527 | if (fileUrls.length === 0) { |
466 | throw new Error('Cannot find video files for ' + video.url) | 528 | throw new Error('Cannot find video files for ' + video.url) |
467 | } | 529 | } |
468 | 530 | ||
469 | const attributes: VideoFileModel[] = [] | 531 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] |
470 | for (const fileUrl of fileUrls) { | 532 | for (const fileUrl of fileUrls) { |
471 | // Fetch associated magnet uri | 533 | // Fetch associated magnet uri |
472 | const magnet = videoObject.url.find(u => { | 534 | const magnet = videoObject.url.find(u => { |
473 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | 535 | const mediaType = u.mediaType || u.mimeType |
536 | return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height | ||
474 | }) | 537 | }) |
475 | 538 | ||
476 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | 539 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) |
@@ -480,14 +543,53 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid | |||
480 | throw new Error('Cannot parse magnet URI ' + magnet.href) | 543 | throw new Error('Cannot parse magnet URI ' + magnet.href) |
481 | } | 544 | } |
482 | 545 | ||
546 | const mediaType = fileUrl.mediaType || fileUrl.mimeType | ||
483 | const attribute = { | 547 | const attribute = { |
484 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | 548 | extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ], |
485 | infoHash: parsed.infoHash, | 549 | infoHash: parsed.infoHash, |
486 | resolution: fileUrl.height, | 550 | resolution: fileUrl.height, |
487 | size: fileUrl.size, | 551 | size: fileUrl.size, |
488 | videoId: video.id, | 552 | videoId: video.id, |
489 | fps: fileUrl.fps || -1 | 553 | fps: fileUrl.fps || -1 |
490 | } as VideoFileModel | 554 | } |
555 | |||
556 | attributes.push(attribute) | ||
557 | } | ||
558 | |||
559 | return attributes | ||
560 | } | ||
561 | |||
562 | function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | ||
563 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
564 | if (playlistUrls.length === 0) return [] | ||
565 | |||
566 | const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] | ||
567 | for (const playlistUrlObject of playlistUrls) { | ||
568 | const p2pMediaLoaderInfohashes = playlistUrlObject.tag | ||
569 | .filter(t => t.type === 'Infohash') | ||
570 | .map(t => t.name) | ||
571 | if (p2pMediaLoaderInfohashes.length === 0) { | ||
572 | logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
573 | continue | ||
574 | } | ||
575 | |||
576 | const segmentsSha256UrlObject = playlistUrlObject.tag | ||
577 | .find(t => { | ||
578 | return isAPPlaylistSegmentHashesUrlObject(t) | ||
579 | }) as ActivityPlaylistSegmentHashesObject | ||
580 | if (!segmentsSha256UrlObject) { | ||
581 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
582 | continue | ||
583 | } | ||
584 | |||
585 | const attribute = { | ||
586 | type: VideoStreamingPlaylistType.HLS, | ||
587 | playlistUrl: playlistUrlObject.href, | ||
588 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
589 | p2pMediaLoaderInfohashes, | ||
590 | videoId: video.id | ||
591 | } | ||
592 | |||
491 | attributes.push(attribute) | 593 | attributes.push(attribute) |
492 | } | 594 | } |
493 | 595 | ||
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 4b6bc3185..021426a1a 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts | |||
@@ -7,10 +7,11 @@ import { AccountModel } from '../models/account/account' | |||
7 | import { VideoChannelModel } from '../models/video/video-channel' | 7 | import { VideoChannelModel } from '../models/video/video-channel' |
8 | import { extname, join } from 'path' | 8 | import { extname, join } from 'path' |
9 | import { retryTransactionWrapper } from '../helpers/database-utils' | 9 | import { retryTransactionWrapper } from '../helpers/database-utils' |
10 | import * as uuidv4 from 'uuid/v4' | ||
10 | 11 | ||
11 | async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { | 12 | async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { |
12 | const extension = extname(avatarPhysicalFile.filename) | 13 | const extension = extname(avatarPhysicalFile.filename) |
13 | const avatarName = accountOrChannel.Actor.uuid + extension | 14 | const avatarName = uuidv4() + extension |
14 | const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) | 15 | const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) |
15 | await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) | 16 | await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) |
16 | 17 | ||
diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts new file mode 100644 index 000000000..1633e500c --- /dev/null +++ b/server/lib/blocklist.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { sequelizeTypescript } from '../initializers' | ||
2 | import { AccountBlocklistModel } from '../models/account/account-blocklist' | ||
3 | import { ServerBlocklistModel } from '../models/server/server-blocklist' | ||
4 | |||
5 | function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { | ||
6 | return sequelizeTypescript.transaction(async t => { | ||
7 | return AccountBlocklistModel.upsert({ | ||
8 | accountId: byAccountId, | ||
9 | targetAccountId: targetAccountId | ||
10 | }, { transaction: t }) | ||
11 | }) | ||
12 | } | ||
13 | |||
14 | function addServerInBlocklist (byAccountId: number, targetServerId: number) { | ||
15 | return sequelizeTypescript.transaction(async t => { | ||
16 | return ServerBlocklistModel.upsert({ | ||
17 | accountId: byAccountId, | ||
18 | targetServerId | ||
19 | }, { transaction: t }) | ||
20 | }) | ||
21 | } | ||
22 | |||
23 | function removeAccountFromBlocklist (accountBlock: AccountBlocklistModel) { | ||
24 | return sequelizeTypescript.transaction(async t => { | ||
25 | return accountBlock.destroy({ transaction: t }) | ||
26 | }) | ||
27 | } | ||
28 | |||
29 | function removeServerFromBlocklist (serverBlock: ServerBlocklistModel) { | ||
30 | return sequelizeTypescript.transaction(async t => { | ||
31 | return serverBlock.destroy({ transaction: t }) | ||
32 | }) | ||
33 | } | ||
34 | |||
35 | export { | ||
36 | addAccountInBlocklist, | ||
37 | addServerInBlocklist, | ||
38 | removeAccountFromBlocklist, | ||
39 | removeServerFromBlocklist | ||
40 | } | ||
diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/cache/actor-follow-score-cache.ts new file mode 100644 index 000000000..d070bde09 --- /dev/null +++ b/server/lib/cache/actor-follow-score-cache.ts | |||
@@ -0,0 +1,46 @@ | |||
1 | import { ACTOR_FOLLOW_SCORE } from '../../initializers' | ||
2 | import { logger } from '../../helpers/logger' | ||
3 | |||
4 | // Cache follows scores, instead of writing them too often in database | ||
5 | // Keep data in memory, we don't really need Redis here as we don't really care to loose some scores | ||
6 | class ActorFollowScoreCache { | ||
7 | |||
8 | private static instance: ActorFollowScoreCache | ||
9 | private pendingFollowsScore: { [ url: string ]: number } = {} | ||
10 | |||
11 | private constructor () {} | ||
12 | |||
13 | static get Instance () { | ||
14 | return this.instance || (this.instance = new this()) | ||
15 | } | ||
16 | |||
17 | updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) { | ||
18 | if (goodInboxes.length === 0 && badInboxes.length === 0) return | ||
19 | |||
20 | logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length) | ||
21 | |||
22 | for (const goodInbox of goodInboxes) { | ||
23 | if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0 | ||
24 | |||
25 | this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS | ||
26 | } | ||
27 | |||
28 | for (const badInbox of badInboxes) { | ||
29 | if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0 | ||
30 | |||
31 | this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY | ||
32 | } | ||
33 | } | ||
34 | |||
35 | getPendingFollowsScoreCopy () { | ||
36 | return this.pendingFollowsScore | ||
37 | } | ||
38 | |||
39 | clearPendingFollowsScore () { | ||
40 | this.pendingFollowsScore = {} | ||
41 | } | ||
42 | } | ||
43 | |||
44 | export { | ||
45 | ActorFollowScoreCache | ||
46 | } | ||
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts index 54eb983fa..e921d04a7 100644 --- a/server/lib/cache/index.ts +++ b/server/lib/cache/index.ts | |||
@@ -1,2 +1,3 @@ | |||
1 | export * from './actor-follow-score-cache' | ||
1 | export * from './videos-preview-cache' | 2 | export * from './videos-preview-cache' |
2 | export * from './videos-caption-cache' | 3 | export * from './videos-caption-cache' |
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index fc013e0c3..b2c376e20 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import * as Bluebird from 'bluebird' | 2 | import * as Bluebird from 'bluebird' |
3 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' | 3 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' |
4 | import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, STATIC_PATHS } from '../initializers' | 4 | import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { escapeHTML } from '../helpers/core-utils' | 6 | import { escapeHTML } from '../helpers/core-utils' |
7 | import { VideoModel } from '../models/video/video' | 7 | import { VideoModel } from '../models/video/video' |
@@ -18,21 +18,13 @@ export class ClientHtml { | |||
18 | ClientHtml.htmlCache = {} | 18 | ClientHtml.htmlCache = {} |
19 | } | 19 | } |
20 | 20 | ||
21 | static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { | 21 | static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { |
22 | const path = ClientHtml.getIndexPath(req, res, paramLang) | 22 | const html = await ClientHtml.getIndexHTML(req, res, paramLang) |
23 | if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | ||
24 | |||
25 | const buffer = await readFile(path) | ||
26 | 23 | ||
27 | let html = buffer.toString() | 24 | let customHtml = ClientHtml.addTitleTag(html) |
28 | 25 | customHtml = ClientHtml.addDescriptionTag(customHtml) | |
29 | html = ClientHtml.addTitleTag(html) | ||
30 | html = ClientHtml.addDescriptionTag(html) | ||
31 | html = ClientHtml.addCustomCSS(html) | ||
32 | 26 | ||
33 | ClientHtml.htmlCache[path] = html | 27 | return customHtml |
34 | |||
35 | return html | ||
36 | } | 28 | } |
37 | 29 | ||
38 | static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { | 30 | static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { |
@@ -55,7 +47,26 @@ export class ClientHtml { | |||
55 | return ClientHtml.getIndexHTML(req, res) | 47 | return ClientHtml.getIndexHTML(req, res) |
56 | } | 48 | } |
57 | 49 | ||
58 | return ClientHtml.addOpenGraphAndOEmbedTags(html, video) | 50 | let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name)) |
51 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description)) | ||
52 | customHtml = ClientHtml.addOpenGraphAndOEmbedTags(customHtml, video) | ||
53 | |||
54 | return customHtml | ||
55 | } | ||
56 | |||
57 | private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { | ||
58 | const path = ClientHtml.getIndexPath(req, res, paramLang) | ||
59 | if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | ||
60 | |||
61 | const buffer = await readFile(path) | ||
62 | |||
63 | let html = buffer.toString() | ||
64 | |||
65 | html = ClientHtml.addCustomCSS(html) | ||
66 | |||
67 | ClientHtml.htmlCache[path] = html | ||
68 | |||
69 | return html | ||
59 | } | 70 | } |
60 | 71 | ||
61 | private static getIndexPath (req: express.Request, res: express.Response, paramLang?: string) { | 72 | private static getIndexPath (req: express.Request, res: express.Response, paramLang?: string) { |
@@ -81,14 +92,18 @@ export class ClientHtml { | |||
81 | return join(__dirname, '../../../client/dist/' + buildFileLocale(lang) + '/index.html') | 92 | return join(__dirname, '../../../client/dist/' + buildFileLocale(lang) + '/index.html') |
82 | } | 93 | } |
83 | 94 | ||
84 | private static addTitleTag (htmlStringPage: string) { | 95 | private static addTitleTag (htmlStringPage: string, title?: string) { |
85 | const titleTag = '<title>' + CONFIG.INSTANCE.NAME + '</title>' | 96 | let text = title || CONFIG.INSTANCE.NAME |
97 | if (title) text += ` - ${CONFIG.INSTANCE.NAME}` | ||
98 | |||
99 | const titleTag = `<title>${text}</title>` | ||
86 | 100 | ||
87 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) | 101 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) |
88 | } | 102 | } |
89 | 103 | ||
90 | private static addDescriptionTag (htmlStringPage: string) { | 104 | private static addDescriptionTag (htmlStringPage: string, description?: string) { |
91 | const descriptionTag = `<meta name="description" content="${CONFIG.INSTANCE.SHORT_DESCRIPTION}" />` | 105 | const content = description || CONFIG.INSTANCE.SHORT_DESCRIPTION |
106 | const descriptionTag = `<meta name="description" content="${content}" />` | ||
92 | 107 | ||
93 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) | 108 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) |
94 | } | 109 | } |
@@ -100,8 +115,8 @@ export class ClientHtml { | |||
100 | } | 115 | } |
101 | 116 | ||
102 | private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { | 117 | private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { |
103 | const previewUrl = CONFIG.WEBSERVER.URL + STATIC_PATHS.PREVIEWS + video.getPreviewName() | 118 | const previewUrl = CONFIG.WEBSERVER.URL + video.getPreviewStaticPath() |
104 | const videoUrl = CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 119 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() |
105 | 120 | ||
106 | const videoNameEscaped = escapeHTML(video.name) | 121 | const videoNameEscaped = escapeHTML(video.name) |
107 | const videoDescriptionEscaped = escapeHTML(video.description) | 122 | const videoDescriptionEscaped = escapeHTML(video.description) |
@@ -172,8 +187,8 @@ export class ClientHtml { | |||
172 | // Schema.org | 187 | // Schema.org |
173 | tagsString += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>` | 188 | tagsString += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>` |
174 | 189 | ||
175 | // SEO | 190 | // SEO, use origin video url so Google does not index remote videos |
176 | tagsString += `<link rel="canonical" href="${videoUrl}" />` | 191 | tagsString += `<link rel="canonical" href="${video.url}" />` |
177 | 192 | ||
178 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) | 193 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) |
179 | } | 194 | } |
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 9327792fb..672414cc0 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import { createTransport, Transporter } from 'nodemailer' | 1 | import { createTransport, Transporter } from 'nodemailer' |
2 | import { UserRight } from '../../shared/models/users' | ||
3 | import { isTestInstance } from '../helpers/core-utils' | 2 | import { isTestInstance } from '../helpers/core-utils' |
4 | import { bunyanLogger, logger } from '../helpers/logger' | 3 | import { bunyanLogger, logger } from '../helpers/logger' |
5 | import { CONFIG } from '../initializers' | 4 | import { CONFIG } from '../initializers' |
@@ -8,6 +7,11 @@ import { VideoModel } from '../models/video/video' | |||
8 | import { JobQueue } from './job-queue' | 7 | import { JobQueue } from './job-queue' |
9 | import { EmailPayload } from './job-queue/handlers/email' | 8 | import { EmailPayload } from './job-queue/handlers/email' |
10 | import { readFileSync } from 'fs-extra' | 9 | import { readFileSync } from 'fs-extra' |
10 | import { VideoCommentModel } from '../models/video/video-comment' | ||
11 | import { VideoAbuseModel } from '../models/video/video-abuse' | ||
12 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | ||
13 | import { VideoImportModel } from '../models/video/video-import' | ||
14 | import { ActorFollowModel } from '../models/activitypub/actor-follow' | ||
11 | 15 | ||
12 | class Emailer { | 16 | class Emailer { |
13 | 17 | ||
@@ -22,7 +26,7 @@ class Emailer { | |||
22 | if (this.initialized === true) return | 26 | if (this.initialized === true) return |
23 | this.initialized = true | 27 | this.initialized = true |
24 | 28 | ||
25 | if (CONFIG.SMTP.HOSTNAME && CONFIG.SMTP.PORT) { | 29 | if (Emailer.isEnabled()) { |
26 | logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) | 30 | logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) |
27 | 31 | ||
28 | let tls | 32 | let tls |
@@ -57,6 +61,10 @@ class Emailer { | |||
57 | } | 61 | } |
58 | } | 62 | } |
59 | 63 | ||
64 | static isEnabled () { | ||
65 | return !!CONFIG.SMTP.HOSTNAME && !!CONFIG.SMTP.PORT | ||
66 | } | ||
67 | |||
60 | async checkConnectionOrDie () { | 68 | async checkConnectionOrDie () { |
61 | if (!this.transporter) return | 69 | if (!this.transporter) return |
62 | 70 | ||
@@ -72,50 +80,158 @@ class Emailer { | |||
72 | } | 80 | } |
73 | } | 81 | } |
74 | 82 | ||
75 | addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { | 83 | addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) { |
84 | const channelName = video.VideoChannel.getDisplayName() | ||
85 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() | ||
86 | |||
76 | const text = `Hi dear user,\n\n` + | 87 | const text = `Hi dear user,\n\n` + |
77 | `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + | 88 | `Your subscription ${channelName} just published a new video: ${video.name}` + |
78 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + | 89 | `\n\n` + |
79 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | 90 | `You can view it on ${videoUrl} ` + |
91 | `\n\n` + | ||
80 | `Cheers,\n` + | 92 | `Cheers,\n` + |
81 | `PeerTube.` | 93 | `PeerTube.` |
82 | 94 | ||
83 | const emailPayload: EmailPayload = { | 95 | const emailPayload: EmailPayload = { |
84 | to: [ to ], | 96 | to, |
85 | subject: 'Reset your PeerTube password', | 97 | subject: channelName + ' just published a new video', |
86 | text | 98 | text |
87 | } | 99 | } |
88 | 100 | ||
89 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 101 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
90 | } | 102 | } |
91 | 103 | ||
92 | addVerifyEmailJob (to: string, verifyEmailUrl: string) { | 104 | addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') { |
93 | const text = `Welcome to PeerTube,\n\n` + | 105 | const followerName = actorFollow.ActorFollower.Account.getDisplayName() |
94 | `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + | 106 | const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName() |
95 | `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + | 107 | |
96 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | 108 | const text = `Hi dear user,\n\n` + |
109 | `Your ${followType} ${followingName} has a new subscriber: ${followerName}` + | ||
110 | `\n\n` + | ||
97 | `Cheers,\n` + | 111 | `Cheers,\n` + |
98 | `PeerTube.` | 112 | `PeerTube.` |
99 | 113 | ||
100 | const emailPayload: EmailPayload = { | 114 | const emailPayload: EmailPayload = { |
101 | to: [ to ], | 115 | to, |
102 | subject: 'Verify your PeerTube email', | 116 | subject: 'New follower on your channel ' + followingName, |
117 | text | ||
118 | } | ||
119 | |||
120 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
121 | } | ||
122 | |||
123 | myVideoPublishedNotification (to: string[], video: VideoModel) { | ||
124 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() | ||
125 | |||
126 | const text = `Hi dear user,\n\n` + | ||
127 | `Your video ${video.name} has been published.` + | ||
128 | `\n\n` + | ||
129 | `You can view it on ${videoUrl} ` + | ||
130 | `\n\n` + | ||
131 | `Cheers,\n` + | ||
132 | `PeerTube.` | ||
133 | |||
134 | const emailPayload: EmailPayload = { | ||
135 | to, | ||
136 | subject: `Your video ${video.name} is published`, | ||
137 | text | ||
138 | } | ||
139 | |||
140 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
141 | } | ||
142 | |||
143 | myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) { | ||
144 | const videoUrl = CONFIG.WEBSERVER.URL + videoImport.Video.getWatchStaticPath() | ||
145 | |||
146 | const text = `Hi dear user,\n\n` + | ||
147 | `Your video import ${videoImport.getTargetIdentifier()} is finished.` + | ||
148 | `\n\n` + | ||
149 | `You can view the imported video on ${videoUrl} ` + | ||
150 | `\n\n` + | ||
151 | `Cheers,\n` + | ||
152 | `PeerTube.` | ||
153 | |||
154 | const emailPayload: EmailPayload = { | ||
155 | to, | ||
156 | subject: `Your video import ${videoImport.getTargetIdentifier()} is finished`, | ||
157 | text | ||
158 | } | ||
159 | |||
160 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
161 | } | ||
162 | |||
163 | myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) { | ||
164 | const importUrl = CONFIG.WEBSERVER.URL + '/my-account/video-imports' | ||
165 | |||
166 | const text = `Hi dear user,\n\n` + | ||
167 | `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` + | ||
168 | `\n\n` + | ||
169 | `See your videos import dashboard for more information: ${importUrl}` + | ||
170 | `\n\n` + | ||
171 | `Cheers,\n` + | ||
172 | `PeerTube.` | ||
173 | |||
174 | const emailPayload: EmailPayload = { | ||
175 | to, | ||
176 | subject: `Your video import ${videoImport.getTargetIdentifier()} encountered an error`, | ||
177 | text | ||
178 | } | ||
179 | |||
180 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
181 | } | ||
182 | |||
183 | addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) { | ||
184 | const accountName = comment.Account.getDisplayName() | ||
185 | const video = comment.Video | ||
186 | const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() | ||
187 | |||
188 | const text = `Hi dear user,\n\n` + | ||
189 | `A new comment has been posted by ${accountName} on your video ${video.name}` + | ||
190 | `\n\n` + | ||
191 | `You can view it on ${commentUrl} ` + | ||
192 | `\n\n` + | ||
193 | `Cheers,\n` + | ||
194 | `PeerTube.` | ||
195 | |||
196 | const emailPayload: EmailPayload = { | ||
197 | to, | ||
198 | subject: 'New comment on your video ' + video.name, | ||
103 | text | 199 | text |
104 | } | 200 | } |
105 | 201 | ||
106 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 202 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
107 | } | 203 | } |
108 | 204 | ||
109 | async addVideoAbuseReportJob (videoId: number) { | 205 | addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) { |
110 | const video = await VideoModel.load(videoId) | 206 | const accountName = comment.Account.getDisplayName() |
111 | if (!video) throw new Error('Unknown Video id during Abuse report.') | 207 | const video = comment.Video |
208 | const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() | ||
209 | |||
210 | const text = `Hi dear user,\n\n` + | ||
211 | `${accountName} mentioned you on video ${video.name}` + | ||
212 | `\n\n` + | ||
213 | `You can view the comment on ${commentUrl} ` + | ||
214 | `\n\n` + | ||
215 | `Cheers,\n` + | ||
216 | `PeerTube.` | ||
217 | |||
218 | const emailPayload: EmailPayload = { | ||
219 | to, | ||
220 | subject: 'Mention on video ' + video.name, | ||
221 | text | ||
222 | } | ||
223 | |||
224 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
225 | } | ||
226 | |||
227 | addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { | ||
228 | const videoUrl = CONFIG.WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath() | ||
112 | 229 | ||
113 | const text = `Hi,\n\n` + | 230 | const text = `Hi,\n\n` + |
114 | `Your instance received an abuse for the following video ${video.url}\n\n` + | 231 | `${CONFIG.WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` + |
115 | `Cheers,\n` + | 232 | `Cheers,\n` + |
116 | `PeerTube.` | 233 | `PeerTube.` |
117 | 234 | ||
118 | const to = await UserModel.listEmailsWithRight(UserRight.MANAGE_VIDEO_ABUSES) | ||
119 | const emailPayload: EmailPayload = { | 235 | const emailPayload: EmailPayload = { |
120 | to, | 236 | to, |
121 | subject: '[PeerTube] Received a video abuse', | 237 | subject: '[PeerTube] Received a video abuse', |
@@ -125,16 +241,27 @@ class Emailer { | |||
125 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 241 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
126 | } | 242 | } |
127 | 243 | ||
128 | async addVideoBlacklistReportJob (videoId: number, reason?: string) { | 244 | addNewUserRegistrationNotification (to: string[], user: UserModel) { |
129 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | 245 | const text = `Hi,\n\n` + |
130 | if (!video) throw new Error('Unknown Video id during Blacklist report.') | 246 | `User ${user.username} just registered on ${CONFIG.WEBSERVER.HOST} PeerTube instance.\n\n` + |
131 | // It's not our user | 247 | `Cheers,\n` + |
132 | if (video.remote === true) return | 248 | `PeerTube.` |
249 | |||
250 | const emailPayload: EmailPayload = { | ||
251 | to, | ||
252 | subject: '[PeerTube] New user registration on ' + CONFIG.WEBSERVER.HOST, | ||
253 | text | ||
254 | } | ||
255 | |||
256 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
257 | } | ||
133 | 258 | ||
134 | const user = await UserModel.loadById(video.VideoChannel.Account.userId) | 259 | addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { |
260 | const videoName = videoBlacklist.Video.name | ||
261 | const videoUrl = CONFIG.WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath() | ||
135 | 262 | ||
136 | const reasonString = reason ? ` for the following reason: ${reason}` : '' | 263 | const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : '' |
137 | const blockedString = `Your video ${video.name} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` | 264 | const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` |
138 | 265 | ||
139 | const text = 'Hi,\n\n' + | 266 | const text = 'Hi,\n\n' + |
140 | blockedString + | 267 | blockedString + |
@@ -142,33 +269,26 @@ class Emailer { | |||
142 | 'Cheers,\n' + | 269 | 'Cheers,\n' + |
143 | `PeerTube.` | 270 | `PeerTube.` |
144 | 271 | ||
145 | const to = user.email | ||
146 | const emailPayload: EmailPayload = { | 272 | const emailPayload: EmailPayload = { |
147 | to: [ to ], | 273 | to, |
148 | subject: `[PeerTube] Video ${video.name} blacklisted`, | 274 | subject: `[PeerTube] Video ${videoName} blacklisted`, |
149 | text | 275 | text |
150 | } | 276 | } |
151 | 277 | ||
152 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 278 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
153 | } | 279 | } |
154 | 280 | ||
155 | async addVideoUnblacklistReportJob (videoId: number) { | 281 | addVideoUnblacklistNotification (to: string[], video: VideoModel) { |
156 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | 282 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() |
157 | if (!video) throw new Error('Unknown Video id during Blacklist report.') | ||
158 | // It's not our user | ||
159 | if (video.remote === true) return | ||
160 | |||
161 | const user = await UserModel.loadById(video.VideoChannel.Account.userId) | ||
162 | 283 | ||
163 | const text = 'Hi,\n\n' + | 284 | const text = 'Hi,\n\n' + |
164 | `Your video ${video.name} on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + | 285 | `Your video ${video.name} (${videoUrl}) on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + |
165 | '\n\n' + | 286 | '\n\n' + |
166 | 'Cheers,\n' + | 287 | 'Cheers,\n' + |
167 | `PeerTube.` | 288 | `PeerTube.` |
168 | 289 | ||
169 | const to = user.email | ||
170 | const emailPayload: EmailPayload = { | 290 | const emailPayload: EmailPayload = { |
171 | to: [ to ], | 291 | to, |
172 | subject: `[PeerTube] Video ${video.name} unblacklisted`, | 292 | subject: `[PeerTube] Video ${video.name} unblacklisted`, |
173 | text | 293 | text |
174 | } | 294 | } |
@@ -176,6 +296,40 @@ class Emailer { | |||
176 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 296 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
177 | } | 297 | } |
178 | 298 | ||
299 | addPasswordResetEmailJob (to: string, resetPasswordUrl: string) { | ||
300 | const text = `Hi dear user,\n\n` + | ||
301 | `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` + | ||
302 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + | ||
303 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | ||
304 | `Cheers,\n` + | ||
305 | `PeerTube.` | ||
306 | |||
307 | const emailPayload: EmailPayload = { | ||
308 | to: [ to ], | ||
309 | subject: 'Reset your PeerTube password', | ||
310 | text | ||
311 | } | ||
312 | |||
313 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
314 | } | ||
315 | |||
316 | addVerifyEmailJob (to: string, verifyEmailUrl: string) { | ||
317 | const text = `Welcome to PeerTube,\n\n` + | ||
318 | `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + | ||
319 | `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + | ||
320 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | ||
321 | `Cheers,\n` + | ||
322 | `PeerTube.` | ||
323 | |||
324 | const emailPayload: EmailPayload = { | ||
325 | to: [ to ], | ||
326 | subject: 'Verify your PeerTube email', | ||
327 | text | ||
328 | } | ||
329 | |||
330 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
331 | } | ||
332 | |||
179 | addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { | 333 | addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { |
180 | const reasonString = reason ? ` for the following reason: ${reason}` : '' | 334 | const reasonString = reason ? ` for the following reason: ${reason}` : '' |
181 | const blockedWord = blocked ? 'blocked' : 'unblocked' | 335 | const blockedWord = blocked ? 'blocked' : 'unblocked' |
@@ -197,13 +351,32 @@ class Emailer { | |||
197 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 351 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
198 | } | 352 | } |
199 | 353 | ||
200 | sendMail (to: string[], subject: string, text: string) { | 354 | addContactFormJob (fromEmail: string, fromName: string, body: string) { |
201 | if (!this.transporter) { | 355 | const text = 'Hello dear admin,\n\n' + |
356 | fromName + ' sent you a message' + | ||
357 | '\n\n---------------------------------------\n\n' + | ||
358 | body + | ||
359 | '\n\n---------------------------------------\n\n' + | ||
360 | 'Cheers,\n' + | ||
361 | 'PeerTube.' | ||
362 | |||
363 | const emailPayload: EmailPayload = { | ||
364 | from: fromEmail, | ||
365 | to: [ CONFIG.ADMIN.EMAIL ], | ||
366 | subject: '[PeerTube] Contact form submitted', | ||
367 | text | ||
368 | } | ||
369 | |||
370 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
371 | } | ||
372 | |||
373 | sendMail (to: string[], subject: string, text: string, from?: string) { | ||
374 | if (!Emailer.isEnabled()) { | ||
202 | throw new Error('Cannot send mail because SMTP is not configured.') | 375 | throw new Error('Cannot send mail because SMTP is not configured.') |
203 | } | 376 | } |
204 | 377 | ||
205 | return this.transporter.sendMail({ | 378 | return this.transporter.sendMail({ |
206 | from: CONFIG.SMTP.FROM_ADDRESS, | 379 | from: from || CONFIG.SMTP.FROM_ADDRESS, |
207 | to: to.join(','), | 380 | to: to.join(','), |
208 | subject, | 381 | subject, |
209 | text | 382 | text |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..3575981f4 --- /dev/null +++ b/server/lib/hls.ts | |||
@@ -0,0 +1,164 @@ | |||
1 | import { VideoModel } from '../models/video/video' | ||
2 | import { basename, join, dirname } from 'path' | ||
3 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' | ||
4 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' | ||
5 | import { getVideoFileSize } from '../helpers/ffmpeg-utils' | ||
6 | import { sha256 } from '../helpers/core-utils' | ||
7 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
8 | import { logger } from '../helpers/logger' | ||
9 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | ||
10 | import { generateRandomString } from '../helpers/utils' | ||
11 | import { flatten, uniq } from 'lodash' | ||
12 | |||
13 | async function updateMasterHLSPlaylist (video: VideoModel) { | ||
14 | const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
15 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | ||
16 | const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
17 | |||
18 | for (const file of video.VideoFiles) { | ||
19 | // If we did not generated a playlist for this resolution, skip | ||
20 | const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
21 | if (await pathExists(filePlaylistPath) === false) continue | ||
22 | |||
23 | const videoFilePath = video.getVideoFilePath(file) | ||
24 | |||
25 | const size = await getVideoFileSize(videoFilePath) | ||
26 | |||
27 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
28 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
29 | |||
30 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
31 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
32 | |||
33 | masterPlaylists.push(line) | ||
34 | masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
35 | } | ||
36 | |||
37 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
38 | } | ||
39 | |||
40 | async function updateSha256Segments (video: VideoModel) { | ||
41 | const json: { [filename: string]: { [range: string]: string } } = {} | ||
42 | |||
43 | const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
44 | |||
45 | // For all the resolutions available for this video | ||
46 | for (const file of video.VideoFiles) { | ||
47 | const rangeHashes: { [range: string]: string } = {} | ||
48 | |||
49 | const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) | ||
50 | const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
51 | |||
52 | // Maybe the playlist is not generated for this resolution yet | ||
53 | if (!await pathExists(playlistPath)) continue | ||
54 | |||
55 | const playlistContent = await readFile(playlistPath) | ||
56 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
57 | |||
58 | const fd = await open(videoPath, 'r') | ||
59 | for (const range of ranges) { | ||
60 | const buf = Buffer.alloc(range.length) | ||
61 | await read(fd, buf, 0, range.length, range.offset) | ||
62 | |||
63 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | ||
64 | } | ||
65 | await close(fd) | ||
66 | |||
67 | const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) | ||
68 | json[videoFilename] = rangeHashes | ||
69 | } | ||
70 | |||
71 | const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
72 | await outputJSON(outputPath, json) | ||
73 | } | ||
74 | |||
75 | function getRangesFromPlaylist (playlistContent: string) { | ||
76 | const ranges: { offset: number, length: number }[] = [] | ||
77 | const lines = playlistContent.split('\n') | ||
78 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | ||
79 | |||
80 | for (const line of lines) { | ||
81 | const captured = regex.exec(line) | ||
82 | |||
83 | if (captured) { | ||
84 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | ||
85 | } | ||
86 | } | ||
87 | |||
88 | return ranges | ||
89 | } | ||
90 | |||
91 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | ||
92 | let timer | ||
93 | |||
94 | logger.info('Importing HLS playlist %s', playlistUrl) | ||
95 | |||
96 | return new Promise<string>(async (res, rej) => { | ||
97 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) | ||
98 | |||
99 | await ensureDir(tmpDirectory) | ||
100 | |||
101 | timer = setTimeout(() => { | ||
102 | deleteTmpDirectory(tmpDirectory) | ||
103 | |||
104 | return rej(new Error('HLS download timeout.')) | ||
105 | }, timeout) | ||
106 | |||
107 | try { | ||
108 | // Fetch master playlist | ||
109 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | ||
110 | |||
111 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | ||
112 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | ||
113 | |||
114 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | ||
115 | |||
116 | for (const fileUrl of fileUrls) { | ||
117 | const destPath = join(tmpDirectory, basename(fileUrl)) | ||
118 | |||
119 | await doRequestAndSaveToFile({ uri: fileUrl }, destPath) | ||
120 | } | ||
121 | |||
122 | clearTimeout(timer) | ||
123 | |||
124 | await move(tmpDirectory, destinationDir, { overwrite: true }) | ||
125 | |||
126 | return res() | ||
127 | } catch (err) { | ||
128 | deleteTmpDirectory(tmpDirectory) | ||
129 | |||
130 | return rej(err) | ||
131 | } | ||
132 | }) | ||
133 | |||
134 | function deleteTmpDirectory (directory: string) { | ||
135 | remove(directory) | ||
136 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | ||
137 | } | ||
138 | |||
139 | async function fetchUniqUrls (playlistUrl: string) { | ||
140 | const { body } = await doRequest<string>({ uri: playlistUrl }) | ||
141 | |||
142 | if (!body) return [] | ||
143 | |||
144 | const urls = body.split('\n') | ||
145 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | ||
146 | .map(url => { | ||
147 | if (url.startsWith('http://') || url.startsWith('https://')) return url | ||
148 | |||
149 | return `${dirname(playlistUrl)}/${url}` | ||
150 | }) | ||
151 | |||
152 | return uniq(urls) | ||
153 | } | ||
154 | } | ||
155 | |||
156 | // --------------------------------------------------------------------------- | ||
157 | |||
158 | export { | ||
159 | updateMasterHLSPlaylist, | ||
160 | updateSha256Segments, | ||
161 | downloadPlaylistSegments | ||
162 | } | ||
163 | |||
164 | // --------------------------------------------------------------------------- | ||
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index 36d0f237b..b4d381062 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts | |||
@@ -8,6 +8,7 @@ import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor' | |||
8 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 8 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
9 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 9 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
10 | import { ActorModel } from '../../../models/activitypub/actor' | 10 | import { ActorModel } from '../../../models/activitypub/actor' |
11 | import { Notifier } from '../../notifier' | ||
11 | 12 | ||
12 | export type ActivitypubFollowPayload = { | 13 | export type ActivitypubFollowPayload = { |
13 | followerActorId: number | 14 | followerActorId: number |
@@ -42,7 +43,7 @@ export { | |||
42 | 43 | ||
43 | // --------------------------------------------------------------------------- | 44 | // --------------------------------------------------------------------------- |
44 | 45 | ||
45 | function follow (fromActor: ActorModel, targetActor: ActorModel) { | 46 | async function follow (fromActor: ActorModel, targetActor: ActorModel) { |
46 | if (fromActor.id === targetActor.id) { | 47 | if (fromActor.id === targetActor.id) { |
47 | throw new Error('Follower is the same than target actor.') | 48 | throw new Error('Follower is the same than target actor.') |
48 | } | 49 | } |
@@ -50,7 +51,7 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) { | |||
50 | // Same server, direct accept | 51 | // Same server, direct accept |
51 | const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' | 52 | const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' |
52 | 53 | ||
53 | return sequelizeTypescript.transaction(async t => { | 54 | const actorFollow = await sequelizeTypescript.transaction(async t => { |
54 | const [ actorFollow ] = await ActorFollowModel.findOrCreate({ | 55 | const [ actorFollow ] = await ActorFollowModel.findOrCreate({ |
55 | where: { | 56 | where: { |
56 | actorId: fromActor.id, | 57 | actorId: fromActor.id, |
@@ -68,5 +69,9 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) { | |||
68 | 69 | ||
69 | // Send a notification to remote server if our follow is not already accepted | 70 | // Send a notification to remote server if our follow is not already accepted |
70 | if (actorFollow.state !== 'accepted') await sendFollow(actorFollow) | 71 | if (actorFollow.state !== 'accepted') await sendFollow(actorFollow) |
72 | |||
73 | return actorFollow | ||
71 | }) | 74 | }) |
75 | |||
76 | if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewFollow(actorFollow) | ||
72 | } | 77 | } |
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 03a9e12a4..9493945ff 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts | |||
@@ -3,8 +3,9 @@ import * as Bluebird from 'bluebird' | |||
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { doRequest } from '../../../helpers/requests' | 4 | import { doRequest } from '../../../helpers/requests' |
5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
6 | import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' | 6 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' |
7 | import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' | 7 | import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' |
8 | import { ActorFollowScoreCache } from '../../cache' | ||
8 | 9 | ||
9 | export type ActivitypubHttpBroadcastPayload = { | 10 | export type ActivitypubHttpBroadcastPayload = { |
10 | uris: string[] | 11 | uris: string[] |
@@ -25,7 +26,8 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { | |||
25 | uri: '', | 26 | uri: '', |
26 | json: body, | 27 | json: body, |
27 | httpSignature: httpSignatureOptions, | 28 | httpSignature: httpSignatureOptions, |
28 | timeout: JOB_REQUEST_TIMEOUT | 29 | timeout: JOB_REQUEST_TIMEOUT, |
30 | headers: buildGlobalHeaders(body) | ||
29 | } | 31 | } |
30 | 32 | ||
31 | const badUrls: string[] = [] | 33 | const badUrls: string[] = [] |
@@ -37,7 +39,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { | |||
37 | .catch(() => badUrls.push(uri)) | 39 | .catch(() => badUrls.push(uri)) |
38 | }, { concurrency: BROADCAST_CONCURRENCY }) | 40 | }, { concurrency: BROADCAST_CONCURRENCY }) |
39 | 41 | ||
40 | return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined) | 42 | return ActorFollowScoreCache.Instance.updateActorFollowsScore(goodUrls, badUrls) |
41 | } | 43 | } |
42 | 44 | ||
43 | // --------------------------------------------------------------------------- | 45 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 42217c27c..67ccfa995 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts | |||
@@ -23,7 +23,7 @@ async function processActivityPubHttpFetcher (job: Bull.Job) { | |||
23 | if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) | 23 | if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) |
24 | 24 | ||
25 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { | 25 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { |
26 | 'activity': items => processActivities(items), | 26 | 'activity': items => processActivities(items, { outboxUrl: payload.uri }), |
27 | 'video-likes': items => createRates(items, video, 'like'), | 27 | 'video-likes': items => createRates(items, video, 'like'), |
28 | 'video-dislikes': items => createRates(items, video, 'dislike'), | 28 | 'video-dislikes': items => createRates(items, video, 'dislike'), |
29 | 'video-shares': items => addVideoShares(items, video), | 29 | 'video-shares': items => addVideoShares(items, video), |
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index c90d735f6..3973dcdc8 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts | |||
@@ -1,9 +1,9 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { doRequest } from '../../../helpers/requests' | 3 | import { doRequest } from '../../../helpers/requests' |
4 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 4 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' |
5 | import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' | ||
6 | import { JOB_REQUEST_TIMEOUT } from '../../../initializers' | 5 | import { JOB_REQUEST_TIMEOUT } from '../../../initializers' |
6 | import { ActorFollowScoreCache } from '../../cache' | ||
7 | 7 | ||
8 | export type ActivitypubHttpUnicastPayload = { | 8 | export type ActivitypubHttpUnicastPayload = { |
9 | uri: string | 9 | uri: string |
@@ -25,14 +25,15 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { | |||
25 | uri, | 25 | uri, |
26 | json: body, | 26 | json: body, |
27 | httpSignature: httpSignatureOptions, | 27 | httpSignature: httpSignatureOptions, |
28 | timeout: JOB_REQUEST_TIMEOUT | 28 | timeout: JOB_REQUEST_TIMEOUT, |
29 | headers: buildGlobalHeaders(body) | ||
29 | } | 30 | } |
30 | 31 | ||
31 | try { | 32 | try { |
32 | await doRequest(options) | 33 | await doRequest(options) |
33 | ActorFollowModel.updateActorFollowsScore([ uri ], [], undefined) | 34 | ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], []) |
34 | } catch (err) { | 35 | } catch (err) { |
35 | ActorFollowModel.updateActorFollowsScore([], [ uri ], undefined) | 36 | ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ]) |
36 | 37 | ||
37 | throw err | 38 | throw err |
38 | } | 39 | } |
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts new file mode 100644 index 000000000..454b975fe --- /dev/null +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts | |||
@@ -0,0 +1,54 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { logger } from '../../../helpers/logger' | ||
3 | import { fetchVideoByUrl } from '../../../helpers/video' | ||
4 | import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' | ||
5 | import { ActorModel } from '../../../models/activitypub/actor' | ||
6 | |||
7 | export type RefreshPayload = { | ||
8 | type: 'video' | 'actor' | ||
9 | url: string | ||
10 | } | ||
11 | |||
12 | async function refreshAPObject (job: Bull.Job) { | ||
13 | const payload = job.data as RefreshPayload | ||
14 | |||
15 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) | ||
16 | |||
17 | if (payload.type === 'video') return refreshVideo(payload.url) | ||
18 | if (payload.type === 'actor') return refreshActor(payload.url) | ||
19 | } | ||
20 | |||
21 | // --------------------------------------------------------------------------- | ||
22 | |||
23 | export { | ||
24 | refreshActor, | ||
25 | refreshAPObject | ||
26 | } | ||
27 | |||
28 | // --------------------------------------------------------------------------- | ||
29 | |||
30 | async function refreshVideo (videoUrl: string) { | ||
31 | const fetchType = 'all' as 'all' | ||
32 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } | ||
33 | |||
34 | const videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
35 | if (videoFromDatabase) { | ||
36 | const refreshOptions = { | ||
37 | video: videoFromDatabase, | ||
38 | fetchedType: fetchType, | ||
39 | syncParam | ||
40 | } | ||
41 | |||
42 | await refreshVideoIfNeeded(refreshOptions) | ||
43 | } | ||
44 | } | ||
45 | |||
46 | async function refreshActor (actorUrl: string) { | ||
47 | const fetchType = 'all' as 'all' | ||
48 | const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) | ||
49 | |||
50 | if (actor) { | ||
51 | await refreshActorIfNeeded(actor, fetchType) | ||
52 | } | ||
53 | |||
54 | } | ||
diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts index 73d98ae54..220d0af32 100644 --- a/server/lib/job-queue/handlers/email.ts +++ b/server/lib/job-queue/handlers/email.ts | |||
@@ -6,13 +6,14 @@ export type EmailPayload = { | |||
6 | to: string[] | 6 | to: string[] |
7 | subject: string | 7 | subject: string |
8 | text: string | 8 | text: string |
9 | from?: string | ||
9 | } | 10 | } |
10 | 11 | ||
11 | async function processEmail (job: Bull.Job) { | 12 | async function processEmail (job: Bull.Job) { |
12 | const payload = job.data as EmailPayload | 13 | const payload = job.data as EmailPayload |
13 | logger.info('Processing email in job %d.', job.id) | 14 | logger.info('Processing email in job %d.', job.id) |
14 | 15 | ||
15 | return Emailer.Instance.sendMail(payload.to, payload.subject, payload.text) | 16 | return Emailer.Instance.sendMail(payload.to, payload.subject, payload.text, payload.from) |
16 | } | 17 | } |
17 | 18 | ||
18 | // --------------------------------------------------------------------------- | 19 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index 36092665e..4961d4502 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts | |||
@@ -1,8 +1,12 @@ | |||
1 | import { buildSignedActivity } from '../../../../helpers/activitypub' | 1 | import { buildSignedActivity } from '../../../../helpers/activitypub' |
2 | import { getServerActor } from '../../../../helpers/utils' | 2 | import { getServerActor } from '../../../../helpers/utils' |
3 | import { ActorModel } from '../../../../models/activitypub/actor' | 3 | import { ActorModel } from '../../../../models/activitypub/actor' |
4 | import { sha256 } from '../../../../helpers/core-utils' | ||
5 | import { HTTP_SIGNATURE } from '../../../../initializers' | ||
4 | 6 | ||
5 | async function computeBody (payload: { body: any, signatureActorId?: number }) { | 7 | type Payload = { body: any, signatureActorId?: number } |
8 | |||
9 | async function computeBody (payload: Payload) { | ||
6 | let body = payload.body | 10 | let body = payload.body |
7 | 11 | ||
8 | if (payload.signatureActorId) { | 12 | if (payload.signatureActorId) { |
@@ -14,7 +18,7 @@ async function computeBody (payload: { body: any, signatureActorId?: number }) { | |||
14 | return body | 18 | return body |
15 | } | 19 | } |
16 | 20 | ||
17 | async function buildSignedRequestOptions (payload: { signatureActorId?: number }) { | 21 | async function buildSignedRequestOptions (payload: Payload) { |
18 | let actor: ActorModel | null | 22 | let actor: ActorModel | null |
19 | if (payload.signatureActorId) { | 23 | if (payload.signatureActorId) { |
20 | actor = await ActorModel.load(payload.signatureActorId) | 24 | actor = await ActorModel.load(payload.signatureActorId) |
@@ -26,14 +30,29 @@ async function buildSignedRequestOptions (payload: { signatureActorId?: number } | |||
26 | 30 | ||
27 | const keyId = actor.getWebfingerUrl() | 31 | const keyId = actor.getWebfingerUrl() |
28 | return { | 32 | return { |
29 | algorithm: 'rsa-sha256', | 33 | algorithm: HTTP_SIGNATURE.ALGORITHM, |
30 | authorizationHeaderName: 'Signature', | 34 | authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, |
31 | keyId, | 35 | keyId, |
32 | key: actor.privateKey | 36 | key: actor.privateKey, |
37 | headers: HTTP_SIGNATURE.HEADERS_TO_SIGN | ||
38 | } | ||
39 | } | ||
40 | |||
41 | function buildGlobalHeaders (body: any) { | ||
42 | return { | ||
43 | 'Digest': buildDigest(body) | ||
33 | } | 44 | } |
34 | } | 45 | } |
35 | 46 | ||
47 | function buildDigest (body: any) { | ||
48 | const rawBody = typeof body === 'string' ? body : JSON.stringify(body) | ||
49 | |||
50 | return 'SHA-256=' + sha256(rawBody, 'base64') | ||
51 | } | ||
52 | |||
36 | export { | 53 | export { |
54 | buildDigest, | ||
55 | buildGlobalHeaders, | ||
37 | computeBody, | 56 | computeBody, |
38 | buildSignedRequestOptions | 57 | buildSignedRequestOptions |
39 | } | 58 | } |
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index 1463c93fc..04983155c 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts | |||
@@ -5,16 +5,18 @@ import { VideoModel } from '../../../models/video/video' | |||
5 | import { JobQueue } from '../job-queue' | 5 | import { JobQueue } from '../job-queue' |
6 | import { federateVideoIfNeeded } from '../../activitypub' | 6 | import { federateVideoIfNeeded } from '../../activitypub' |
7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
8 | import { sequelizeTypescript } from '../../../initializers' | 8 | import { sequelizeTypescript, CONFIG } from '../../../initializers' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' | 10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' |
11 | import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding' | 11 | import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' |
12 | import { Notifier } from '../../notifier' | ||
12 | 13 | ||
13 | export type VideoFilePayload = { | 14 | export type VideoFilePayload = { |
14 | videoUUID: string | 15 | videoUUID: string |
15 | isNewVideo?: boolean | ||
16 | resolution?: VideoResolution | 16 | resolution?: VideoResolution |
17 | isNewVideo?: boolean | ||
17 | isPortraitMode?: boolean | 18 | isPortraitMode?: boolean |
19 | generateHlsPlaylist?: boolean | ||
18 | } | 20 | } |
19 | 21 | ||
20 | export type VideoFileImportPayload = { | 22 | export type VideoFileImportPayload = { |
@@ -50,34 +52,51 @@ async function processVideoFile (job: Bull.Job) { | |||
50 | return undefined | 52 | return undefined |
51 | } | 53 | } |
52 | 54 | ||
53 | // Transcoding in other resolution | 55 | if (payload.generateHlsPlaylist) { |
54 | if (payload.resolution) { | 56 | await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) |
57 | |||
58 | await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) | ||
59 | } else if (payload.resolution) { // Transcoding in other resolution | ||
55 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) | 60 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) |
56 | 61 | ||
57 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) | 62 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload) |
58 | } else { | 63 | } else { |
59 | await optimizeOriginalVideofile(video) | 64 | await optimizeVideofile(video) |
60 | 65 | ||
61 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) | 66 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) |
62 | } | 67 | } |
63 | 68 | ||
64 | return video | 69 | return video |
65 | } | 70 | } |
66 | 71 | ||
67 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | 72 | async function onHlsPlaylistGenerationSuccess (video: VideoModel) { |
73 | if (video === undefined) return undefined | ||
74 | |||
75 | await sequelizeTypescript.transaction(async t => { | ||
76 | // Maybe the video changed in database, refresh it | ||
77 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
78 | // Video does not exist anymore | ||
79 | if (!videoDatabase) return undefined | ||
80 | |||
81 | // If the video was not published, we consider it is a new one for other instances | ||
82 | await federateVideoIfNeeded(videoDatabase, false, t) | ||
83 | }) | ||
84 | } | ||
85 | |||
86 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) { | ||
68 | if (video === undefined) return undefined | 87 | if (video === undefined) return undefined |
69 | 88 | ||
70 | return sequelizeTypescript.transaction(async t => { | 89 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { |
71 | // Maybe the video changed in database, refresh it | 90 | // Maybe the video changed in database, refresh it |
72 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | 91 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
73 | // Video does not exist anymore | 92 | // Video does not exist anymore |
74 | if (!videoDatabase) return undefined | 93 | if (!videoDatabase) return undefined |
75 | 94 | ||
76 | let isNewVideo = false | 95 | let videoPublished = false |
77 | 96 | ||
78 | // We transcoded the video file in another format, now we can publish it | 97 | // We transcoded the video file in another format, now we can publish it |
79 | if (videoDatabase.state !== VideoState.PUBLISHED) { | 98 | if (videoDatabase.state !== VideoState.PUBLISHED) { |
80 | isNewVideo = true | 99 | videoPublished = true |
81 | 100 | ||
82 | videoDatabase.state = VideoState.PUBLISHED | 101 | videoDatabase.state = VideoState.PUBLISHED |
83 | videoDatabase.publishedAt = new Date() | 102 | videoDatabase.publishedAt = new Date() |
@@ -85,21 +104,29 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | |||
85 | } | 104 | } |
86 | 105 | ||
87 | // If the video was not published, we consider it is a new one for other instances | 106 | // If the video was not published, we consider it is a new one for other instances |
88 | await federateVideoIfNeeded(videoDatabase, isNewVideo, t) | 107 | await federateVideoIfNeeded(videoDatabase, videoPublished, t) |
89 | 108 | ||
90 | return undefined | 109 | return { videoDatabase, videoPublished } |
91 | }) | 110 | }) |
111 | |||
112 | // don't notify prior to scheduled video update | ||
113 | if (videoPublished && !videoDatabase.ScheduleVideoUpdate) { | ||
114 | Notifier.Instance.notifyOnNewVideo(videoDatabase) | ||
115 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | ||
116 | } | ||
117 | |||
118 | await createHlsJobIfEnabled(payload) | ||
92 | } | 119 | } |
93 | 120 | ||
94 | async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boolean) { | 121 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) { |
95 | if (video === undefined) return undefined | 122 | if (videoArg === undefined) return undefined |
96 | 123 | ||
97 | // Outside the transaction (IO on disk) | 124 | // Outside the transaction (IO on disk) |
98 | const { videoFileResolution } = await video.getOriginalFileResolution() | 125 | const { videoFileResolution } = await videoArg.getOriginalFileResolution() |
99 | 126 | ||
100 | return sequelizeTypescript.transaction(async t => { | 127 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { |
101 | // Maybe the video changed in database, refresh it | 128 | // Maybe the video changed in database, refresh it |
102 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | 129 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t) |
103 | // Video does not exist anymore | 130 | // Video does not exist anymore |
104 | if (!videoDatabase) return undefined | 131 | if (!videoDatabase) return undefined |
105 | 132 | ||
@@ -110,8 +137,10 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole | |||
110 | { resolutions: resolutionsEnabled } | 137 | { resolutions: resolutionsEnabled } |
111 | ) | 138 | ) |
112 | 139 | ||
140 | let videoPublished = false | ||
141 | |||
113 | if (resolutionsEnabled.length !== 0) { | 142 | if (resolutionsEnabled.length !== 0) { |
114 | const tasks: Bluebird<any>[] = [] | 143 | const tasks: Bluebird<Bull.Job<any>>[] = [] |
115 | 144 | ||
116 | for (const resolution of resolutionsEnabled) { | 145 | for (const resolution of resolutionsEnabled) { |
117 | const dataInput = { | 146 | const dataInput = { |
@@ -127,15 +156,27 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole | |||
127 | 156 | ||
128 | logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) | 157 | logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) |
129 | } else { | 158 | } else { |
159 | videoPublished = true | ||
160 | |||
130 | // No transcoding to do, it's now published | 161 | // No transcoding to do, it's now published |
131 | video.state = VideoState.PUBLISHED | 162 | videoDatabase.state = VideoState.PUBLISHED |
132 | video = await video.save({ transaction: t }) | 163 | videoDatabase = await videoDatabase.save({ transaction: t }) |
133 | 164 | ||
134 | logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid) | 165 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) |
135 | } | 166 | } |
136 | 167 | ||
137 | return federateVideoIfNeeded(video, isNewVideo, t) | 168 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) |
169 | |||
170 | return { videoDatabase, videoPublished } | ||
138 | }) | 171 | }) |
172 | |||
173 | // don't notify prior to scheduled video update | ||
174 | if (!videoDatabase.ScheduleVideoUpdate) { | ||
175 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | ||
176 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | ||
177 | } | ||
178 | |||
179 | await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) | ||
139 | } | 180 | } |
140 | 181 | ||
141 | // --------------------------------------------------------------------------- | 182 | // --------------------------------------------------------------------------- |
@@ -144,3 +185,20 @@ export { | |||
144 | processVideoFile, | 185 | processVideoFile, |
145 | processVideoFileImport | 186 | processVideoFileImport |
146 | } | 187 | } |
188 | |||
189 | // --------------------------------------------------------------------------- | ||
190 | |||
191 | function createHlsJobIfEnabled (payload?: VideoFilePayload) { | ||
192 | // Generate HLS playlist? | ||
193 | if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { | ||
194 | const hlsTranscodingPayload = { | ||
195 | videoUUID: payload.videoUUID, | ||
196 | resolution: payload.resolution, | ||
197 | isPortraitMode: payload.isPortraitMode, | ||
198 | |||
199 | generateHlsPlaylist: true | ||
200 | } | ||
201 | |||
202 | return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload }) | ||
203 | } | ||
204 | } | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index e3f2a276c..12004dcd7 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -6,15 +6,16 @@ import { VideoImportState } from '../../../../shared/models/videos' | |||
6 | import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' | 6 | import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' |
7 | import { extname, join } from 'path' | 7 | import { extname, join } from 'path' |
8 | import { VideoFileModel } from '../../../models/video/video-file' | 8 | import { VideoFileModel } from '../../../models/video/video-file' |
9 | import { CONFIG, sequelizeTypescript, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' | 9 | import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' |
10 | import { doRequestAndSaveToFile } from '../../../helpers/requests' | 10 | import { downloadImage } from '../../../helpers/requests' |
11 | import { VideoState } from '../../../../shared' | 11 | import { VideoState } from '../../../../shared' |
12 | import { JobQueue } from '../index' | 12 | import { JobQueue } from '../index' |
13 | import { federateVideoIfNeeded } from '../../activitypub' | 13 | import { federateVideoIfNeeded } from '../../activitypub' |
14 | import { VideoModel } from '../../../models/video/video' | 14 | import { VideoModel } from '../../../models/video/video' |
15 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 15 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
16 | import { getSecureTorrentName } from '../../../helpers/utils' | 16 | import { getSecureTorrentName } from '../../../helpers/utils' |
17 | import { remove, rename, stat } from 'fs-extra' | 17 | import { remove, move, stat } from 'fs-extra' |
18 | import { Notifier } from '../../notifier' | ||
18 | 19 | ||
19 | type VideoImportYoutubeDLPayload = { | 20 | type VideoImportYoutubeDLPayload = { |
20 | type: 'youtube-dl' | 21 | type: 'youtube-dl' |
@@ -109,6 +110,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
109 | let tempVideoPath: string | 110 | let tempVideoPath: string |
110 | let videoDestFile: string | 111 | let videoDestFile: string |
111 | let videoFile: VideoFileModel | 112 | let videoFile: VideoFileModel |
113 | |||
112 | try { | 114 | try { |
113 | // Download video from youtubeDL | 115 | // Download video from youtubeDL |
114 | tempVideoPath = await downloader() | 116 | tempVideoPath = await downloader() |
@@ -133,19 +135,18 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
133 | videoId: videoImport.videoId | 135 | videoId: videoImport.videoId |
134 | } | 136 | } |
135 | videoFile = new VideoFileModel(videoFileData) | 137 | videoFile = new VideoFileModel(videoFileData) |
136 | // Import if the import fails, to clean files | 138 | // To clean files if the import fails |
137 | videoImport.Video.VideoFiles = [ videoFile ] | 139 | videoImport.Video.VideoFiles = [ videoFile ] |
138 | 140 | ||
139 | // Move file | 141 | // Move file |
140 | videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) | 142 | videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) |
141 | await rename(tempVideoPath, videoDestFile) | 143 | await move(tempVideoPath, videoDestFile) |
142 | tempVideoPath = null // This path is not used anymore | 144 | tempVideoPath = null // This path is not used anymore |
143 | 145 | ||
144 | // Process thumbnail | 146 | // Process thumbnail |
145 | if (options.downloadThumbnail) { | 147 | if (options.downloadThumbnail) { |
146 | if (options.thumbnailUrl) { | 148 | if (options.thumbnailUrl) { |
147 | const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) | 149 | await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE) |
148 | await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath) | ||
149 | } else { | 150 | } else { |
150 | await videoImport.Video.createThumbnail(videoFile) | 151 | await videoImport.Video.createThumbnail(videoFile) |
151 | } | 152 | } |
@@ -156,8 +157,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
156 | // Process preview | 157 | // Process preview |
157 | if (options.downloadPreview) { | 158 | if (options.downloadPreview) { |
158 | if (options.thumbnailUrl) { | 159 | if (options.thumbnailUrl) { |
159 | const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) | 160 | await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE) |
160 | await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath) | ||
161 | } else { | 161 | } else { |
162 | await videoImport.Video.createPreview(videoFile) | 162 | await videoImport.Video.createPreview(videoFile) |
163 | } | 163 | } |
@@ -180,7 +180,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
180 | // Update video DB object | 180 | // Update video DB object |
181 | video.duration = duration | 181 | video.duration = duration |
182 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED | 182 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED |
183 | const videoUpdated = await video.save({ transaction: t }) | 183 | await video.save({ transaction: t }) |
184 | 184 | ||
185 | // Now we can federate the video (reload from database, we need more attributes) | 185 | // Now we can federate the video (reload from database, we need more attributes) |
186 | const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | 186 | const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
@@ -192,10 +192,13 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
192 | 192 | ||
193 | logger.info('Video %s imported.', video.uuid) | 193 | logger.info('Video %s imported.', video.uuid) |
194 | 194 | ||
195 | videoImportUpdated.Video = videoUpdated | 195 | videoImportUpdated.Video = videoForFederation |
196 | return videoImportUpdated | 196 | return videoImportUpdated |
197 | }) | 197 | }) |
198 | 198 | ||
199 | Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) | ||
200 | Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true) | ||
201 | |||
199 | // Create transcoding jobs? | 202 | // Create transcoding jobs? |
200 | if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { | 203 | if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { |
201 | // Put uuid because we don't have id auto incremented for now | 204 | // Put uuid because we don't have id auto incremented for now |
@@ -218,6 +221,8 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
218 | videoImport.state = VideoImportState.FAILED | 221 | videoImport.state = VideoImportState.FAILED |
219 | await videoImport.save() | 222 | await videoImport.save() |
220 | 223 | ||
224 | Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false) | ||
225 | |||
221 | throw err | 226 | throw err |
222 | } | 227 | } |
223 | } | 228 | } |
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index cf180a11a..fa1fd13b3 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts | |||
@@ -3,8 +3,9 @@ import { logger } from '../../../helpers/logger' | |||
3 | import { VideoModel } from '../../../models/video/video' | 3 | import { VideoModel } from '../../../models/video/video' |
4 | import { VideoViewModel } from '../../../models/video/video-views' | 4 | import { VideoViewModel } from '../../../models/video/video-views' |
5 | import { isTestInstance } from '../../../helpers/core-utils' | 5 | import { isTestInstance } from '../../../helpers/core-utils' |
6 | import { federateVideoIfNeeded } from '../../activitypub' | ||
6 | 7 | ||
7 | async function processVideosViewsViews () { | 8 | async function processVideosViews () { |
8 | const lastHour = new Date() | 9 | const lastHour = new Date() |
9 | 10 | ||
10 | // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour | 11 | // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour |
@@ -22,13 +23,9 @@ async function processVideosViewsViews () { | |||
22 | for (const videoId of videoIds) { | 23 | for (const videoId of videoIds) { |
23 | try { | 24 | try { |
24 | const views = await Redis.Instance.getVideoViews(videoId, hour) | 25 | const views = await Redis.Instance.getVideoViews(videoId, hour) |
25 | if (isNaN(views)) { | 26 | if (views) { |
26 | logger.error('Cannot process videos views of video %d in hour %d: views number is NaN.', videoId, hour) | ||
27 | } else { | ||
28 | logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) | 27 | logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) |
29 | 28 | ||
30 | await VideoModel.incrementViews(videoId, views) | ||
31 | |||
32 | try { | 29 | try { |
33 | await VideoViewModel.create({ | 30 | await VideoViewModel.create({ |
34 | startDate, | 31 | startDate, |
@@ -36,6 +33,16 @@ async function processVideosViewsViews () { | |||
36 | views, | 33 | views, |
37 | videoId | 34 | videoId |
38 | }) | 35 | }) |
36 | |||
37 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
38 | if (video.isOwned()) { | ||
39 | // If this is a remote video, the origin instance will send us an update | ||
40 | await VideoModel.incrementViews(videoId, views) | ||
41 | |||
42 | // Send video update | ||
43 | video.views += views | ||
44 | await federateVideoIfNeeded(video, false) | ||
45 | } | ||
39 | } catch (err) { | 46 | } catch (err) { |
40 | logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) | 47 | logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) |
41 | } | 48 | } |
@@ -51,5 +58,5 @@ async function processVideosViewsViews () { | |||
51 | // --------------------------------------------------------------------------- | 58 | // --------------------------------------------------------------------------- |
52 | 59 | ||
53 | export { | 60 | export { |
54 | processVideosViewsViews | 61 | processVideosViews |
55 | } | 62 | } |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 0696ba43c..ba9cbe0d9 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -10,7 +10,8 @@ import { EmailPayload, processEmail } from './handlers/email' | |||
10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' | 10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' |
11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' | 11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' | 12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' |
13 | import { processVideosViewsViews } from './handlers/video-views' | 13 | import { processVideosViews } from './handlers/video-views' |
14 | import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' | ||
14 | 15 | ||
15 | type CreateJobArgument = | 16 | type CreateJobArgument = |
16 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 17 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -21,6 +22,7 @@ type CreateJobArgument = | |||
21 | { type: 'video-file', payload: VideoFilePayload } | | 22 | { type: 'video-file', payload: VideoFilePayload } | |
22 | { type: 'email', payload: EmailPayload } | | 23 | { type: 'email', payload: EmailPayload } | |
23 | { type: 'video-import', payload: VideoImportPayload } | | 24 | { type: 'video-import', payload: VideoImportPayload } | |
25 | { type: 'activitypub-refresher', payload: RefreshPayload } | | ||
24 | { type: 'videos-views', payload: {} } | 26 | { type: 'videos-views', payload: {} } |
25 | 27 | ||
26 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | 28 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { |
@@ -32,7 +34,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | |||
32 | 'video-file': processVideoFile, | 34 | 'video-file': processVideoFile, |
33 | 'email': processEmail, | 35 | 'email': processEmail, |
34 | 'video-import': processVideoImport, | 36 | 'video-import': processVideoImport, |
35 | 'videos-views': processVideosViewsViews | 37 | 'videos-views': processVideosViews, |
38 | 'activitypub-refresher': refreshAPObject | ||
36 | } | 39 | } |
37 | 40 | ||
38 | const jobTypes: JobType[] = [ | 41 | const jobTypes: JobType[] = [ |
@@ -44,7 +47,8 @@ const jobTypes: JobType[] = [ | |||
44 | 'video-file', | 47 | 'video-file', |
45 | 'video-file-import', | 48 | 'video-file-import', |
46 | 'video-import', | 49 | 'video-import', |
47 | 'videos-views' | 50 | 'videos-views', |
51 | 'activitypub-refresher' | ||
48 | ] | 52 | ] |
49 | 53 | ||
50 | class JobQueue { | 54 | class JobQueue { |
@@ -84,7 +88,6 @@ class JobQueue { | |||
84 | 88 | ||
85 | queue.on('error', err => { | 89 | queue.on('error', err => { |
86 | logger.error('Error in job queue %s.', handlerName, { err }) | 90 | logger.error('Error in job queue %s.', handlerName, { err }) |
87 | process.exit(-1) | ||
88 | }) | 91 | }) |
89 | 92 | ||
90 | this.queues[handlerName] = queue | 93 | this.queues[handlerName] = queue |
@@ -162,10 +165,10 @@ class JobQueue { | |||
162 | return total | 165 | return total |
163 | } | 166 | } |
164 | 167 | ||
165 | removeOldJobs () { | 168 | async removeOldJobs () { |
166 | for (const key of Object.keys(this.queues)) { | 169 | for (const key of Object.keys(this.queues)) { |
167 | const queue = this.queues[key] | 170 | const queue = this.queues[key] |
168 | queue.clean(JOB_COMPLETED_LIFETIME, 'completed') | 171 | await queue.clean(JOB_COMPLETED_LIFETIME, 'completed') |
169 | } | 172 | } |
170 | } | 173 | } |
171 | 174 | ||
diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts new file mode 100644 index 000000000..d1b331346 --- /dev/null +++ b/server/lib/notifier.ts | |||
@@ -0,0 +1,455 @@ | |||
1 | import { UserNotificationSettingValue, UserNotificationType, UserRight } from '../../shared/models/users' | ||
2 | import { logger } from '../helpers/logger' | ||
3 | import { VideoModel } from '../models/video/video' | ||
4 | import { Emailer } from './emailer' | ||
5 | import { UserNotificationModel } from '../models/account/user-notification' | ||
6 | import { VideoCommentModel } from '../models/video/video-comment' | ||
7 | import { UserModel } from '../models/account/user' | ||
8 | import { PeerTubeSocket } from './peertube-socket' | ||
9 | import { CONFIG } from '../initializers/constants' | ||
10 | import { VideoPrivacy, VideoState } from '../../shared/models/videos' | ||
11 | import { VideoAbuseModel } from '../models/video/video-abuse' | ||
12 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | ||
13 | import * as Bluebird from 'bluebird' | ||
14 | import { VideoImportModel } from '../models/video/video-import' | ||
15 | import { AccountBlocklistModel } from '../models/account/account-blocklist' | ||
16 | import { ActorFollowModel } from '../models/activitypub/actor-follow' | ||
17 | import { AccountModel } from '../models/account/account' | ||
18 | |||
19 | class Notifier { | ||
20 | |||
21 | private static instance: Notifier | ||
22 | |||
23 | private constructor () {} | ||
24 | |||
25 | notifyOnNewVideo (video: VideoModel): void { | ||
26 | // Only notify on public and published videos | ||
27 | if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED) return | ||
28 | |||
29 | this.notifySubscribersOfNewVideo(video) | ||
30 | .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) | ||
31 | } | ||
32 | |||
33 | notifyOnPendingVideoPublished (video: VideoModel): void { | ||
34 | // Only notify on public videos that has been published while the user waited transcoding/scheduled update | ||
35 | if (video.waitTranscoding === false && !video.ScheduleVideoUpdate) return | ||
36 | |||
37 | this.notifyOwnedVideoHasBeenPublished(video) | ||
38 | .catch(err => logger.error('Cannot notify owner that its video %s has been published.', video.url, { err })) | ||
39 | } | ||
40 | |||
41 | notifyOnNewComment (comment: VideoCommentModel): void { | ||
42 | this.notifyVideoOwnerOfNewComment(comment) | ||
43 | .catch(err => logger.error('Cannot notify video owner of new comment %s.', comment.url, { err })) | ||
44 | |||
45 | this.notifyOfCommentMention(comment) | ||
46 | .catch(err => logger.error('Cannot notify mentions of comment %s.', comment.url, { err })) | ||
47 | } | ||
48 | |||
49 | notifyOnNewVideoAbuse (videoAbuse: VideoAbuseModel): void { | ||
50 | this.notifyModeratorsOfNewVideoAbuse(videoAbuse) | ||
51 | .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err })) | ||
52 | } | ||
53 | |||
54 | notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void { | ||
55 | this.notifyVideoOwnerOfBlacklist(videoBlacklist) | ||
56 | .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) | ||
57 | } | ||
58 | |||
59 | notifyOnVideoUnblacklist (video: VideoModel): void { | ||
60 | this.notifyVideoOwnerOfUnblacklist(video) | ||
61 | .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err })) | ||
62 | } | ||
63 | |||
64 | notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void { | ||
65 | this.notifyOwnerVideoImportIsFinished(videoImport, success) | ||
66 | .catch(err => logger.error('Cannot notify owner that its video import %s is finished.', videoImport.getTargetIdentifier(), { err })) | ||
67 | } | ||
68 | |||
69 | notifyOnNewUserRegistration (user: UserModel): void { | ||
70 | this.notifyModeratorsOfNewUserRegistration(user) | ||
71 | .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) | ||
72 | } | ||
73 | |||
74 | notifyOfNewFollow (actorFollow: ActorFollowModel): void { | ||
75 | this.notifyUserOfNewActorFollow(actorFollow) | ||
76 | .catch(err => { | ||
77 | logger.error( | ||
78 | 'Cannot notify owner of channel %s of a new follow by %s.', | ||
79 | actorFollow.ActorFollowing.VideoChannel.getDisplayName(), | ||
80 | actorFollow.ActorFollower.Account.getDisplayName(), | ||
81 | err | ||
82 | ) | ||
83 | }) | ||
84 | } | ||
85 | |||
86 | private async notifySubscribersOfNewVideo (video: VideoModel) { | ||
87 | // List all followers that are users | ||
88 | const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) | ||
89 | |||
90 | logger.info('Notifying %d users of new video %s.', users.length, video.url) | ||
91 | |||
92 | function settingGetter (user: UserModel) { | ||
93 | return user.NotificationSetting.newVideoFromSubscription | ||
94 | } | ||
95 | |||
96 | async function notificationCreator (user: UserModel) { | ||
97 | const notification = await UserNotificationModel.create({ | ||
98 | type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION, | ||
99 | userId: user.id, | ||
100 | videoId: video.id | ||
101 | }) | ||
102 | notification.Video = video | ||
103 | |||
104 | return notification | ||
105 | } | ||
106 | |||
107 | function emailSender (emails: string[]) { | ||
108 | return Emailer.Instance.addNewVideoFromSubscriberNotification(emails, video) | ||
109 | } | ||
110 | |||
111 | return this.notify({ users, settingGetter, notificationCreator, emailSender }) | ||
112 | } | ||
113 | |||
114 | private async notifyVideoOwnerOfNewComment (comment: VideoCommentModel) { | ||
115 | if (comment.Video.isOwned() === false) return | ||
116 | |||
117 | const user = await UserModel.loadByVideoId(comment.videoId) | ||
118 | |||
119 | // Not our user or user comments its own video | ||
120 | if (!user || comment.Account.userId === user.id) return | ||
121 | |||
122 | const accountMuted = await AccountBlocklistModel.isAccountMutedBy(user.Account.id, comment.accountId) | ||
123 | if (accountMuted) return | ||
124 | |||
125 | logger.info('Notifying user %s of new comment %s.', user.username, comment.url) | ||
126 | |||
127 | function settingGetter (user: UserModel) { | ||
128 | return user.NotificationSetting.newCommentOnMyVideo | ||
129 | } | ||
130 | |||
131 | async function notificationCreator (user: UserModel) { | ||
132 | const notification = await UserNotificationModel.create({ | ||
133 | type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, | ||
134 | userId: user.id, | ||
135 | commentId: comment.id | ||
136 | }) | ||
137 | notification.Comment = comment | ||
138 | |||
139 | return notification | ||
140 | } | ||
141 | |||
142 | function emailSender (emails: string[]) { | ||
143 | return Emailer.Instance.addNewCommentOnMyVideoNotification(emails, comment) | ||
144 | } | ||
145 | |||
146 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | ||
147 | } | ||
148 | |||
149 | private async notifyOfCommentMention (comment: VideoCommentModel) { | ||
150 | const usernames = comment.extractMentions() | ||
151 | let users = await UserModel.listByUsernames(usernames) | ||
152 | |||
153 | if (comment.Video.isOwned()) { | ||
154 | const userException = await UserModel.loadByVideoId(comment.videoId) | ||
155 | users = users.filter(u => u.id !== userException.id) | ||
156 | } | ||
157 | |||
158 | // Don't notify if I mentioned myself | ||
159 | users = users.filter(u => u.Account.id !== comment.accountId) | ||
160 | |||
161 | if (users.length === 0) return | ||
162 | |||
163 | const accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(users.map(u => u.Account.id), comment.accountId) | ||
164 | |||
165 | logger.info('Notifying %d users of new comment %s.', users.length, comment.url) | ||
166 | |||
167 | function settingGetter (user: UserModel) { | ||
168 | if (accountMutedHash[user.Account.id] === true) return UserNotificationSettingValue.NONE | ||
169 | |||
170 | return user.NotificationSetting.commentMention | ||
171 | } | ||
172 | |||
173 | async function notificationCreator (user: UserModel) { | ||
174 | const notification = await UserNotificationModel.create({ | ||
175 | type: UserNotificationType.COMMENT_MENTION, | ||
176 | userId: user.id, | ||
177 | commentId: comment.id | ||
178 | }) | ||
179 | notification.Comment = comment | ||
180 | |||
181 | return notification | ||
182 | } | ||
183 | |||
184 | function emailSender (emails: string[]) { | ||
185 | return Emailer.Instance.addNewCommentMentionNotification(emails, comment) | ||
186 | } | ||
187 | |||
188 | return this.notify({ users, settingGetter, notificationCreator, emailSender }) | ||
189 | } | ||
190 | |||
191 | private async notifyUserOfNewActorFollow (actorFollow: ActorFollowModel) { | ||
192 | if (actorFollow.ActorFollowing.isOwned() === false) return | ||
193 | |||
194 | // Account follows one of our account? | ||
195 | let followType: 'account' | 'channel' = 'channel' | ||
196 | let user = await UserModel.loadByChannelActorId(actorFollow.ActorFollowing.id) | ||
197 | |||
198 | // Account follows one of our channel? | ||
199 | if (!user) { | ||
200 | user = await UserModel.loadByAccountActorId(actorFollow.ActorFollowing.id) | ||
201 | followType = 'account' | ||
202 | } | ||
203 | |||
204 | if (!user) return | ||
205 | |||
206 | if (!actorFollow.ActorFollower.Account || !actorFollow.ActorFollower.Account.name) { | ||
207 | actorFollow.ActorFollower.Account = await actorFollow.ActorFollower.$get('Account') as AccountModel | ||
208 | } | ||
209 | const followerAccount = actorFollow.ActorFollower.Account | ||
210 | |||
211 | const accountMuted = await AccountBlocklistModel.isAccountMutedBy(user.Account.id, followerAccount.id) | ||
212 | if (accountMuted) return | ||
213 | |||
214 | logger.info('Notifying user %s of new follower: %s.', user.username, followerAccount.getDisplayName()) | ||
215 | |||
216 | function settingGetter (user: UserModel) { | ||
217 | return user.NotificationSetting.newFollow | ||
218 | } | ||
219 | |||
220 | async function notificationCreator (user: UserModel) { | ||
221 | const notification = await UserNotificationModel.create({ | ||
222 | type: UserNotificationType.NEW_FOLLOW, | ||
223 | userId: user.id, | ||
224 | actorFollowId: actorFollow.id | ||
225 | }) | ||
226 | notification.ActorFollow = actorFollow | ||
227 | |||
228 | return notification | ||
229 | } | ||
230 | |||
231 | function emailSender (emails: string[]) { | ||
232 | return Emailer.Instance.addNewFollowNotification(emails, actorFollow, followType) | ||
233 | } | ||
234 | |||
235 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | ||
236 | } | ||
237 | |||
238 | private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { | ||
239 | const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) | ||
240 | if (moderators.length === 0) return | ||
241 | |||
242 | logger.info('Notifying %s user/moderators of new video abuse %s.', moderators.length, videoAbuse.Video.url) | ||
243 | |||
244 | function settingGetter (user: UserModel) { | ||
245 | return user.NotificationSetting.videoAbuseAsModerator | ||
246 | } | ||
247 | |||
248 | async function notificationCreator (user: UserModel) { | ||
249 | const notification = await UserNotificationModel.create({ | ||
250 | type: UserNotificationType.NEW_VIDEO_ABUSE_FOR_MODERATORS, | ||
251 | userId: user.id, | ||
252 | videoAbuseId: videoAbuse.id | ||
253 | }) | ||
254 | notification.VideoAbuse = videoAbuse | ||
255 | |||
256 | return notification | ||
257 | } | ||
258 | |||
259 | function emailSender (emails: string[]) { | ||
260 | return Emailer.Instance.addVideoAbuseModeratorsNotification(emails, videoAbuse) | ||
261 | } | ||
262 | |||
263 | return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) | ||
264 | } | ||
265 | |||
266 | private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { | ||
267 | const user = await UserModel.loadByVideoId(videoBlacklist.videoId) | ||
268 | if (!user) return | ||
269 | |||
270 | logger.info('Notifying user %s that its video %s has been blacklisted.', user.username, videoBlacklist.Video.url) | ||
271 | |||
272 | function settingGetter (user: UserModel) { | ||
273 | return user.NotificationSetting.blacklistOnMyVideo | ||
274 | } | ||
275 | |||
276 | async function notificationCreator (user: UserModel) { | ||
277 | const notification = await UserNotificationModel.create({ | ||
278 | type: UserNotificationType.BLACKLIST_ON_MY_VIDEO, | ||
279 | userId: user.id, | ||
280 | videoBlacklistId: videoBlacklist.id | ||
281 | }) | ||
282 | notification.VideoBlacklist = videoBlacklist | ||
283 | |||
284 | return notification | ||
285 | } | ||
286 | |||
287 | function emailSender (emails: string[]) { | ||
288 | return Emailer.Instance.addVideoBlacklistNotification(emails, videoBlacklist) | ||
289 | } | ||
290 | |||
291 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | ||
292 | } | ||
293 | |||
294 | private async notifyVideoOwnerOfUnblacklist (video: VideoModel) { | ||
295 | const user = await UserModel.loadByVideoId(video.id) | ||
296 | if (!user) return | ||
297 | |||
298 | logger.info('Notifying user %s that its video %s has been unblacklisted.', user.username, video.url) | ||
299 | |||
300 | function settingGetter (user: UserModel) { | ||
301 | return user.NotificationSetting.blacklistOnMyVideo | ||
302 | } | ||
303 | |||
304 | async function notificationCreator (user: UserModel) { | ||
305 | const notification = await UserNotificationModel.create({ | ||
306 | type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO, | ||
307 | userId: user.id, | ||
308 | videoId: video.id | ||
309 | }) | ||
310 | notification.Video = video | ||
311 | |||
312 | return notification | ||
313 | } | ||
314 | |||
315 | function emailSender (emails: string[]) { | ||
316 | return Emailer.Instance.addVideoUnblacklistNotification(emails, video) | ||
317 | } | ||
318 | |||
319 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | ||
320 | } | ||
321 | |||
322 | private async notifyOwnedVideoHasBeenPublished (video: VideoModel) { | ||
323 | const user = await UserModel.loadByVideoId(video.id) | ||
324 | if (!user) return | ||
325 | |||
326 | logger.info('Notifying user %s of the publication of its video %s.', user.username, video.url) | ||
327 | |||
328 | function settingGetter (user: UserModel) { | ||
329 | return user.NotificationSetting.myVideoPublished | ||
330 | } | ||
331 | |||
332 | async function notificationCreator (user: UserModel) { | ||
333 | const notification = await UserNotificationModel.create({ | ||
334 | type: UserNotificationType.MY_VIDEO_PUBLISHED, | ||
335 | userId: user.id, | ||
336 | videoId: video.id | ||
337 | }) | ||
338 | notification.Video = video | ||
339 | |||
340 | return notification | ||
341 | } | ||
342 | |||
343 | function emailSender (emails: string[]) { | ||
344 | return Emailer.Instance.myVideoPublishedNotification(emails, video) | ||
345 | } | ||
346 | |||
347 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | ||
348 | } | ||
349 | |||
350 | private async notifyOwnerVideoImportIsFinished (videoImport: VideoImportModel, success: boolean) { | ||
351 | const user = await UserModel.loadByVideoImportId(videoImport.id) | ||
352 | if (!user) return | ||
353 | |||
354 | logger.info('Notifying user %s its video import %s is finished.', user.username, videoImport.getTargetIdentifier()) | ||
355 | |||
356 | function settingGetter (user: UserModel) { | ||
357 | return user.NotificationSetting.myVideoImportFinished | ||
358 | } | ||
359 | |||
360 | async function notificationCreator (user: UserModel) { | ||
361 | const notification = await UserNotificationModel.create({ | ||
362 | type: success ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS : UserNotificationType.MY_VIDEO_IMPORT_ERROR, | ||
363 | userId: user.id, | ||
364 | videoImportId: videoImport.id | ||
365 | }) | ||
366 | notification.VideoImport = videoImport | ||
367 | |||
368 | return notification | ||
369 | } | ||
370 | |||
371 | function emailSender (emails: string[]) { | ||
372 | return success | ||
373 | ? Emailer.Instance.myVideoImportSuccessNotification(emails, videoImport) | ||
374 | : Emailer.Instance.myVideoImportErrorNotification(emails, videoImport) | ||
375 | } | ||
376 | |||
377 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | ||
378 | } | ||
379 | |||
380 | private async notifyModeratorsOfNewUserRegistration (registeredUser: UserModel) { | ||
381 | const moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS) | ||
382 | if (moderators.length === 0) return | ||
383 | |||
384 | logger.info( | ||
385 | 'Notifying %s moderators of new user registration of %s.', | ||
386 | moderators.length, registeredUser.Account.Actor.preferredUsername | ||
387 | ) | ||
388 | |||
389 | function settingGetter (user: UserModel) { | ||
390 | return user.NotificationSetting.newUserRegistration | ||
391 | } | ||
392 | |||
393 | async function notificationCreator (user: UserModel) { | ||
394 | const notification = await UserNotificationModel.create({ | ||
395 | type: UserNotificationType.NEW_USER_REGISTRATION, | ||
396 | userId: user.id, | ||
397 | accountId: registeredUser.Account.id | ||
398 | }) | ||
399 | notification.Account = registeredUser.Account | ||
400 | |||
401 | return notification | ||
402 | } | ||
403 | |||
404 | function emailSender (emails: string[]) { | ||
405 | return Emailer.Instance.addNewUserRegistrationNotification(emails, registeredUser) | ||
406 | } | ||
407 | |||
408 | return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) | ||
409 | } | ||
410 | |||
411 | private async notify (options: { | ||
412 | users: UserModel[], | ||
413 | notificationCreator: (user: UserModel) => Promise<UserNotificationModel>, | ||
414 | emailSender: (emails: string[]) => Promise<any> | Bluebird<any>, | ||
415 | settingGetter: (user: UserModel) => UserNotificationSettingValue | ||
416 | }) { | ||
417 | const emails: string[] = [] | ||
418 | |||
419 | for (const user of options.users) { | ||
420 | if (this.isWebNotificationEnabled(options.settingGetter(user))) { | ||
421 | const notification = await options.notificationCreator(user) | ||
422 | |||
423 | PeerTubeSocket.Instance.sendNotification(user.id, notification) | ||
424 | } | ||
425 | |||
426 | if (this.isEmailEnabled(user, options.settingGetter(user))) { | ||
427 | emails.push(user.email) | ||
428 | } | ||
429 | } | ||
430 | |||
431 | if (emails.length !== 0) { | ||
432 | await options.emailSender(emails) | ||
433 | } | ||
434 | } | ||
435 | |||
436 | private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) { | ||
437 | if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified !== true) return false | ||
438 | |||
439 | return value & UserNotificationSettingValue.EMAIL | ||
440 | } | ||
441 | |||
442 | private isWebNotificationEnabled (value: UserNotificationSettingValue) { | ||
443 | return value & UserNotificationSettingValue.WEB | ||
444 | } | ||
445 | |||
446 | static get Instance () { | ||
447 | return this.instance || (this.instance = new this()) | ||
448 | } | ||
449 | } | ||
450 | |||
451 | // --------------------------------------------------------------------------- | ||
452 | |||
453 | export { | ||
454 | Notifier | ||
455 | } | ||
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 5cbe60b82..2cd2ae97c 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
1 | import { AccessDeniedError } from 'oauth2-server' | 2 | import { AccessDeniedError } from 'oauth2-server' |
2 | import { logger } from '../helpers/logger' | 3 | import { logger } from '../helpers/logger' |
3 | import { UserModel } from '../models/account/user' | 4 | import { UserModel } from '../models/account/user' |
@@ -37,7 +38,7 @@ function clearCacheByToken (token: string) { | |||
37 | function getAccessToken (bearerToken: string) { | 38 | function getAccessToken (bearerToken: string) { |
38 | logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') | 39 | logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') |
39 | 40 | ||
40 | if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken] | 41 | if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) |
41 | 42 | ||
42 | return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) | 43 | return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) |
43 | .then(tokenModel => { | 44 | .then(tokenModel => { |
diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts new file mode 100644 index 000000000..eb84ecd4b --- /dev/null +++ b/server/lib/peertube-socket.ts | |||
@@ -0,0 +1,52 @@ | |||
1 | import * as SocketIO from 'socket.io' | ||
2 | import { authenticateSocket } from '../middlewares' | ||
3 | import { UserNotificationModel } from '../models/account/user-notification' | ||
4 | import { logger } from '../helpers/logger' | ||
5 | import { Server } from 'http' | ||
6 | |||
7 | class PeerTubeSocket { | ||
8 | |||
9 | private static instance: PeerTubeSocket | ||
10 | |||
11 | private userNotificationSockets: { [ userId: number ]: SocketIO.Socket } = {} | ||
12 | |||
13 | private constructor () {} | ||
14 | |||
15 | init (server: Server) { | ||
16 | const io = SocketIO(server) | ||
17 | |||
18 | io.of('/user-notifications') | ||
19 | .use(authenticateSocket) | ||
20 | .on('connection', socket => { | ||
21 | const userId = socket.handshake.query.user.id | ||
22 | |||
23 | logger.debug('User %d connected on the notification system.', userId) | ||
24 | |||
25 | this.userNotificationSockets[userId] = socket | ||
26 | |||
27 | socket.on('disconnect', () => { | ||
28 | logger.debug('User %d disconnected from SocketIO notifications.', userId) | ||
29 | |||
30 | delete this.userNotificationSockets[userId] | ||
31 | }) | ||
32 | }) | ||
33 | } | ||
34 | |||
35 | sendNotification (userId: number, notification: UserNotificationModel) { | ||
36 | const socket = this.userNotificationSockets[userId] | ||
37 | |||
38 | if (!socket) return | ||
39 | |||
40 | socket.emit('new-notification', notification.toFormattedJSON()) | ||
41 | } | ||
42 | |||
43 | static get Instance () { | ||
44 | return this.instance || (this.instance = new this()) | ||
45 | } | ||
46 | } | ||
47 | |||
48 | // --------------------------------------------------------------------------- | ||
49 | |||
50 | export { | ||
51 | PeerTubeSocket | ||
52 | } | ||
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index abd75d512..3628c0583 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -2,7 +2,13 @@ import * as express from 'express' | |||
2 | import { createClient, RedisClient } from 'redis' | 2 | import { createClient, RedisClient } from 'redis' |
3 | import { logger } from '../helpers/logger' | 3 | import { logger } from '../helpers/logger' |
4 | import { generateRandomString } from '../helpers/utils' | 4 | import { generateRandomString } from '../helpers/utils' |
5 | import { CONFIG, USER_PASSWORD_RESET_LIFETIME, USER_EMAIL_VERIFY_LIFETIME, VIDEO_VIEW_LIFETIME } from '../initializers' | 5 | import { |
6 | CONFIG, | ||
7 | CONTACT_FORM_LIFETIME, | ||
8 | USER_EMAIL_VERIFY_LIFETIME, | ||
9 | USER_PASSWORD_RESET_LIFETIME, | ||
10 | VIDEO_VIEW_LIFETIME | ||
11 | } from '../initializers' | ||
6 | 12 | ||
7 | type CachedRoute = { | 13 | type CachedRoute = { |
8 | body: string, | 14 | body: string, |
@@ -76,6 +82,16 @@ class Redis { | |||
76 | return this.getValue(this.generateVerifyEmailKey(userId)) | 82 | return this.getValue(this.generateVerifyEmailKey(userId)) |
77 | } | 83 | } |
78 | 84 | ||
85 | /************* Contact form per IP *************/ | ||
86 | |||
87 | async setContactFormIp (ip: string) { | ||
88 | return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) | ||
89 | } | ||
90 | |||
91 | async isContactFormIpExists (ip: string) { | ||
92 | return this.exists(this.generateContactFormKey(ip)) | ||
93 | } | ||
94 | |||
79 | /************* Views per IP *************/ | 95 | /************* Views per IP *************/ |
80 | 96 | ||
81 | setIPVideoView (ip: string, videoUUID: string) { | 97 | setIPVideoView (ip: string, videoUUID: string) { |
@@ -121,7 +137,14 @@ class Redis { | |||
121 | const key = this.generateVideoViewKey(videoId, hour) | 137 | const key = this.generateVideoViewKey(videoId, hour) |
122 | 138 | ||
123 | const valueString = await this.getValue(key) | 139 | const valueString = await this.getValue(key) |
124 | return parseInt(valueString, 10) | 140 | const valueInt = parseInt(valueString, 10) |
141 | |||
142 | if (isNaN(valueInt)) { | ||
143 | logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) | ||
144 | return undefined | ||
145 | } | ||
146 | |||
147 | return valueInt | ||
125 | } | 148 | } |
126 | 149 | ||
127 | async getVideosIdViewed (hour: number) { | 150 | async getVideosIdViewed (hour: number) { |
@@ -168,7 +191,11 @@ class Redis { | |||
168 | } | 191 | } |
169 | 192 | ||
170 | private generateViewKey (ip: string, videoUUID: string) { | 193 | private generateViewKey (ip: string, videoUUID: string) { |
171 | return videoUUID + '-' + ip | 194 | return `views-${videoUUID}-${ip}` |
195 | } | ||
196 | |||
197 | private generateContactFormKey (ip: string) { | ||
198 | return 'contact-form-' + ip | ||
172 | } | 199 | } |
173 | 200 | ||
174 | /************* Redis helpers *************/ | 201 | /************* Redis helpers *************/ |
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts index b9d0a4d17..86ea7aa38 100644 --- a/server/lib/schedulers/abstract-scheduler.ts +++ b/server/lib/schedulers/abstract-scheduler.ts | |||
@@ -1,8 +1,11 @@ | |||
1 | import { logger } from '../../helpers/logger' | ||
2 | |||
1 | export abstract class AbstractScheduler { | 3 | export abstract class AbstractScheduler { |
2 | 4 | ||
3 | protected abstract schedulerIntervalMs: number | 5 | protected abstract schedulerIntervalMs: number |
4 | 6 | ||
5 | private interval: NodeJS.Timer | 7 | private interval: NodeJS.Timer |
8 | private isRunning = false | ||
6 | 9 | ||
7 | enable () { | 10 | enable () { |
8 | if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') | 11 | if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') |
@@ -14,5 +17,18 @@ export abstract class AbstractScheduler { | |||
14 | clearInterval(this.interval) | 17 | clearInterval(this.interval) |
15 | } | 18 | } |
16 | 19 | ||
17 | abstract execute () | 20 | async execute () { |
21 | if (this.isRunning === true) return | ||
22 | this.isRunning = true | ||
23 | |||
24 | try { | ||
25 | await this.internalExecute() | ||
26 | } catch (err) { | ||
27 | logger.error('Cannot execute %s scheduler.', this.constructor.name, { err }) | ||
28 | } finally { | ||
29 | this.isRunning = false | ||
30 | } | ||
31 | } | ||
32 | |||
33 | protected abstract internalExecute (): Promise<any> | ||
18 | } | 34 | } |
diff --git a/server/lib/schedulers/bad-actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts index 617149aaf..3967be7f8 100644 --- a/server/lib/schedulers/bad-actor-follow-scheduler.ts +++ b/server/lib/schedulers/actor-follow-scheduler.ts | |||
@@ -3,18 +3,35 @@ import { logger } from '../../helpers/logger' | |||
3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' |
4 | import { AbstractScheduler } from './abstract-scheduler' | 4 | import { AbstractScheduler } from './abstract-scheduler' |
5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' | 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' |
6 | import { ActorFollowScoreCache } from '../cache' | ||
6 | 7 | ||
7 | export class BadActorFollowScheduler extends AbstractScheduler { | 8 | export class ActorFollowScheduler extends AbstractScheduler { |
8 | 9 | ||
9 | private static instance: AbstractScheduler | 10 | private static instance: AbstractScheduler |
10 | 11 | ||
11 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.badActorFollow | 12 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.actorFollowScores |
12 | 13 | ||
13 | private constructor () { | 14 | private constructor () { |
14 | super() | 15 | super() |
15 | } | 16 | } |
16 | 17 | ||
17 | async execute () { | 18 | protected async internalExecute () { |
19 | await this.processPendingScores() | ||
20 | |||
21 | await this.removeBadActorFollows() | ||
22 | } | ||
23 | |||
24 | private async processPendingScores () { | ||
25 | const pendingScores = ActorFollowScoreCache.Instance.getPendingFollowsScoreCopy() | ||
26 | |||
27 | ActorFollowScoreCache.Instance.clearPendingFollowsScore() | ||
28 | |||
29 | for (const inbox of Object.keys(pendingScores)) { | ||
30 | await ActorFollowModel.updateFollowScore(inbox, pendingScores[inbox]) | ||
31 | } | ||
32 | } | ||
33 | |||
34 | private async removeBadActorFollows () { | ||
18 | if (!isTestInstance()) logger.info('Removing bad actor follows (scheduler).') | 35 | if (!isTestInstance()) logger.info('Removing bad actor follows (scheduler).') |
19 | 36 | ||
20 | try { | 37 | try { |
diff --git a/server/lib/schedulers/remove-old-jobs-scheduler.ts b/server/lib/schedulers/remove-old-jobs-scheduler.ts index a29a6b800..4a4341ba9 100644 --- a/server/lib/schedulers/remove-old-jobs-scheduler.ts +++ b/server/lib/schedulers/remove-old-jobs-scheduler.ts | |||
@@ -14,10 +14,10 @@ export class RemoveOldJobsScheduler extends AbstractScheduler { | |||
14 | super() | 14 | super() |
15 | } | 15 | } |
16 | 16 | ||
17 | async execute () { | 17 | protected internalExecute () { |
18 | if (!isTestInstance()) logger.info('Removing old jobs (scheduler).') | 18 | if (!isTestInstance()) logger.info('Removing old jobs in scheduler.') |
19 | 19 | ||
20 | JobQueue.Instance.removeOldJobs() | 20 | return JobQueue.Instance.removeOldJobs() |
21 | } | 21 | } |
22 | 22 | ||
23 | static get Instance () { | 23 | static get Instance () { |
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index fd2edfd17..2618a5857 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts | |||
@@ -5,6 +5,8 @@ import { retryTransactionWrapper } from '../../helpers/database-utils' | |||
5 | import { federateVideoIfNeeded } from '../activitypub' | 5 | import { federateVideoIfNeeded } from '../activitypub' |
6 | import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' | 6 | import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | 7 | import { VideoPrivacy } from '../../../shared/models/videos' |
8 | import { Notifier } from '../notifier' | ||
9 | import { VideoModel } from '../../models/video/video' | ||
8 | 10 | ||
9 | export class UpdateVideosScheduler extends AbstractScheduler { | 11 | export class UpdateVideosScheduler extends AbstractScheduler { |
10 | 12 | ||
@@ -12,30 +14,20 @@ export class UpdateVideosScheduler extends AbstractScheduler { | |||
12 | 14 | ||
13 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.updateVideos | 15 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.updateVideos |
14 | 16 | ||
15 | private isRunning = false | ||
16 | |||
17 | private constructor () { | 17 | private constructor () { |
18 | super() | 18 | super() |
19 | } | 19 | } |
20 | 20 | ||
21 | async execute () { | 21 | protected async internalExecute () { |
22 | if (this.isRunning === true) return | 22 | return retryTransactionWrapper(this.updateVideos.bind(this)) |
23 | this.isRunning = true | ||
24 | |||
25 | try { | ||
26 | await retryTransactionWrapper(this.updateVideos.bind(this)) | ||
27 | } catch (err) { | ||
28 | logger.error('Cannot execute update videos scheduler.', { err }) | ||
29 | } finally { | ||
30 | this.isRunning = false | ||
31 | } | ||
32 | } | 23 | } |
33 | 24 | ||
34 | private async updateVideos () { | 25 | private async updateVideos () { |
35 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined | 26 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined |
36 | 27 | ||
37 | return sequelizeTypescript.transaction(async t => { | 28 | const publishedVideos = await sequelizeTypescript.transaction(async t => { |
38 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) | 29 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) |
30 | const publishedVideos: VideoModel[] = [] | ||
39 | 31 | ||
40 | for (const schedule of schedules) { | 32 | for (const schedule of schedules) { |
41 | const video = schedule.Video | 33 | const video = schedule.Video |
@@ -50,11 +42,23 @@ export class UpdateVideosScheduler extends AbstractScheduler { | |||
50 | 42 | ||
51 | await video.save({ transaction: t }) | 43 | await video.save({ transaction: t }) |
52 | await federateVideoIfNeeded(video, isNewVideo, t) | 44 | await federateVideoIfNeeded(video, isNewVideo, t) |
45 | |||
46 | if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) { | ||
47 | video.ScheduleVideoUpdate = schedule | ||
48 | publishedVideos.push(video) | ||
49 | } | ||
53 | } | 50 | } |
54 | 51 | ||
55 | await schedule.destroy({ transaction: t }) | 52 | await schedule.destroy({ transaction: t }) |
56 | } | 53 | } |
54 | |||
55 | return publishedVideos | ||
57 | }) | 56 | }) |
57 | |||
58 | for (const v of publishedVideos) { | ||
59 | Notifier.Instance.notifyOnNewVideo(v) | ||
60 | Notifier.Instance.notifyOnPendingVideoPublished(v) | ||
61 | } | ||
58 | } | 62 | } |
59 | 63 | ||
60 | static get Instance () { | 64 | static get Instance () { |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c49a8c89a..1a48f2bd0 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,22 +1,31 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' | 2 | import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | import { VideoFileModel } from '../../models/video/video-file' | 6 | import { VideoFileModel } from '../../models/video/video-file' |
7 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | 7 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' |
8 | import { join } from 'path' | 8 | import { join } from 'path' |
9 | import { rename } from 'fs-extra' | 9 | import { move } from 'fs-extra' |
10 | import { getServerActor } from '../../helpers/utils' | 10 | import { getServerActor } from '../../helpers/utils' |
11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
12 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 12 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
13 | import { removeVideoRedundancy } from '../redundancy' | 13 | import { removeVideoRedundancy } from '../redundancy' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | 14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { downloadPlaylistSegments } from '../hls' | ||
18 | |||
19 | type CandidateToDuplicate = { | ||
20 | redundancy: VideosRedundancy, | ||
21 | video: VideoModel, | ||
22 | files: VideoFileModel[], | ||
23 | streamingPlaylists: VideoStreamingPlaylistModel[] | ||
24 | } | ||
15 | 25 | ||
16 | export class VideosRedundancyScheduler extends AbstractScheduler { | 26 | export class VideosRedundancyScheduler extends AbstractScheduler { |
17 | 27 | ||
18 | private static instance: AbstractScheduler | 28 | private static instance: AbstractScheduler |
19 | private executing = false | ||
20 | 29 | ||
21 | protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL | 30 | protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL |
22 | 31 | ||
@@ -24,41 +33,39 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
24 | super() | 33 | super() |
25 | } | 34 | } |
26 | 35 | ||
27 | async execute () { | 36 | protected async internalExecute () { |
28 | if (this.executing) return | 37 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
29 | 38 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) | |
30 | this.executing = true | ||
31 | |||
32 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | ||
33 | logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) | ||
34 | 39 | ||
35 | try { | 40 | try { |
36 | const videoToDuplicate = await this.findVideoToDuplicate(obj) | 41 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) |
37 | if (!videoToDuplicate) continue | 42 | if (!videoToDuplicate) continue |
38 | 43 | ||
39 | const videoFiles = videoToDuplicate.VideoFiles | 44 | const candidateToDuplicate = { |
40 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 45 | video: videoToDuplicate, |
46 | redundancy: redundancyConfig, | ||
47 | files: videoToDuplicate.VideoFiles, | ||
48 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
49 | } | ||
41 | 50 | ||
42 | await this.purgeCacheIfNeeded(obj, videoFiles) | 51 | await this.purgeCacheIfNeeded(candidateToDuplicate) |
43 | 52 | ||
44 | if (await this.isTooHeavy(obj, videoFiles)) { | 53 | if (await this.isTooHeavy(candidateToDuplicate)) { |
45 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 54 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
46 | continue | 55 | continue |
47 | } | 56 | } |
48 | 57 | ||
49 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | 58 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) |
50 | 59 | ||
51 | await this.createVideoRedundancy(obj, videoFiles) | 60 | await this.createVideoRedundancies(candidateToDuplicate) |
52 | } catch (err) { | 61 | } catch (err) { |
53 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | 62 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) |
54 | } | 63 | } |
55 | } | 64 | } |
56 | 65 | ||
57 | await this.extendsLocalExpiration() | 66 | await this.extendsLocalExpiration() |
58 | 67 | ||
59 | await this.purgeRemoteExpired() | 68 | await this.purgeRemoteExpired() |
60 | |||
61 | this.executing = false | ||
62 | } | 69 | } |
63 | 70 | ||
64 | static get Instance () { | 71 | static get Instance () { |
@@ -70,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
70 | 77 | ||
71 | for (const redundancyModel of expired) { | 78 | for (const redundancyModel of expired) { |
72 | try { | 79 | try { |
73 | await this.extendsOrDeleteRedundancy(redundancyModel) | 80 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
81 | const candidate = { | ||
82 | redundancy: redundancyConfig, | ||
83 | video: null, | ||
84 | files: [], | ||
85 | streamingPlaylists: [] | ||
86 | } | ||
87 | |||
88 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | ||
89 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { | ||
90 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | ||
91 | await removeVideoRedundancy(redundancyModel) | ||
92 | } else { | ||
93 | await this.extendsRedundancy(redundancyModel) | ||
94 | } | ||
74 | } catch (err) { | 95 | } catch (err) { |
75 | logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) | 96 | logger.error( |
97 | 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), | ||
98 | { err } | ||
99 | ) | ||
76 | } | 100 | } |
77 | } | 101 | } |
78 | } | 102 | } |
79 | 103 | ||
80 | private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { | 104 | private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { |
81 | // Refresh the video, maybe it was deleted | ||
82 | const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) | ||
83 | |||
84 | if (!video) { | ||
85 | logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) | ||
86 | |||
87 | await redundancyModel.destroy() | ||
88 | return | ||
89 | } | ||
90 | |||
91 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 105 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
106 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | ||
107 | if (!redundancy) await removeVideoRedundancy(redundancyModel) | ||
108 | |||
92 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | 109 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) |
93 | } | 110 | } |
94 | 111 | ||
@@ -119,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
119 | } | 136 | } |
120 | } | 137 | } |
121 | 138 | ||
122 | private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 139 | private async createVideoRedundancies (data: CandidateToDuplicate) { |
123 | const serverActor = await getServerActor() | 140 | const video = await this.loadAndRefreshVideo(data.video.url) |
124 | 141 | ||
125 | for (const file of filesToDuplicate) { | 142 | if (!video) { |
126 | const video = await this.loadAndRefreshVideo(file.Video.url) | 143 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) |
127 | 144 | ||
145 | return | ||
146 | } | ||
147 | |||
148 | for (const file of data.files) { | ||
128 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) | 149 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) |
129 | if (existingRedundancy) { | 150 | if (existingRedundancy) { |
130 | await this.extendsOrDeleteRedundancy(existingRedundancy) | 151 | await this.extendsRedundancy(existingRedundancy) |
131 | 152 | ||
132 | continue | 153 | continue |
133 | } | 154 | } |
134 | 155 | ||
135 | if (!video) { | 156 | await this.createVideoFileRedundancy(data.redundancy, video, file) |
136 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) | 157 | } |
158 | |||
159 | for (const streamingPlaylist of data.streamingPlaylists) { | ||
160 | const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) | ||
161 | if (existingRedundancy) { | ||
162 | await this.extendsRedundancy(existingRedundancy) | ||
137 | 163 | ||
138 | continue | 164 | continue |
139 | } | 165 | } |
140 | 166 | ||
141 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 167 | await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) |
168 | } | ||
169 | } | ||
142 | 170 | ||
143 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 171 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { |
144 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | 172 | file.Video = video |
145 | 173 | ||
146 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | 174 | const serverActor = await getServerActor() |
147 | 175 | ||
148 | const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) | 176 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
149 | await rename(tmpPath, destPath) | ||
150 | 177 | ||
151 | const createdModel = await VideoRedundancyModel.create({ | 178 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
152 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 179 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) |
153 | url: getVideoCacheFileActivityPubUrl(file), | ||
154 | fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), | ||
155 | strategy: redundancy.strategy, | ||
156 | videoFileId: file.id, | ||
157 | actorId: serverActor.id | ||
158 | }) | ||
159 | createdModel.VideoFile = file | ||
160 | 180 | ||
161 | await sendCreateCacheFile(serverActor, createdModel) | 181 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) |
162 | 182 | ||
163 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 183 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) |
164 | } | 184 | await move(tmpPath, destPath) |
185 | |||
186 | const createdModel = await VideoRedundancyModel.create({ | ||
187 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
188 | url: getVideoCacheFileActivityPubUrl(file), | ||
189 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
190 | strategy: redundancy.strategy, | ||
191 | videoFileId: file.id, | ||
192 | actorId: serverActor.id | ||
193 | }) | ||
194 | |||
195 | createdModel.VideoFile = file | ||
196 | |||
197 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
198 | |||
199 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | ||
200 | } | ||
201 | |||
202 | private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
203 | playlist.Video = video | ||
204 | |||
205 | const serverActor = await getServerActor() | ||
206 | |||
207 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) | ||
208 | |||
209 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | ||
210 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | ||
211 | |||
212 | const createdModel = await VideoRedundancyModel.create({ | ||
213 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
214 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | ||
215 | fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL), | ||
216 | strategy: redundancy.strategy, | ||
217 | videoStreamingPlaylistId: playlist.id, | ||
218 | actorId: serverActor.id | ||
219 | }) | ||
220 | |||
221 | createdModel.VideoStreamingPlaylist = playlist | ||
222 | |||
223 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
224 | |||
225 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) | ||
165 | } | 226 | } |
166 | 227 | ||
167 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { | 228 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { |
@@ -175,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
175 | await sendUpdateCacheFile(serverActor, redundancy) | 236 | await sendUpdateCacheFile(serverActor, redundancy) |
176 | } | 237 | } |
177 | 238 | ||
178 | private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 239 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { |
179 | while (this.isTooHeavy(redundancy, filesToDuplicate)) { | 240 | while (this.isTooHeavy(candidateToDuplicate)) { |
241 | const redundancy = candidateToDuplicate.redundancy | ||
180 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | 242 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) |
181 | if (!toDelete) return | 243 | if (!toDelete) return |
182 | 244 | ||
@@ -184,12 +246,13 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
184 | } | 246 | } |
185 | } | 247 | } |
186 | 248 | ||
187 | private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 249 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { |
188 | const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate) | 250 | const maxSize = candidateToDuplicate.redundancy.size |
189 | 251 | ||
190 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) | 252 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) |
253 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) | ||
191 | 254 | ||
192 | return totalDuplicated > maxSize | 255 | return totalWillDuplicate > maxSize |
193 | } | 256 | } |
194 | 257 | ||
195 | private buildNewExpiration (expiresAfterMs: number) { | 258 | private buildNewExpiration (expiresAfterMs: number) { |
@@ -197,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
197 | } | 260 | } |
198 | 261 | ||
199 | private buildEntryLogId (object: VideoRedundancyModel) { | 262 | private buildEntryLogId (object: VideoRedundancyModel) { |
200 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | 263 | if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` |
264 | |||
265 | return `${object.VideoStreamingPlaylist.playlistUrl}` | ||
201 | } | 266 | } |
202 | 267 | ||
203 | private getTotalFileSizes (files: VideoFileModel[]) { | 268 | private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { |
204 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | 269 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size |
205 | 270 | ||
206 | return files.reduce(fileReducer, 0) | 271 | return files.reduce(fileReducer, 0) * playlists.length |
207 | } | 272 | } |
208 | 273 | ||
209 | private async loadAndRefreshVideo (videoUrl: string) { | 274 | private async loadAndRefreshVideo (videoUrl: string) { |
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index 461cd045e..aa027116d 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts | |||
@@ -12,7 +12,7 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { | |||
12 | super() | 12 | super() |
13 | } | 13 | } |
14 | 14 | ||
15 | execute () { | 15 | protected internalExecute () { |
16 | return updateYoutubeDLBinary() | 16 | return updateYoutubeDLBinary() |
17 | } | 17 | } |
18 | 18 | ||
diff --git a/server/lib/user.ts b/server/lib/user.ts index db29469eb..a39ef6c3d 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts | |||
@@ -9,6 +9,8 @@ import { createVideoChannel } from './video-channel' | |||
9 | import { VideoChannelModel } from '../models/video/video-channel' | 9 | import { VideoChannelModel } from '../models/video/video-channel' |
10 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | 10 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' |
11 | import { ActorModel } from '../models/activitypub/actor' | 11 | import { ActorModel } from '../models/activitypub/actor' |
12 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' | ||
13 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' | ||
12 | 14 | ||
13 | async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { | 15 | async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { |
14 | const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { | 16 | const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { |
@@ -18,7 +20,9 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse | |||
18 | } | 20 | } |
19 | 21 | ||
20 | const userCreated = await userToCreate.save(userOptions) | 22 | const userCreated = await userToCreate.save(userOptions) |
21 | const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) | 23 | userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t) |
24 | |||
25 | const accountCreated = await createLocalAccountWithoutKeys(userCreated.username, userCreated.id, null, t) | ||
22 | userCreated.Account = accountCreated | 26 | userCreated.Account = accountCreated |
23 | 27 | ||
24 | let channelName = userCreated.username + '_channel' | 28 | let channelName = userCreated.username + '_channel' |
@@ -37,8 +41,13 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse | |||
37 | return { user: userCreated, account: accountCreated, videoChannel } | 41 | return { user: userCreated, account: accountCreated, videoChannel } |
38 | }) | 42 | }) |
39 | 43 | ||
40 | account.Actor = await setAsyncActorKeys(account.Actor) | 44 | const [ accountKeys, channelKeys ] = await Promise.all([ |
41 | videoChannel.Actor = await setAsyncActorKeys(videoChannel.Actor) | 45 | setAsyncActorKeys(account.Actor), |
46 | setAsyncActorKeys(videoChannel.Actor) | ||
47 | ]) | ||
48 | |||
49 | account.Actor = accountKeys | ||
50 | videoChannel.Actor = channelKeys | ||
42 | 51 | ||
43 | return { user, account, videoChannel } as { user: UserModel, account: AccountModel, videoChannel: VideoChannelModel } | 52 | return { user, account, videoChannel } as { user: UserModel, account: AccountModel, videoChannel: VideoChannelModel } |
44 | } | 53 | } |
@@ -83,3 +92,22 @@ export { | |||
83 | createUserAccountAndChannel, | 92 | createUserAccountAndChannel, |
84 | createLocalAccountWithoutKeys | 93 | createLocalAccountWithoutKeys |
85 | } | 94 | } |
95 | |||
96 | // --------------------------------------------------------------------------- | ||
97 | |||
98 | function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Transaction | undefined) { | ||
99 | const values: UserNotificationSetting & { userId: number } = { | ||
100 | userId: user.id, | ||
101 | newVideoFromSubscription: UserNotificationSettingValue.WEB, | ||
102 | newCommentOnMyVideo: UserNotificationSettingValue.WEB, | ||
103 | myVideoImportFinished: UserNotificationSettingValue.WEB, | ||
104 | myVideoPublished: UserNotificationSettingValue.WEB, | ||
105 | videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
106 | blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
107 | newUserRegistration: UserNotificationSettingValue.WEB, | ||
108 | commentMention: UserNotificationSettingValue.WEB, | ||
109 | newFollow: UserNotificationSettingValue.WEB | ||
110 | } | ||
111 | |||
112 | return UserNotificationSettingModel.create(values, { transaction: t }) | ||
113 | } | ||
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts index 70ba7c303..59bce7520 100644 --- a/server/lib/video-comment.ts +++ b/server/lib/video-comment.ts | |||
@@ -64,10 +64,8 @@ function buildFormattedCommentTree (resultList: ResultList<VideoCommentModel>): | |||
64 | } | 64 | } |
65 | 65 | ||
66 | const parentCommentThread = idx[childComment.inReplyToCommentId] | 66 | const parentCommentThread = idx[childComment.inReplyToCommentId] |
67 | if (!parentCommentThread) { | 67 | // Maybe the parent comment was blocked by the admin/user |
68 | const msg = `Cannot format video thread tree, parent ${childComment.inReplyToCommentId} not found for child ${childComment.id}` | 68 | if (!parentCommentThread) continue |
69 | throw new Error(msg) | ||
70 | } | ||
71 | 69 | ||
72 | parentCommentThread.children.push(childCommentThread) | 70 | parentCommentThread.children.push(childCommentThread) |
73 | idx[childComment.id] = childCommentThread | 71 | idx[childComment.id] = childCommentThread |
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index bf3ff78c2..086b860a2 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts | |||
@@ -1,22 +1,27 @@ | |||
1 | import { CONFIG } from '../initializers' | 1 | import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' |
2 | import { join, extname } from 'path' | 2 | import { extname, join } from 'path' |
3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' | 3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' |
4 | import { copy, remove, rename, stat } from 'fs-extra' | 4 | import { copy, ensureDir, move, remove, stat } from 'fs-extra' |
5 | import { logger } from '../helpers/logger' | 5 | import { logger } from '../helpers/logger' |
6 | import { VideoResolution } from '../../shared/models/videos' | 6 | import { VideoResolution } from '../../shared/models/videos' |
7 | import { VideoFileModel } from '../models/video/video-file' | 7 | import { VideoFileModel } from '../models/video/video-file' |
8 | import { VideoModel } from '../models/video/video' | 8 | import { VideoModel } from '../models/video/video' |
9 | import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' | ||
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
11 | import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' | ||
9 | 12 | ||
10 | async function optimizeOriginalVideofile (video: VideoModel) { | 13 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { |
11 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 14 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
12 | const newExtname = '.mp4' | 15 | const newExtname = '.mp4' |
13 | const inputVideoFile = video.getOriginalFile() | 16 | |
17 | const inputVideoFile = inputVideoFileArg ? inputVideoFileArg : video.getOriginalFile() | ||
14 | const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) | 18 | const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) |
15 | const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) | 19 | const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) |
16 | 20 | ||
17 | const transcodeOptions = { | 21 | const transcodeOptions = { |
18 | inputPath: videoInputPath, | 22 | inputPath: videoInputPath, |
19 | outputPath: videoTranscodedPath | 23 | outputPath: videoTranscodedPath, |
24 | resolution: inputVideoFile.resolution | ||
20 | } | 25 | } |
21 | 26 | ||
22 | // Could be very long! | 27 | // Could be very long! |
@@ -29,7 +34,7 @@ async function optimizeOriginalVideofile (video: VideoModel) { | |||
29 | inputVideoFile.set('extname', newExtname) | 34 | inputVideoFile.set('extname', newExtname) |
30 | 35 | ||
31 | const videoOutputPath = video.getVideoFilePath(inputVideoFile) | 36 | const videoOutputPath = video.getVideoFilePath(inputVideoFile) |
32 | await rename(videoTranscodedPath, videoOutputPath) | 37 | await move(videoTranscodedPath, videoOutputPath) |
33 | const stats = await stat(videoOutputPath) | 38 | const stats = await stat(videoOutputPath) |
34 | const fps = await getVideoFileFPS(videoOutputPath) | 39 | const fps = await getVideoFileFPS(videoOutputPath) |
35 | 40 | ||
@@ -46,7 +51,7 @@ async function optimizeOriginalVideofile (video: VideoModel) { | |||
46 | } | 51 | } |
47 | } | 52 | } |
48 | 53 | ||
49 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | 54 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { |
50 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 55 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
51 | const extname = '.mp4' | 56 | const extname = '.mp4' |
52 | 57 | ||
@@ -59,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
59 | size: 0, | 64 | size: 0, |
60 | videoId: video.id | 65 | videoId: video.id |
61 | }) | 66 | }) |
62 | const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) | 67 | const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) |
63 | 68 | ||
64 | const transcodeOptions = { | 69 | const transcodeOptions = { |
65 | inputPath: videoInputPath, | 70 | inputPath: videoInputPath, |
66 | outputPath: videoOutputPath, | 71 | outputPath: videoOutputPath, |
67 | resolution, | 72 | resolution, |
68 | isPortraitMode | 73 | isPortraitMode: isPortrait |
69 | } | 74 | } |
70 | 75 | ||
71 | await transcode(transcodeOptions) | 76 | await transcode(transcodeOptions) |
@@ -83,6 +88,41 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
83 | video.VideoFiles.push(newVideoFile) | 88 | video.VideoFiles.push(newVideoFile) |
84 | } | 89 | } |
85 | 90 | ||
91 | async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | ||
92 | const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) | ||
93 | await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid)) | ||
94 | |||
95 | const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) | ||
96 | const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) | ||
97 | |||
98 | const transcodeOptions = { | ||
99 | inputPath: videoInputPath, | ||
100 | outputPath, | ||
101 | resolution, | ||
102 | isPortraitMode, | ||
103 | |||
104 | hlsPlaylist: { | ||
105 | videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) | ||
106 | } | ||
107 | } | ||
108 | |||
109 | await transcode(transcodeOptions) | ||
110 | |||
111 | await updateMasterHLSPlaylist(video) | ||
112 | await updateSha256Segments(video) | ||
113 | |||
114 | const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) | ||
115 | |||
116 | await VideoStreamingPlaylistModel.upsert({ | ||
117 | videoId: video.id, | ||
118 | playlistUrl, | ||
119 | segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), | ||
120 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), | ||
121 | |||
122 | type: VideoStreamingPlaylistType.HLS | ||
123 | }) | ||
124 | } | ||
125 | |||
86 | async function importVideoFile (video: VideoModel, inputFilePath: string) { | 126 | async function importVideoFile (video: VideoModel, inputFilePath: string) { |
87 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | 127 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) |
88 | const { size } = await stat(inputFilePath) | 128 | const { size } = await stat(inputFilePath) |
@@ -124,7 +164,8 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) { | |||
124 | } | 164 | } |
125 | 165 | ||
126 | export { | 166 | export { |
127 | optimizeOriginalVideofile, | 167 | generateHlsPlaylist, |
168 | optimizeVideofile, | ||
128 | transcodeOriginalVideofile, | 169 | transcodeOriginalVideofile, |
129 | importVideoFile | 170 | importVideoFile |
130 | } | 171 | } |