aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-02-11 11:52:34 +0100
committerChocobozzz <me@florianbigard.com>2019-02-11 11:52:34 +0100
commit88108880bbdba473cfe36ecbebc1c3c4f972e102 (patch)
treeb242efb3b4f0d7e49d88f2d1f2063b5b3b0489c0 /server/lib
parent53a94c7cfa8368da4cd248d65df8346905938f0c (diff)
parent9b712a2017e4ab3cf12cd6bd58278905520159d0 (diff)
downloadPeerTube-88108880bbdba473cfe36ecbebc1c3c4f972e102.tar.gz
PeerTube-88108880bbdba473cfe36ecbebc1c3c4f972e102.tar.zst
PeerTube-88108880bbdba473cfe36ecbebc1c3c4f972e102.zip
Merge branch 'develop' into pr/1217
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts157
-rw-r--r--server/lib/activitypub/cache-file.ts23
-rw-r--r--server/lib/activitypub/crawl.ts7
-rw-r--r--server/lib/activitypub/process/index.ts8
-rw-r--r--server/lib/activitypub/process/process-accept.ts1
-rw-r--r--server/lib/activitypub/process/process-announce.ts8
-rw-r--r--server/lib/activitypub/process/process-create.ts122
-rw-r--r--server/lib/activitypub/process/process-dislike.ts52
-rw-r--r--server/lib/activitypub/process/process-flag.ts49
-rw-r--r--server/lib/activitypub/process/process-follow.ts14
-rw-r--r--server/lib/activitypub/process/process-like.ts6
-rw-r--r--server/lib/activitypub/process/process-undo.ts14
-rw-r--r--server/lib/activitypub/process/process-update.ts3
-rw-r--r--server/lib/activitypub/process/process-view.ts35
-rw-r--r--server/lib/activitypub/process/process.ts35
-rw-r--r--server/lib/activitypub/send/send-create.ts74
-rw-r--r--server/lib/activitypub/send/send-dislike.ts41
-rw-r--r--server/lib/activitypub/send/send-flag.ts39
-rw-r--r--server/lib/activitypub/send/send-like.ts2
-rw-r--r--server/lib/activitypub/send/send-undo.ts17
-rw-r--r--server/lib/activitypub/send/send-update.ts2
-rw-r--r--server/lib/activitypub/send/send-view.ts40
-rw-r--r--server/lib/activitypub/share.ts31
-rw-r--r--server/lib/activitypub/url.ts19
-rw-r--r--server/lib/activitypub/video-comments.ts21
-rw-r--r--server/lib/activitypub/video-rates.ts41
-rw-r--r--server/lib/activitypub/videos.ts280
-rw-r--r--server/lib/avatar.ts3
-rw-r--r--server/lib/blocklist.ts40
-rw-r--r--server/lib/cache/actor-follow-score-cache.ts46
-rw-r--r--server/lib/cache/index.ts1
-rw-r--r--server/lib/client-html.ts61
-rw-r--r--server/lib/emailer.ts261
-rw-r--r--server/lib/hls.ts164
-rw-r--r--server/lib/job-queue/handlers/activitypub-follow.ts9
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-broadcast.ts8
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts2
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-unicast.ts11
-rw-r--r--server/lib/job-queue/handlers/activitypub-refresher.ts54
-rw-r--r--server/lib/job-queue/handlers/email.ts3
-rw-r--r--server/lib/job-queue/handlers/utils/activitypub-http-utils.ts29
-rw-r--r--server/lib/job-queue/handlers/video-file.ts106
-rw-r--r--server/lib/job-queue/handlers/video-import.ts27
-rw-r--r--server/lib/job-queue/handlers/video-views.ts21
-rw-r--r--server/lib/job-queue/job-queue.ts15
-rw-r--r--server/lib/notifier.ts455
-rw-r--r--server/lib/oauth-model.ts3
-rw-r--r--server/lib/peertube-socket.ts52
-rw-r--r--server/lib/redis.ts33
-rw-r--r--server/lib/schedulers/abstract-scheduler.ts18
-rw-r--r--server/lib/schedulers/actor-follow-scheduler.ts (renamed from server/lib/schedulers/bad-actor-follow-scheduler.ts)23
-rw-r--r--server/lib/schedulers/remove-old-jobs-scheduler.ts6
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts32
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts201
-rw-r--r--server/lib/schedulers/youtube-dl-update-scheduler.ts2
-rw-r--r--server/lib/user.ts34
-rw-r--r--server/lib/video-comment.ts6
-rw-r--r--server/lib/video-transcoding.ts63
58 files changed, 2287 insertions, 643 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 45dd4443d..a3f379b76 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -1,19 +1,18 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { join } from 'path'
3import { Transaction } from 'sequelize' 2import { Transaction } from 'sequelize'
4import * as url from 'url' 3import * as url from 'url'
5import * as uuidv4 from 'uuid/v4' 4import * as uuidv4 from 'uuid/v4'
6import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' 5import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
7import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' 6import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
8import { getActorUrl } from '../../helpers/activitypub' 7import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
9import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor' 8import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
10import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 9import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
11import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' 10import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
12import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
13import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' 12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
14import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 13import { doRequest, downloadImage } from '../../helpers/requests'
15import { getUrlFromWebfinger } from '../../helpers/webfinger' 14import { getUrlFromWebfinger } from '../../helpers/webfinger'
16import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' 15import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
17import { AccountModel } from '../../models/account/account' 16import { AccountModel } from '../../models/account/account'
18import { ActorModel } from '../../models/activitypub/actor' 17import { ActorModel } from '../../models/activitypub/actor'
19import { AvatarModel } from '../../models/avatar/avatar' 18import { AvatarModel } from '../../models/avatar/avatar'
@@ -43,7 +42,7 @@ async function getOrCreateActorAndServerAndModel (
43 recurseIfNeeded = true, 42 recurseIfNeeded = true,
44 updateCollections = false 43 updateCollections = false
45) { 44) {
46 const actorUrl = getActorUrl(activityActor) 45 const actorUrl = getAPId(activityActor)
47 let created = false 46 let created = false
48 47
49 let actor = await fetchActorByUrl(actorUrl, fetchType) 48 let actor = await fetchActorByUrl(actorUrl, fetchType)
@@ -65,8 +64,12 @@ async function getOrCreateActorAndServerAndModel (
65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') 64 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) 65 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
67 66
67 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
68 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
69 }
70
68 try { 71 try {
69 // Assert we don't recurse another time 72 // Don't recurse another time
70 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) 73 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
71 } catch (err) { 74 } catch (err) {
72 logger.error('Cannot get or create account attributed to video channel ' + actor.url) 75 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
@@ -168,18 +171,13 @@ async function fetchActorTotalItems (url: string) {
168 171
169async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { 172async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
170 if ( 173 if (
171 actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && 174 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
172 isActivityPubUrlValid(actorJSON.icon.url) 175 isActivityPubUrlValid(actorJSON.icon.url)
173 ) { 176 ) {
174 const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] 177 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
175 178
176 const avatarName = uuidv4() + extension 179 const avatarName = uuidv4() + extension
177 const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) 180 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
178
179 await doRequestAndSaveToFile({
180 method: 'GET',
181 uri: actorJSON.icon.url
182 }, destPath)
183 181
184 return avatarName 182 return avatarName
185 } 183 }
@@ -203,6 +201,69 @@ async function addFetchOutboxJob (actor: ActorModel) {
203 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) 201 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
204} 202}
205 203
204async function refreshActorIfNeeded (
205 actorArg: ActorModel,
206 fetchedType: ActorFetchByUrlType
207): Promise<{ actor: ActorModel, refreshed: boolean }> {
208 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
209
210 // We need more attributes
211 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
212
213 try {
214 let actorUrl: string
215 try {
216 actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
217 } catch (err) {
218 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
219 actorUrl = actor.url
220 }
221
222 const { result, statusCode } = await fetchRemoteActor(actorUrl)
223
224 if (statusCode === 404) {
225 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
226 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
227 return { actor: undefined, refreshed: false }
228 }
229
230 if (result === undefined) {
231 logger.warn('Cannot fetch remote actor in refresh actor.')
232 return { actor, refreshed: false }
233 }
234
235 return sequelizeTypescript.transaction(async t => {
236 updateInstanceWithAnother(actor, result.actor)
237
238 if (result.avatarName !== undefined) {
239 await updateActorAvatarInstance(actor, result.avatarName, t)
240 }
241
242 // Force update
243 actor.setDataValue('updatedAt', new Date())
244 await actor.save({ transaction: t })
245
246 if (actor.Account) {
247 actor.Account.set('name', result.name)
248 actor.Account.set('description', result.summary)
249
250 await actor.Account.save({ transaction: t })
251 } else if (actor.VideoChannel) {
252 actor.VideoChannel.set('name', result.name)
253 actor.VideoChannel.set('description', result.summary)
254 actor.VideoChannel.set('support', result.support)
255
256 await actor.VideoChannel.save({ transaction: t })
257 }
258
259 return { refreshed: true, actor }
260 })
261 } catch (err) {
262 logger.warn('Cannot refresh actor.', { err })
263 return { actor, refreshed: false }
264 }
265}
266
206export { 267export {
207 getOrCreateActorAndServerAndModel, 268 getOrCreateActorAndServerAndModel,
208 buildActorInstance, 269 buildActorInstance,
@@ -210,6 +271,7 @@ export {
210 fetchActorTotalItems, 271 fetchActorTotalItems,
211 fetchAvatarIfExists, 272 fetchAvatarIfExists,
212 updateActorInstance, 273 updateActorInstance,
274 refreshActorIfNeeded,
213 updateActorAvatarInstance, 275 updateActorAvatarInstance,
214 addFetchOutboxJob 276 addFetchOutboxJob
215} 277}
@@ -293,16 +355,19 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
293 355
294 logger.info('Fetching remote actor %s.', actorUrl) 356 logger.info('Fetching remote actor %s.', actorUrl)
295 357
296 const requestResult = await doRequest(options) 358 const requestResult = await doRequest<ActivityPubActor>(options)
297 normalizeActor(requestResult.body) 359 normalizeActor(requestResult.body)
298 360
299 const actorJSON: ActivityPubActor = requestResult.body 361 const actorJSON = requestResult.body
300
301 if (isActorObjectValid(actorJSON) === false) { 362 if (isActorObjectValid(actorJSON) === false) {
302 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON }) 363 logger.debug('Remote actor JSON is not valid.', { actorJSON })
303 return { result: undefined, statusCode: requestResult.response.statusCode } 364 return { result: undefined, statusCode: requestResult.response.statusCode }
304 } 365 }
305 366
367 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
368 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
369 }
370
306 const followersCount = await fetchActorTotalItems(actorJSON.followers) 371 const followersCount = await fetchActorTotalItems(actorJSON.followers)
307 const followingCount = await fetchActorTotalItems(actorJSON.following) 372 const followingCount = await fetchActorTotalItems(actorJSON.following)
308 373
@@ -371,59 +436,3 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu
371 436
372 return videoChannelCreated 437 return videoChannelCreated
373} 438}
374
375async function refreshActorIfNeeded (
376 actorArg: ActorModel,
377 fetchedType: ActorFetchByUrlType
378): Promise<{ actor: ActorModel, refreshed: boolean }> {
379 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
380
381 // We need more attributes
382 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
383
384 try {
385 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
386 const { result, statusCode } = await fetchRemoteActor(actorUrl)
387
388 if (statusCode === 404) {
389 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
390 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
391 return { actor: undefined, refreshed: false }
392 }
393
394 if (result === undefined) {
395 logger.warn('Cannot fetch remote actor in refresh actor.')
396 return { actor, refreshed: false }
397 }
398
399 return sequelizeTypescript.transaction(async t => {
400 updateInstanceWithAnother(actor, result.actor)
401
402 if (result.avatarName !== undefined) {
403 await updateActorAvatarInstance(actor, result.avatarName, t)
404 }
405
406 // Force update
407 actor.setDataValue('updatedAt', new Date())
408 await actor.save({ transaction: t })
409
410 if (actor.Account) {
411 actor.Account.set('name', result.name)
412 actor.Account.set('description', result.summary)
413
414 await actor.Account.save({ transaction: t })
415 } else if (actor.VideoChannel) {
416 actor.VideoChannel.set('name', result.name)
417 actor.VideoChannel.set('description', result.summary)
418 actor.VideoChannel.set('support', result.support)
419
420 await actor.VideoChannel.save({ transaction: t })
421 }
422
423 return { refreshed: true, actor }
424 })
425 } catch (err) {
426 logger.warn('Cannot refresh actor.', { err })
427 return { actor, refreshed: false }
428 }
429}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index f6f068b45..9a40414bb 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,11 +1,28 @@
1import { CacheFileObject } from '../../../shared/index' 1import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize' 4import { Transaction } from 'sequelize'
5import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5 6
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url
8 8
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14
15 return {
16 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 }
24
25 const url = cacheFileObject.url
9 const videoFile = video.VideoFiles.find(f => { 26 const videoFile = video.VideoFiles.find(f => {
10 return f.resolution === url.height && f.fps === url.fps 27 return f.resolution === url.height && f.fps === url.fps
11 }) 28 })
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
15 return { 32 return {
16 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id, 34 url: cacheFileObject.id,
18 fileUrl: cacheFileObject.url.href, 35 fileUrl: url.href,
19 strategy: null, 36 strategy: null,
20 videoFileId: videoFile.id, 37 videoFileId: videoFile.id,
21 actorId: byActor.id 38 actorId: byActor.id
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 55912341c..1b9b14c2e 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -1,7 +1,8 @@
1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' 1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
2import { doRequest } from '../../helpers/requests' 2import { doRequest } from '../../helpers/requests'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import Bluebird = require('bluebird') 4import * as Bluebird from 'bluebird'
5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
5 6
6async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { 7async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) {
7 logger.info('Crawling ActivityPub data on %s.', uri) 8 logger.info('Crawling ActivityPub data on %s.', uri)
@@ -14,7 +15,7 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
14 timeout: JOB_REQUEST_TIMEOUT 15 timeout: JOB_REQUEST_TIMEOUT
15 } 16 }
16 17
17 const response = await doRequest(options) 18 const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
18 const firstBody = response.body 19 const firstBody = response.body
19 20
20 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT 21 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
@@ -23,7 +24,7 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
23 while (nextLink && i < limit) { 24 while (nextLink && i < limit) {
24 options.uri = nextLink 25 options.uri = nextLink
25 26
26 const { body } = await doRequest(options) 27 const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options)
27 nextLink = body.next 28 nextLink = body.next
28 i++ 29 i++
29 30
diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts
index db4980a72..5466739c1 100644
--- a/server/lib/activitypub/process/index.ts
+++ b/server/lib/activitypub/process/index.ts
@@ -1,9 +1 @@
1export * from './process' export * from './process'
2export * from './process-accept'
3export * from './process-announce'
4export * from './process-create'
5export * from './process-delete'
6export * from './process-follow'
7export * from './process-like'
8export * from './process-undo'
9export * from './process-update'
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts
index 89bda9c32..ebb275e34 100644
--- a/server/lib/activitypub/process/process-accept.ts
+++ b/server/lib/activitypub/process/process-accept.ts
@@ -24,6 +24,7 @@ async function processAccept (actor: ActorModel, targetActor: ActorModel) {
24 if (follow.state !== 'accepted') { 24 if (follow.state !== 'accepted') {
25 follow.set('state', 'accepted') 25 follow.set('state', 'accepted')
26 await follow.save() 26 await follow.save()
27
27 await addFetchOutboxJob(targetActor) 28 await addFetchOutboxJob(targetActor)
28 } 29 }
29} 30}
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index cc88b5423..23310b41e 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -5,6 +5,8 @@ import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoShareModel } from '../../../models/video/video-share' 5import { VideoShareModel } from '../../../models/video/video-share'
6import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
7import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
8import { VideoPrivacy } from '../../../../shared/models/videos'
9import { Notifier } from '../../notifier'
8 10
9async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) { 11async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) {
10 return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) 12 return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity)
@@ -21,9 +23,9 @@ export {
21async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 23async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
22 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id 24 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
23 25
24 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) 26 const { video, created: videoCreated } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri })
25 27
26 return sequelizeTypescript.transaction(async t => { 28 await sequelizeTypescript.transaction(async t => {
27 // Add share entry 29 // Add share entry
28 30
29 const share = { 31 const share = {
@@ -49,4 +51,6 @@ async function processVideoShare (actorAnnouncer: ActorModel, activity: Activity
49 51
50 return undefined 52 return undefined
51 }) 53 })
54
55 if (videoCreated) Notifier.Instance.notifyOnNewVideo(video)
52} 56}
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index cefe89db0..5f4d793a5 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -1,34 +1,44 @@
1import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' 1import { ActivityCreate, CacheFileObject, VideoTorrentObject } from '../../../../shared'
2import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects'
3import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' 2import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
4import { retryTransactionWrapper } from '../../../helpers/database-utils' 3import { retryTransactionWrapper } from '../../../helpers/database-utils'
5import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
6import { sequelizeTypescript } from '../../../initializers' 5import { sequelizeTypescript } from '../../../initializers'
7import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
8import { ActorModel } from '../../../models/activitypub/actor' 6import { ActorModel } from '../../../models/activitypub/actor'
9import { VideoAbuseModel } from '../../../models/video/video-abuse'
10import { addVideoComment, resolveThread } from '../video-comments' 7import { addVideoComment, resolveThread } from '../video-comments'
11import { getOrCreateVideoAndAccountAndChannel } from '../videos' 8import { getOrCreateVideoAndAccountAndChannel } from '../videos'
12import { forwardVideoRelatedActivity } from '../send/utils' 9import { forwardVideoRelatedActivity } from '../send/utils'
13import { Redis } from '../../redis'
14import { createOrUpdateCacheFile } from '../cache-file' 10import { createOrUpdateCacheFile } from '../cache-file'
11import { Notifier } from '../../notifier'
12import { processViewActivity } from './process-view'
13import { processDislikeActivity } from './process-dislike'
14import { processFlagActivity } from './process-flag'
15 15
16async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { 16async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
17 const activityObject = activity.object 17 const activityObject = activity.object
18 const activityType = activityObject.type 18 const activityType = activityObject.type
19 19
20 if (activityType === 'View') { 20 if (activityType === 'View') {
21 return processCreateView(byActor, activity) 21 return processViewActivity(activity, byActor)
22 } else if (activityType === 'Dislike') { 22 }
23 return retryTransactionWrapper(processCreateDislike, byActor, activity) 23
24 } else if (activityType === 'Video') { 24 if (activityType === 'Dislike') {
25 return retryTransactionWrapper(processDislikeActivity, activity, byActor)
26 }
27
28 if (activityType === 'Flag') {
29 return retryTransactionWrapper(processFlagActivity, activity, byActor)
30 }
31
32 if (activityType === 'Video') {
25 return processCreateVideo(activity) 33 return processCreateVideo(activity)
26 } else if (activityType === 'Flag') { 34 }
27 return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) 35
28 } else if (activityType === 'Note') { 36 if (activityType === 'Note') {
29 return retryTransactionWrapper(processCreateVideoComment, byActor, activity) 37 return retryTransactionWrapper(processCreateVideoComment, activity, byActor)
30 } else if (activityType === 'CacheFile') { 38 }
31 return retryTransactionWrapper(processCacheFile, byActor, activity) 39
40 if (activityType === 'CacheFile') {
41 return retryTransactionWrapper(processCacheFile, activity, byActor)
32 } 42 }
33 43
34 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) 44 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -46,60 +56,14 @@ export {
46async function processCreateVideo (activity: ActivityCreate) { 56async function processCreateVideo (activity: ActivityCreate) {
47 const videoToCreateData = activity.object as VideoTorrentObject 57 const videoToCreateData = activity.object as VideoTorrentObject
48 58
49 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) 59 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData })
50
51 return video
52}
53
54async function processCreateDislike (byActor: ActorModel, activity: ActivityCreate) {
55 const dislike = activity.object as DislikeObject
56 const byAccount = byActor.Account
57
58 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
59
60 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
61
62 return sequelizeTypescript.transaction(async t => {
63 const rate = {
64 type: 'dislike' as 'dislike',
65 videoId: video.id,
66 accountId: byAccount.id
67 }
68 const [ , created ] = await AccountVideoRateModel.findOrCreate({
69 where: rate,
70 defaults: rate,
71 transaction: t
72 })
73 if (created === true) await video.increment('dislikes', { transaction: t })
74
75 if (video.isOwned() && created === true) {
76 // Don't resend the activity to the sender
77 const exceptions = [ byActor ]
78
79 await forwardVideoRelatedActivity(activity, t, exceptions, video)
80 }
81 })
82}
83
84async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
85 const view = activity.object as ViewObject
86
87 const options = {
88 videoObject: view.object,
89 fetchType: 'only-video' as 'only-video'
90 }
91 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
92 60
93 await Redis.Instance.addVideoView(video.id) 61 if (created) Notifier.Instance.notifyOnNewVideo(video)
94 62
95 if (video.isOwned()) { 63 return video
96 // Don't resend the activity to the sender
97 const exceptions = [ byActor ]
98 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
99 }
100} 64}
101 65
102async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { 66async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) {
103 const cacheFile = activity.object as CacheFileObject 67 const cacheFile = activity.object as CacheFileObject
104 68
105 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 69 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
@@ -115,29 +79,7 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate)
115 } 79 }
116} 80}
117 81
118async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { 82async function processCreateVideoComment (activity: ActivityCreate, byActor: ActorModel) {
119 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
120
121 const account = byActor.Account
122 if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
123
124 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object })
125
126 return sequelizeTypescript.transaction(async t => {
127 const videoAbuseData = {
128 reporterAccountId: account.id,
129 reason: videoAbuseToCreateData.content,
130 videoId: video.id,
131 state: VideoAbuseState.PENDING
132 }
133
134 await VideoAbuseModel.create(videoAbuseData, { transaction: t })
135
136 logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object)
137 })
138}
139
140async function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreate) {
141 const commentObject = activity.object as VideoCommentObject 83 const commentObject = activity.object as VideoCommentObject
142 const byAccount = byActor.Account 84 const byAccount = byActor.Account
143 85
@@ -145,7 +87,7 @@ async function processCreateVideoComment (byActor: ActorModel, activity: Activit
145 87
146 const { video } = await resolveThread(commentObject.inReplyTo) 88 const { video } = await resolveThread(commentObject.inReplyTo)
147 89
148 const { created } = await addVideoComment(video, commentObject.id) 90 const { comment, created } = await addVideoComment(video, commentObject.id)
149 91
150 if (video.isOwned() && created === true) { 92 if (video.isOwned() && created === true) {
151 // Don't resend the activity to the sender 93 // Don't resend the activity to the sender
@@ -153,4 +95,6 @@ async function processCreateVideoComment (byActor: ActorModel, activity: Activit
153 95
154 await forwardVideoRelatedActivity(activity, undefined, exceptions, video) 96 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
155 } 97 }
98
99 if (created === true) Notifier.Instance.notifyOnNewComment(comment)
156} 100}
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts
new file mode 100644
index 000000000..bfd69e07a
--- /dev/null
+++ b/server/lib/activitypub/process/process-dislike.ts
@@ -0,0 +1,52 @@
1import { ActivityCreate, ActivityDislike } from '../../../../shared'
2import { DislikeObject } from '../../../../shared/models/activitypub/objects'
3import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { sequelizeTypescript } from '../../../initializers'
5import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
6import { ActorModel } from '../../../models/activitypub/actor'
7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getVideoDislikeActivityPubUrl } from '../url'
10
11async function processDislikeActivity (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) {
12 return retryTransactionWrapper(processDislike, activity, byActor)
13}
14
15// ---------------------------------------------------------------------------
16
17export {
18 processDislikeActivity
19}
20
21// ---------------------------------------------------------------------------
22
23async function processDislike (activity: ActivityCreate | ActivityDislike, byActor: ActorModel) {
24 const dislikeObject = activity.type === 'Dislike' ? activity.object : (activity.object as DislikeObject).object
25 const byAccount = byActor.Account
26
27 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
28
29 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject })
30
31 return sequelizeTypescript.transaction(async t => {
32 const rate = {
33 type: 'dislike' as 'dislike',
34 videoId: video.id,
35 accountId: byAccount.id
36 }
37
38 const [ , created ] = await AccountVideoRateModel.findOrCreate({
39 where: rate,
40 defaults: Object.assign({}, rate, { url: getVideoDislikeActivityPubUrl(byActor, video) }),
41 transaction: t
42 })
43 if (created === true) await video.increment('dislikes', { transaction: t })
44
45 if (video.isOwned() && created === true) {
46 // Don't resend the activity to the sender
47 const exceptions = [ byActor ]
48
49 await forwardVideoRelatedActivity(activity, t, exceptions, video)
50 }
51 })
52}
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts
new file mode 100644
index 000000000..79ce6fb41
--- /dev/null
+++ b/server/lib/activitypub/process/process-flag.ts
@@ -0,0 +1,49 @@
1import { ActivityCreate, ActivityFlag, VideoAbuseState } from '../../../../shared'
2import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects'
3import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger'
5import { sequelizeTypescript } from '../../../initializers'
6import { ActorModel } from '../../../models/activitypub/actor'
7import { VideoAbuseModel } from '../../../models/video/video-abuse'
8import { getOrCreateVideoAndAccountAndChannel } from '../videos'
9import { Notifier } from '../../notifier'
10import { getAPId } from '../../../helpers/activitypub'
11
12async function processFlagActivity (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) {
13 return retryTransactionWrapper(processCreateVideoAbuse, activity, byActor)
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 processFlagActivity
20}
21
22// ---------------------------------------------------------------------------
23
24async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag, byActor: ActorModel) {
25 const flag = activity.type === 'Flag' ? activity : (activity.object as VideoAbuseObject)
26
27 logger.debug('Reporting remote abuse for video %s.', getAPId(flag.object))
28
29 const account = byActor.Account
30 if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
31
32 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object })
33
34 return sequelizeTypescript.transaction(async t => {
35 const videoAbuseData = {
36 reporterAccountId: account.id,
37 reason: flag.content,
38 videoId: video.id,
39 state: VideoAbuseState.PENDING
40 }
41
42 const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t })
43 videoAbuseInstance.Video = video
44
45 Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
46
47 logger.info('Remote abuse for video uuid %s created', flag.object)
48 })
49}
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index 24c9085f7..0cd537187 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -5,9 +5,11 @@ import { sequelizeTypescript } from '../../../initializers'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 6import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { sendAccept } from '../send' 7import { sendAccept } from '../send'
8import { Notifier } from '../../notifier'
9import { getAPId } from '../../../helpers/activitypub'
8 10
9async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { 11async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) {
10 const activityObject = activity.object 12 const activityObject = getAPId(activity.object)
11 13
12 return retryTransactionWrapper(processFollow, byActor, activityObject) 14 return retryTransactionWrapper(processFollow, byActor, activityObject)
13} 15}
@@ -21,13 +23,13 @@ export {
21// --------------------------------------------------------------------------- 23// ---------------------------------------------------------------------------
22 24
23async function processFollow (actor: ActorModel, targetActorURL: string) { 25async function processFollow (actor: ActorModel, targetActorURL: string) {
24 await sequelizeTypescript.transaction(async t => { 26 const { actorFollow, created } = await sequelizeTypescript.transaction(async t => {
25 const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) 27 const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t)
26 28
27 if (!targetActor) throw new Error('Unknown actor') 29 if (!targetActor) throw new Error('Unknown actor')
28 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') 30 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.')
29 31
30 const [ actorFollow ] = await ActorFollowModel.findOrCreate({ 32 const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({
31 where: { 33 where: {
32 actorId: actor.id, 34 actorId: actor.id,
33 targetActorId: targetActor.id 35 targetActorId: targetActor.id
@@ -52,8 +54,12 @@ async function processFollow (actor: ActorModel, targetActorURL: string) {
52 actorFollow.ActorFollowing = targetActor 54 actorFollow.ActorFollowing = targetActor
53 55
54 // Target sends to actor he accepted the follow request 56 // Target sends to actor he accepted the follow request
55 return sendAccept(actorFollow) 57 await sendAccept(actorFollow)
58
59 return { actorFollow, created }
56 }) 60 })
57 61
62 if (created) Notifier.Instance.notifyOfNewFollow(actorFollow)
63
58 logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) 64 logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url)
59} 65}
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index f7200db61..2a04167d7 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -5,6 +5,8 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
7import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
8import { getVideoLikeActivityPubUrl } from '../url'
9import { getAPId } from '../../../helpers/activitypub'
8 10
9async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { 11async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) {
10 return retryTransactionWrapper(processLikeVideo, byActor, activity) 12 return retryTransactionWrapper(processLikeVideo, byActor, activity)
@@ -19,7 +21,7 @@ export {
19// --------------------------------------------------------------------------- 21// ---------------------------------------------------------------------------
20 22
21async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { 23async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
22 const videoUrl = activity.object 24 const videoUrl = getAPId(activity.object)
23 25
24 const byAccount = byActor.Account 26 const byAccount = byActor.Account
25 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 27 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
@@ -34,7 +36,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
34 } 36 }
35 const [ , created ] = await AccountVideoRateModel.findOrCreate({ 37 const [ , created ] = await AccountVideoRateModel.findOrCreate({
36 where: rate, 38 where: rate,
37 defaults: rate, 39 defaults: Object.assign({}, rate, { url: getVideoLikeActivityPubUrl(byActor, video) }),
38 transaction: t 40 transaction: t
39 }) 41 })
40 if (created === true) await video.increment('likes', { transaction: t }) 42 if (created === true) await video.increment('likes', { transaction: t })
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index ff019cd8c..ed0177a67 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -26,6 +26,10 @@ async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel)
26 } 26 }
27 } 27 }
28 28
29 if (activityToUndo.type === 'Dislike') {
30 return retryTransactionWrapper(processUndoDislike, byActor, activity)
31 }
32
29 if (activityToUndo.type === 'Follow') { 33 if (activityToUndo.type === 'Follow') {
30 return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) 34 return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo)
31 } 35 }
@@ -55,7 +59,8 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) {
55 return sequelizeTypescript.transaction(async t => { 59 return sequelizeTypescript.transaction(async t => {
56 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 60 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
57 61
58 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) 62 let rate = await AccountVideoRateModel.loadByUrl(likeActivity.id, t)
63 if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
59 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) 64 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
60 65
61 await rate.destroy({ transaction: t }) 66 await rate.destroy({ transaction: t })
@@ -71,14 +76,17 @@ async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) {
71} 76}
72 77
73async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { 78async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) {
74 const dislike = activity.object.object as DislikeObject 79 const dislike = activity.object.type === 'Dislike'
80 ? activity.object
81 : activity.object.object as DislikeObject
75 82
76 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) 83 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
77 84
78 return sequelizeTypescript.transaction(async t => { 85 return sequelizeTypescript.transaction(async t => {
79 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
80 87
81 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) 88 let rate = await AccountVideoRateModel.loadByUrl(dislike.id, t)
89 if (!rate) rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
82 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) 90 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
83 91
84 await rate.destroy({ transaction: t }) 92 await rate.destroy({ transaction: t })
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index bd4013555..c6b42d846 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -51,7 +51,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
51 return undefined 51 return undefined
52 } 52 }
53 53
54 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) 54 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id, allowRefresh: false })
55 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 55 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
56 56
57 const updateOptions = { 57 const updateOptions = {
@@ -59,7 +59,6 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
59 videoObject, 59 videoObject,
60 account: actor.Account, 60 account: actor.Account,
61 channel: channelActor.VideoChannel, 61 channel: channelActor.VideoChannel,
62 updateViews: true,
63 overrideTo: activity.to 62 overrideTo: activity.to
64 } 63 }
65 return updateVideoFromAP(updateOptions) 64 return updateVideoFromAP(updateOptions)
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
new file mode 100644
index 000000000..8f66d3630
--- /dev/null
+++ b/server/lib/activitypub/process/process-view.ts
@@ -0,0 +1,35 @@
1import { ActorModel } from '../../../models/activitypub/actor'
2import { getOrCreateVideoAndAccountAndChannel } from '../videos'
3import { forwardVideoRelatedActivity } from '../send/utils'
4import { Redis } from '../../redis'
5import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub'
6
7async function processViewActivity (activity: ActivityView | ActivityCreate, byActor: ActorModel) {
8 return processCreateView(activity, byActor)
9}
10
11// ---------------------------------------------------------------------------
12
13export {
14 processViewActivity
15}
16
17// ---------------------------------------------------------------------------
18
19async function processCreateView (activity: ActivityView | ActivityCreate, byActor: ActorModel) {
20 const videoObject = activity.type === 'View' ? activity.object : (activity.object as ViewObject).object
21
22 const options = {
23 videoObject: videoObject,
24 fetchType: 'only-video' as 'only-video'
25 }
26 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
27
28 await Redis.Instance.addVideoView(video.id)
29
30 if (video.isOwned()) {
31 // Don't resend the activity to the sender
32 const exceptions = [ byActor ]
33 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
34 }
35}
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index b263f1ea2..9dd241402 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -1,5 +1,5 @@
1import { Activity, ActivityType } from '../../../../shared/models/activitypub' 1import { Activity, ActivityType } from '../../../../shared/models/activitypub'
2import { getActorUrl } from '../../../helpers/activitypub' 2import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { processAcceptActivity } from './process-accept' 5import { processAcceptActivity } from './process-accept'
@@ -12,6 +12,9 @@ import { processRejectActivity } from './process-reject'
12import { processUndoActivity } from './process-undo' 12import { processUndoActivity } from './process-undo'
13import { processUpdateActivity } from './process-update' 13import { processUpdateActivity } from './process-update'
14import { getOrCreateActorAndServerAndModel } from '../actor' 14import { getOrCreateActorAndServerAndModel } from '../actor'
15import { processDislikeActivity } from './process-dislike'
16import { processFlagActivity } from './process-flag'
17import { processViewActivity } from './process-view'
15 18
16const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = { 19const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = {
17 Create: processCreateActivity, 20 Create: processCreateActivity,
@@ -22,27 +25,41 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: Ac
22 Reject: processRejectActivity, 25 Reject: processRejectActivity,
23 Announce: processAnnounceActivity, 26 Announce: processAnnounceActivity,
24 Undo: processUndoActivity, 27 Undo: processUndoActivity,
25 Like: processLikeActivity 28 Like: processLikeActivity,
29 Dislike: processDislikeActivity,
30 Flag: processFlagActivity,
31 View: processViewActivity
26} 32}
27 33
28async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { 34async function processActivities (
35 activities: Activity[],
36 options: {
37 signatureActor?: ActorModel
38 inboxActor?: ActorModel
39 outboxUrl?: string
40 } = {}) {
29 const actorsCache: { [ url: string ]: ActorModel } = {} 41 const actorsCache: { [ url: string ]: ActorModel } = {}
30 42
31 for (const activity of activities) { 43 for (const activity of activities) {
32 if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { 44 if (!options.signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) {
33 logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) 45 logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type)
34 continue 46 continue
35 } 47 }
36 48
37 const actorUrl = getActorUrl(activity.actor) 49 const actorUrl = getAPId(activity.actor)
38 50
39 // When we fetch remote data, we don't have signature 51 // When we fetch remote data, we don't have signature
40 if (signatureActor && actorUrl !== signatureActor.url) { 52 if (options.signatureActor && actorUrl !== options.signatureActor.url) {
41 logger.warn('Signature mismatch between %s and %s.', actorUrl, signatureActor.url) 53 logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, options.signatureActor.url)
42 continue 54 continue
43 } 55 }
44 56
45 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) 57 if (options.outboxUrl && checkUrlsSameHost(options.outboxUrl, actorUrl) !== true) {
58 logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', options.outboxUrl, actorUrl)
59 continue
60 }
61
62 const byActor = options.signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl)
46 actorsCache[actorUrl] = byActor 63 actorsCache[actorUrl] = byActor
47 64
48 const activityProcessor = processActivity[activity.type] 65 const activityProcessor = processActivity[activity.type]
@@ -52,7 +69,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
52 } 69 }
53 70
54 try { 71 try {
55 await activityProcessor(activity, byActor, inboxActor) 72 await activityProcessor(activity, byActor, options.inboxActor)
56 } catch (err) { 73 } catch (err) {
57 logger.warn('Cannot process activity %s.', activity.type, { err }) 74 logger.warn('Cannot process activity %s.', activity.type, { err })
58 } 75 }
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 285edba3b..ef20e404c 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -3,9 +3,7 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti
3import { VideoPrivacy } from '../../../../shared/models/videos' 3import { VideoPrivacy } from '../../../../shared/models/videos'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
6import { VideoAbuseModel } from '../../../models/video/video-abuse'
7import { VideoCommentModel } from '../../../models/video/video-comment' 6import { VideoCommentModel } from '../../../models/video/video-comment'
8import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url'
9import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 7import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
10import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' 8import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
11import { logger } from '../../../helpers/logger' 9import { logger } from '../../../helpers/logger'
@@ -25,31 +23,14 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) {
25 return broadcastToFollowers(createActivity, byActor, [ byActor ], t) 23 return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
26} 24}
27 25
28async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { 26async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
29 if (!video.VideoChannel.Account.Actor.serverId) return // Local
30
31 const url = getVideoAbuseActivityPubUrl(videoAbuse)
32
33 logger.info('Creating job to send video abuse %s.', url)
34
35 // Custom audience, we only send the abuse to the origin instance
36 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
37 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
38
39 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
40}
41
42async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 27 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
44 28
45 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
46 const redundancyObject = fileRedundancy.toActivityPubObject()
47
48 return sendVideoRelatedCreateActivity({ 29 return sendVideoRelatedCreateActivity({
49 byActor, 30 byActor,
50 video, 31 video,
51 url: fileRedundancy.url, 32 url: fileRedundancy.url,
52 object: redundancyObject 33 object: fileRedundancy.toActivityPubObject()
53 }) 34 })
54} 35}
55 36
@@ -91,37 +72,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
91 return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) 72 return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
92} 73}
93 74
94async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) {
95 logger.info('Creating job to send view of %s.', video.url)
96
97 const url = getVideoViewActivityPubUrl(byActor, video)
98 const viewActivity = buildViewActivity(byActor, video)
99
100 return sendVideoRelatedCreateActivity({
101 // Use the server actor to send the view
102 byActor,
103 video,
104 url,
105 object: viewActivity,
106 transaction: t
107 })
108}
109
110async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
111 logger.info('Creating job to dislike %s.', video.url)
112
113 const url = getVideoDislikeActivityPubUrl(byActor, video)
114 const dislikeActivity = buildDislikeActivity(byActor, video)
115
116 return sendVideoRelatedCreateActivity({
117 byActor,
118 video,
119 url,
120 object: dislikeActivity,
121 transaction: t
122 })
123}
124
125function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { 75function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
126 if (!audience) audience = getAudience(byActor) 76 if (!audience) audience = getAudience(byActor)
127 77
@@ -136,31 +86,11 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud
136 ) 86 )
137} 87}
138 88
139function buildDislikeActivity (byActor: ActorModel, video: VideoModel) {
140 return {
141 type: 'Dislike',
142 actor: byActor.url,
143 object: video.url
144 }
145}
146
147function buildViewActivity (byActor: ActorModel, video: VideoModel) {
148 return {
149 type: 'View',
150 actor: byActor.url,
151 object: video.url
152 }
153}
154
155// --------------------------------------------------------------------------- 89// ---------------------------------------------------------------------------
156 90
157export { 91export {
158 sendCreateVideo, 92 sendCreateVideo,
159 sendVideoAbuse,
160 buildCreateActivity, 93 buildCreateActivity,
161 sendCreateView,
162 sendCreateDislike,
163 buildDislikeActivity,
164 sendCreateVideoComment, 94 sendCreateVideoComment,
165 sendCreateCacheFile 95 sendCreateCacheFile
166} 96}
diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts
new file mode 100644
index 000000000..a88436f2c
--- /dev/null
+++ b/server/lib/activitypub/send/send-dislike.ts
@@ -0,0 +1,41 @@
1import { Transaction } from 'sequelize'
2import { ActorModel } from '../../../models/activitypub/actor'
3import { VideoModel } from '../../../models/video/video'
4import { getVideoDislikeActivityPubUrl } from '../url'
5import { logger } from '../../../helpers/logger'
6import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub'
7import { sendVideoRelatedActivity } from './utils'
8import { audiencify, getAudience } from '../audience'
9
10async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
11 logger.info('Creating job to dislike %s.', video.url)
12
13 const activityBuilder = (audience: ActivityAudience) => {
14 const url = getVideoDislikeActivityPubUrl(byActor, video)
15
16 return buildDislikeActivity(url, byActor, video, audience)
17 }
18
19 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
20}
21
22function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike {
23 if (!audience) audience = getAudience(byActor)
24
25 return audiencify(
26 {
27 id: url,
28 type: 'Dislike' as 'Dislike',
29 actor: byActor.url,
30 object: video.url
31 },
32 audience
33 )
34}
35
36// ---------------------------------------------------------------------------
37
38export {
39 sendDislike,
40 buildDislikeActivity
41}
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts
new file mode 100644
index 000000000..96a7311b9
--- /dev/null
+++ b/server/lib/activitypub/send/send-flag.ts
@@ -0,0 +1,39 @@
1import { ActorModel } from '../../../models/activitypub/actor'
2import { VideoModel } from '../../../models/video/video'
3import { VideoAbuseModel } from '../../../models/video/video-abuse'
4import { getVideoAbuseActivityPubUrl } from '../url'
5import { unicastTo } from './utils'
6import { logger } from '../../../helpers/logger'
7import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub'
8import { audiencify, getAudience } from '../audience'
9
10async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
11 if (!video.VideoChannel.Account.Actor.serverId) return // Local user
12
13 const url = getVideoAbuseActivityPubUrl(videoAbuse)
14
15 logger.info('Creating job to send video abuse %s.', url)
16
17 // Custom audience, we only send the abuse to the origin instance
18 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
19 const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
20
21 return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
22}
23
24function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag {
25 if (!audience) audience = getAudience(byActor)
26
27 const activity = Object.assign(
28 { id: url, actor: byActor.url },
29 videoAbuse.toActivityPubObject()
30 )
31
32 return audiencify(activity, audience)
33}
34
35// ---------------------------------------------------------------------------
36
37export {
38 sendVideoAbuse
39}
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts
index 89307acc6..35227887a 100644
--- a/server/lib/activitypub/send/send-like.ts
+++ b/server/lib/activitypub/send/send-like.ts
@@ -24,8 +24,8 @@ function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel,
24 24
25 return audiencify( 25 return audiencify(
26 { 26 {
27 type: 'Like' as 'Like',
28 id: url, 27 id: url,
28 type: 'Like' as 'Like',
29 actor: byActor.url, 29 actor: byActor.url,
30 object: video.url 30 object: video.url
31 }, 31 },
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index 5236d2cb3..ecbf605d6 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize'
2import { 2import {
3 ActivityAnnounce, 3 ActivityAnnounce,
4 ActivityAudience, 4 ActivityAudience,
5 ActivityCreate, 5 ActivityCreate, ActivityDislike,
6 ActivityFollow, 6 ActivityFollow,
7 ActivityLike, 7 ActivityLike,
8 ActivityUndo 8 ActivityUndo
@@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video'
13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' 13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
14import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 14import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
15import { audiencify, getAudience } from '../audience' 15import { audiencify, getAudience } from '../audience'
16import { buildCreateActivity, buildDislikeActivity } from './send-create' 16import { buildCreateActivity } from './send-create'
17import { buildFollowActivity } from './send-follow' 17import { buildFollowActivity } from './send-follow'
18import { buildLikeActivity } from './send-like' 18import { buildLikeActivity } from './send-like'
19import { VideoShareModel } from '../../../models/video/video-share' 19import { VideoShareModel } from '../../../models/video/video-share'
20import { buildAnnounceWithVideoAudience } from './send-announce' 20import { buildAnnounceWithVideoAudience } from './send-announce'
21import { logger } from '../../../helpers/logger' 21import { logger } from '../../../helpers/logger'
22import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 22import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
23import { buildDislikeActivity } from './send-dislike'
23 24
24async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { 25async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
25 const me = actorFollow.ActorFollower 26 const me = actorFollow.ActorFollower
@@ -64,16 +65,16 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
64 logger.info('Creating job to undo a dislike of video %s.', video.url) 65 logger.info('Creating job to undo a dislike of video %s.', video.url)
65 66
66 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) 67 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
67 const dislikeActivity = buildDislikeActivity(byActor, video) 68 const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video)
68 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
69 69
70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) 70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t })
71} 71}
72 72
73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
74 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
75 75
76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const videoId = redundancyModel.getVideo().id
77 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 78 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
78 79
79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) 80 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
@@ -94,7 +95,7 @@ export {
94function undoActivityData ( 95function undoActivityData (
95 url: string, 96 url: string,
96 byActor: ActorModel, 97 byActor: ActorModel,
97 object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, 98 object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce,
98 audience?: ActivityAudience 99 audience?: ActivityAudience
99): ActivityUndo { 100): ActivityUndo {
100 if (!audience) audience = getAudience(byActor) 101 if (!audience) audience = getAudience(byActor)
@@ -114,7 +115,7 @@ async function sendUndoVideoRelatedActivity (options: {
114 byActor: ActorModel, 115 byActor: ActorModel,
115 video: VideoModel, 116 video: VideoModel,
116 url: string, 117 url: string,
117 activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, 118 activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce,
118 transaction: Transaction 119 transaction: Transaction
119}) { 120}) {
120 const activityBuilder = (audience: ActivityAudience) => { 121 const activityBuilder = (audience: ActivityAudience) => {
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index a68f03edf..839f66470 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 63
64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)
65 65
66 const activityBuilder = (audience: ActivityAudience) => { 66 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject() 67 const redundancyObject = redundancyModel.toActivityPubObject()
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts
new file mode 100644
index 000000000..8ad126be0
--- /dev/null
+++ b/server/lib/activitypub/send/send-view.ts
@@ -0,0 +1,40 @@
1import { Transaction } from 'sequelize'
2import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub'
3import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video'
5import { getVideoLikeActivityPubUrl } from '../url'
6import { sendVideoRelatedActivity } from './utils'
7import { audiencify, getAudience } from '../audience'
8import { logger } from '../../../helpers/logger'
9
10async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) {
11 logger.info('Creating job to send view of %s.', video.url)
12
13 const activityBuilder = (audience: ActivityAudience) => {
14 const url = getVideoLikeActivityPubUrl(byActor, video)
15
16 return buildViewActivity(url, byActor, video, audience)
17 }
18
19 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
20}
21
22function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView {
23 if (!audience) audience = getAudience(byActor)
24
25 return audiencify(
26 {
27 id: url,
28 type: 'View' as 'View',
29 actor: byActor.url,
30 object: video.url
31 },
32 audience
33 )
34}
35
36// ---------------------------------------------------------------------------
37
38export {
39 sendView
40}
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 3ff60a97c..1767df0ae 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -4,13 +4,14 @@ import { getServerActor } from '../../helpers/utils'
4import { VideoModel } from '../../models/video/video' 4import { VideoModel } from '../../models/video/video'
5import { VideoShareModel } from '../../models/video/video-share' 5import { VideoShareModel } from '../../models/video/video-share'
6import { sendUndoAnnounce, sendVideoAnnounce } from './send' 6import { sendUndoAnnounce, sendVideoAnnounce } from './send'
7import { getAnnounceActivityPubUrl } from './url' 7import { getVideoAnnounceActivityPubUrl } from './url'
8import { VideoChannelModel } from '../../models/video/video-channel' 8import { VideoChannelModel } from '../../models/video/video-channel'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { doRequest } from '../../helpers/requests' 10import { doRequest } from '../../helpers/requests'
11import { getOrCreateActorAndServerAndModel } from './actor' 11import { getOrCreateActorAndServerAndModel } from './actor'
12import { logger } from '../../helpers/logger' 12import { logger } from '../../helpers/logger'
13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
14import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
14 15
15async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { 16async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
16 if (video.privacy === VideoPrivacy.PRIVATE) return undefined 17 if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -38,9 +39,13 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) {
38 json: true, 39 json: true,
39 activityPub: true 40 activityPub: true
40 }) 41 })
41 if (!body || !body.actor) throw new Error('Body of body actor is invalid') 42 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
43
44 const actorUrl = getAPId(body.actor)
45 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
46 throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
47 }
42 48
43 const actorUrl = body.actor
44 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 49 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
45 50
46 const entry = { 51 const entry = {
@@ -72,8 +77,8 @@ export {
72async function shareByServer (video: VideoModel, t: Transaction) { 77async function shareByServer (video: VideoModel, t: Transaction) {
73 const serverActor = await getServerActor() 78 const serverActor = await getServerActor()
74 79
75 const serverShareUrl = getAnnounceActivityPubUrl(video.url, serverActor) 80 const serverShareUrl = getVideoAnnounceActivityPubUrl(serverActor, video)
76 return VideoShareModel.findOrCreate({ 81 const [ serverShare ] = await VideoShareModel.findOrCreate({
77 defaults: { 82 defaults: {
78 actorId: serverActor.id, 83 actorId: serverActor.id,
79 videoId: video.id, 84 videoId: video.id,
@@ -83,16 +88,14 @@ async function shareByServer (video: VideoModel, t: Transaction) {
83 url: serverShareUrl 88 url: serverShareUrl
84 }, 89 },
85 transaction: t 90 transaction: t
86 }).then(([ serverShare, created ]) => {
87 if (created) return sendVideoAnnounce(serverActor, serverShare, video, t)
88
89 return undefined
90 }) 91 })
92
93 return sendVideoAnnounce(serverActor, serverShare, video, t)
91} 94}
92 95
93async function shareByVideoChannel (video: VideoModel, t: Transaction) { 96async function shareByVideoChannel (video: VideoModel, t: Transaction) {
94 const videoChannelShareUrl = getAnnounceActivityPubUrl(video.url, video.VideoChannel.Actor) 97 const videoChannelShareUrl = getVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video)
95 return VideoShareModel.findOrCreate({ 98 const [ videoChannelShare ] = await VideoShareModel.findOrCreate({
96 defaults: { 99 defaults: {
97 actorId: video.VideoChannel.actorId, 100 actorId: video.VideoChannel.actorId,
98 videoId: video.id, 101 videoId: video.id,
@@ -102,11 +105,9 @@ async function shareByVideoChannel (video: VideoModel, t: Transaction) {
102 url: videoChannelShareUrl 105 url: videoChannelShareUrl
103 }, 106 },
104 transaction: t 107 transaction: t
105 }).then(([ videoChannelShare, created ]) => {
106 if (created) return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t)
107
108 return undefined
109 }) 108 })
109
110 return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t)
110} 111}
111 112
112async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) { 113async function undoShareByVideoChannel (video: VideoModel, oldVideoChannel: VideoChannelModel, t: Transaction) {
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index e792be698..4229fe094 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseModel } from '../../models/video/video-abuse'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoFileModel } from '../../models/video/video-file' 7import { VideoFileModel } from '../../models/video/video-file'
8import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
9import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
8 10
9function getVideoActivityPubUrl (video: VideoModel) { 11function getVideoActivityPubUrl (video: VideoModel) {
10 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 12 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
16 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` 18 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
17} 19}
18 20
21function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
22 return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}`
23}
24
19function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { 25function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
20 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id 26 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
21} 27}
@@ -33,14 +39,14 @@ function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) {
33} 39}
34 40
35function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { 41function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) {
36 return video.url + '/views/' + byActor.uuid + '/' + new Date().toISOString() 42 return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString()
37} 43}
38 44
39function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { 45function getVideoLikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) {
40 return byActor.url + '/likes/' + video.id 46 return byActor.url + '/likes/' + video.id
41} 47}
42 48
43function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel) { 49function getVideoDislikeActivityPubUrl (byActor: ActorModel, video: VideoModel | { id: number }) {
44 return byActor.url + '/dislikes/' + video.id 50 return byActor.url + '/dislikes/' + video.id
45} 51}
46 52
@@ -74,8 +80,8 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) {
74 return follower.url + '/accepts/follows/' + me.id 80 return follower.url + '/accepts/follows/' + me.id
75} 81}
76 82
77function getAnnounceActivityPubUrl (originalUrl: string, byActor: ActorModel) { 83function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) {
78 return originalUrl + '/announces/' + byActor.id 84 return video.url + '/announces/' + byActor.id
79} 85}
80 86
81function getDeleteActivityPubUrl (originalUrl: string) { 87function getDeleteActivityPubUrl (originalUrl: string) {
@@ -92,12 +98,13 @@ function getUndoActivityPubUrl (originalUrl: string) {
92 98
93export { 99export {
94 getVideoActivityPubUrl, 100 getVideoActivityPubUrl,
101 getVideoCacheStreamingPlaylistActivityPubUrl,
95 getVideoChannelActivityPubUrl, 102 getVideoChannelActivityPubUrl,
96 getAccountActivityPubUrl, 103 getAccountActivityPubUrl,
97 getVideoAbuseActivityPubUrl, 104 getVideoAbuseActivityPubUrl,
98 getActorFollowActivityPubUrl, 105 getActorFollowActivityPubUrl,
99 getActorFollowAcceptActivityPubUrl, 106 getActorFollowAcceptActivityPubUrl,
100 getAnnounceActivityPubUrl, 107 getVideoAnnounceActivityPubUrl,
101 getUpdateActivityPubUrl, 108 getUpdateActivityPubUrl,
102 getUndoActivityPubUrl, 109 getUndoActivityPubUrl,
103 getVideoViewActivityPubUrl, 110 getVideoViewActivityPubUrl,
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index c8c17f4c4..e87301fe7 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -9,6 +9,7 @@ import { VideoCommentModel } from '../../models/video/video-comment'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateActorAndServerAndModel } from './actor'
10import { getOrCreateVideoAndAccountAndChannel } from './videos' 10import { getOrCreateVideoAndAccountAndChannel } from './videos'
11import * as Bluebird from 'bluebird' 11import * as Bluebird from 'bluebird'
12import { checkUrlsSameHost } from '../../helpers/activitypub'
12 13
13async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { 14async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
14 let originCommentId: number = null 15 let originCommentId: number = null
@@ -61,7 +62,15 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
61 const actorUrl = body.attributedTo 62 const actorUrl = body.attributedTo
62 if (!actorUrl) return { created: false } 63 if (!actorUrl) return { created: false }
63 64
64 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 65 if (checkUrlsSameHost(commentUrl, actorUrl) !== true) {
66 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${commentUrl}`)
67 }
68
69 if (checkUrlsSameHost(body.id, commentUrl) !== true) {
70 throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`)
71 }
72
73 const actor = await getOrCreateActorAndServerAndModel(actorUrl, 'all')
65 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) 74 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
66 if (!entry) return { created: false } 75 if (!entry) return { created: false }
67 76
@@ -71,6 +80,8 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
71 }, 80 },
72 defaults: entry 81 defaults: entry
73 }) 82 })
83 comment.Account = actor.Account
84 comment.Video = videoInstance
74 85
75 return { comment, created } 86 return { comment, created }
76} 87}
@@ -134,6 +145,14 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
134 const actorUrl = body.attributedTo 145 const actorUrl = body.attributedTo
135 if (!actorUrl) throw new Error('Miss attributed to in comment') 146 if (!actorUrl) throw new Error('Miss attributed to in comment')
136 147
148 if (checkUrlsSameHost(url, actorUrl) !== true) {
149 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)
150 }
151
152 if (checkUrlsSameHost(body.id, url) !== true) {
153 throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)
154 }
155
137 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 156 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
138 const comment = new VideoCommentModel({ 157 const comment = new VideoCommentModel({
139 url: body.id, 158 url: body.id,
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 1619251c3..7aac79118 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -1,20 +1,43 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { AccountModel } from '../../models/account/account' 2import { AccountModel } from '../../models/account/account'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' 4import { sendLike, sendUndoDislike, sendUndoLike } from './send'
5import { VideoRateType } from '../../../shared/models/videos' 5import { VideoRateType } from '../../../shared/models/videos'
6import * as Bluebird from 'bluebird' 6import * as Bluebird from 'bluebird'
7import { getOrCreateActorAndServerAndModel } from './actor' 7import { getOrCreateActorAndServerAndModel } from './actor'
8import { AccountVideoRateModel } from '../../models/account/account-video-rate' 8import { AccountVideoRateModel } from '../../models/account/account-video-rate'
9import { logger } from '../../helpers/logger' 9import { logger } from '../../helpers/logger'
10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
11import { doRequest } from '../../helpers/requests'
12import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
13import { ActorModel } from '../../models/activitypub/actor'
14import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url'
15import { sendDislike } from './send/send-dislike'
11 16
12async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { 17async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) {
13 let rateCounts = 0 18 let rateCounts = 0
14 19
15 await Bluebird.map(actorUrls, async actorUrl => { 20 await Bluebird.map(ratesUrl, async rateUrl => {
16 try { 21 try {
22 // Fetch url
23 const { body } = await doRequest({
24 uri: rateUrl,
25 json: true,
26 activityPub: true
27 })
28 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
29
30 const actorUrl = getAPId(body.actor)
31 if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
32 throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
33 }
34
35 if (checkUrlsSameHost(body.id, rateUrl) !== true) {
36 throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
37 }
38
17 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 39 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
40
18 const [ , created ] = await AccountVideoRateModel 41 const [ , created ] = await AccountVideoRateModel
19 .findOrCreate({ 42 .findOrCreate({
20 where: { 43 where: {
@@ -24,13 +47,14 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
24 defaults: { 47 defaults: {
25 videoId: video.id, 48 videoId: video.id,
26 accountId: actor.Account.id, 49 accountId: actor.Account.id,
27 type: rate 50 type: rate,
51 url: body.id
28 } 52 }
29 }) 53 })
30 54
31 if (created) rateCounts += 1 55 if (created) rateCounts += 1
32 } catch (err) { 56 } catch (err) {
33 logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) 57 logger.warn('Cannot add rate %s.', rateUrl, { err })
34 } 58 }
35 }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) 59 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
36 60
@@ -59,10 +83,15 @@ async function sendVideoRateChange (account: AccountModel,
59 // Like 83 // Like
60 if (likes > 0) await sendLike(actor, video, t) 84 if (likes > 0) await sendLike(actor, video, t)
61 // Dislike 85 // Dislike
62 if (dislikes > 0) await sendCreateDislike(actor, video, t) 86 if (dislikes > 0) await sendDislike(actor, video, t)
87}
88
89function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) {
90 return rateType === 'like' ? getVideoLikeActivityPubUrl(actor, video) : getVideoDislikeActivityPubUrl(actor, video)
63} 91}
64 92
65export { 93export {
94 getRateUrl,
66 createRates, 95 createRates,
67 sendVideoRateChange 96 sendVideoRateChange
68} 97}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 8521572a1..710929aac 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -1,17 +1,23 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import * as sequelize from 'sequelize' 2import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import { join } from 'path'
5import * as request from 'request' 4import * as request from 'request'
6import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' 5import {
6 ActivityIconObject,
7 ActivityPlaylistSegmentHashesObject,
8 ActivityPlaylistUrlObject,
9 ActivityUrlObject,
10 ActivityVideoUrlObject,
11 VideoState
12} from '../../../shared/index'
7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 13import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8import { VideoPrivacy } from '../../../shared/models/videos' 14import { VideoPrivacy } from '../../../shared/models/videos'
9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 15import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
10import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 16import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
11import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 17import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
12import { logger } from '../../helpers/logger' 18import { logger } from '../../helpers/logger'
13import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 19import { doRequest, downloadImage } from '../../helpers/requests'
14import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' 20import { ACTIVITY_PUB, CONFIG, MIMETYPES, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers'
15import { ActorModel } from '../../models/activitypub/actor' 21import { ActorModel } from '../../models/activitypub/actor'
16import { TagModel } from '../../models/video/tag' 22import { TagModel } from '../../models/video/tag'
17import { VideoModel } from '../../models/video/video' 23import { VideoModel } from '../../models/video/video'
@@ -29,6 +35,11 @@ import { createRates } from './video-rates'
29import { addVideoShares, shareVideoByServerAndChannel } from './share' 35import { addVideoShares, shareVideoByServerAndChannel } from './share'
30import { AccountModel } from '../../models/account/account' 36import { AccountModel } from '../../models/account/account'
31import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 37import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
38import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
39import { Notifier } from '../notifier'
40import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
42import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
32 43
33async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 44async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
34 // If the video is not private and published, we federate it 45 // If the video is not private and published, we federate it
@@ -63,7 +74,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.
63 74
64 const { response, body } = await doRequest(options) 75 const { response, body } = await doRequest(options)
65 76
66 if (sanitizeAndCheckVideoTorrentObject(body) === false) { 77 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
67 logger.debug('Remote video JSON is not valid.', { body }) 78 logger.debug('Remote video JSON is not valid.', { body })
68 return { response, videoObject: undefined } 79 return { response, videoObject: undefined }
69 } 80 }
@@ -94,19 +105,18 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu
94 105
95function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { 106function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
96 const thumbnailName = video.getThumbnailName() 107 const thumbnailName = video.getThumbnailName()
97 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
98 108
99 const options = { 109 return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
100 method: 'GET',
101 uri: icon.url
102 }
103 return doRequestAndSaveToFile(options, thumbnailPath)
104} 110}
105 111
106function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 112function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
107 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 113 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
108 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 114 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
109 115
116 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
117 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
118 }
119
110 return getOrCreateActorAndServerAndModel(channel.id, 'all') 120 return getOrCreateActorAndServerAndModel(channel.id, 'all')
111} 121}
112 122
@@ -116,7 +126,7 @@ type SyncParam = {
116 shares: boolean 126 shares: boolean
117 comments: boolean 127 comments: boolean
118 thumbnail: boolean 128 thumbnail: boolean
119 refreshVideo: boolean 129 refreshVideo?: boolean
120} 130}
121async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { 131async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
122 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) 132 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
@@ -155,31 +165,34 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
155} 165}
156 166
157async function getOrCreateVideoAndAccountAndChannel (options: { 167async function getOrCreateVideoAndAccountAndChannel (options: {
158 videoObject: VideoTorrentObject | string, 168 videoObject: { id: string } | string,
159 syncParam?: SyncParam, 169 syncParam?: SyncParam,
160 fetchType?: VideoFetchByUrlType, 170 fetchType?: VideoFetchByUrlType,
161 refreshViews?: boolean 171 allowRefresh?: boolean // true by default
162}) { 172}) {
163 // Default params 173 // Default params
164 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } 174 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
165 const fetchType = options.fetchType || 'all' 175 const fetchType = options.fetchType || 'all'
166 const refreshViews = options.refreshViews || false 176 const allowRefresh = options.allowRefresh !== false
167 177
168 // Get video url 178 // Get video url
169 const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id 179 const videoUrl = getAPId(options.videoObject)
170 180
171 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) 181 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
172 if (videoFromDatabase) { 182 if (videoFromDatabase) {
173 const refreshOptions = { 183
174 video: videoFromDatabase, 184 if (allowRefresh === true) {
175 fetchedType: fetchType, 185 const refreshOptions = {
176 syncParam, 186 video: videoFromDatabase,
177 refreshViews 187 fetchedType: fetchType,
188 syncParam
189 }
190
191 if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
192 else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
178 } 193 }
179 const p = refreshVideoIfNeeded(refreshOptions)
180 if (syncParam.refreshVideo === true) videoFromDatabase = await p
181 194
182 return { video: videoFromDatabase } 195 return { video: videoFromDatabase, created: false }
183 } 196 }
184 197
185 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) 198 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
@@ -190,7 +203,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
190 203
191 await syncVideoExternalAttributes(video, fetchedVideo, syncParam) 204 await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
192 205
193 return { video } 206 return { video, created: true }
194} 207}
195 208
196async function updateVideoFromAP (options: { 209async function updateVideoFromAP (options: {
@@ -198,17 +211,17 @@ async function updateVideoFromAP (options: {
198 videoObject: VideoTorrentObject, 211 videoObject: VideoTorrentObject,
199 account: AccountModel, 212 account: AccountModel,
200 channel: VideoChannelModel, 213 channel: VideoChannelModel,
201 updateViews: boolean,
202 overrideTo?: string[] 214 overrideTo?: string[]
203}) { 215}) {
204 logger.debug('Updating remote video "%s".', options.videoObject.uuid) 216 logger.debug('Updating remote video "%s".', options.videoObject.uuid)
217
205 let videoFieldsSave: any 218 let videoFieldsSave: any
219 const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE
220 const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED
206 221
207 try { 222 try {
208 await sequelizeTypescript.transaction(async t => { 223 await sequelizeTypescript.transaction(async t => {
209 const sequelizeOptions = { 224 const sequelizeOptions = { transaction: t }
210 transaction: t
211 }
212 225
213 videoFieldsSave = options.video.toJSON() 226 videoFieldsSave = options.video.toJSON()
214 227
@@ -238,14 +251,10 @@ async function updateVideoFromAP (options: {
238 options.video.set('publishedAt', videoData.publishedAt) 251 options.video.set('publishedAt', videoData.publishedAt)
239 options.video.set('privacy', videoData.privacy) 252 options.video.set('privacy', videoData.privacy)
240 options.video.set('channelId', videoData.channelId) 253 options.video.set('channelId', videoData.channelId)
254 options.video.set('views', videoData.views)
241 255
242 if (options.updateViews === true) options.video.set('views', videoData.views)
243 await options.video.save(sequelizeOptions) 256 await options.video.save(sequelizeOptions)
244 257
245 // Don't block on request
246 generateThumbnailFromUrl(options.video, options.videoObject.icon)
247 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
248
249 { 258 {
250 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) 259 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
251 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) 260 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
@@ -266,6 +275,25 @@ async function updateVideoFromAP (options: {
266 } 275 }
267 276
268 { 277 {
278 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
279 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
280
281 // Remove video files that do not exist anymore
282 const destroyTasks = options.video.VideoStreamingPlaylists
283 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
284 .map(f => f.destroy(sequelizeOptions))
285 await Promise.all(destroyTasks)
286
287 // Update or add other one
288 const upsertTasks = streamingPlaylistAttributes.map(a => {
289 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
290 .then(([ streamingPlaylist ]) => streamingPlaylist)
291 })
292
293 options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
294 }
295
296 {
269 // Update Tags 297 // Update Tags
270 const tags = options.videoObject.tag.map(tag => tag.name) 298 const tags = options.videoObject.tag.map(tag => tag.name)
271 const tagInstances = await TagModel.findOrCreateTags(tags, t) 299 const tagInstances = await TagModel.findOrCreateTags(tags, t)
@@ -283,6 +311,11 @@ async function updateVideoFromAP (options: {
283 } 311 }
284 }) 312 })
285 313
314 // Notify our users?
315 if (wasPrivateVideo || wasUnlistedVideo) {
316 Notifier.Instance.notifyOnNewVideo(options.video)
317 }
318
286 logger.info('Remote video with uuid %s updated', options.videoObject.uuid) 319 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
287 } catch (err) { 320 } catch (err) {
288 if (options.video !== undefined && videoFieldsSave !== undefined) { 321 if (options.video !== undefined && videoFieldsSave !== undefined) {
@@ -293,10 +326,66 @@ async function updateVideoFromAP (options: {
293 logger.debug('Cannot update the remote video.', { err }) 326 logger.debug('Cannot update the remote video.', { err })
294 throw err 327 throw err
295 } 328 }
329
330 try {
331 await generateThumbnailFromUrl(options.video, options.videoObject.icon)
332 } catch (err) {
333 logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
334 }
335}
336
337async function refreshVideoIfNeeded (options: {
338 video: VideoModel,
339 fetchedType: VideoFetchByUrlType,
340 syncParam: SyncParam
341}): Promise<VideoModel> {
342 if (!options.video.isOutdated()) return options.video
343
344 // We need more attributes if the argument video was fetched with not enough joints
345 const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
346
347 try {
348 const { response, videoObject } = await fetchRemoteVideo(video.url)
349 if (response.statusCode === 404) {
350 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
351
352 // Video does not exist anymore
353 await video.destroy()
354 return undefined
355 }
356
357 if (videoObject === undefined) {
358 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
359
360 await video.setAsRefreshed()
361 return video
362 }
363
364 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
365 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
366
367 const updateOptions = {
368 video,
369 videoObject,
370 account,
371 channel: channelActor.VideoChannel
372 }
373 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
374 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
375
376 return video
377 } catch (err) {
378 logger.warn('Cannot refresh video %s.', options.video.url, { err })
379
380 // Don't refresh in loop
381 await video.setAsRefreshed()
382 return video
383 }
296} 384}
297 385
298export { 386export {
299 updateVideoFromAP, 387 updateVideoFromAP,
388 refreshVideoIfNeeded,
300 federateVideoIfNeeded, 389 federateVideoIfNeeded,
301 fetchRemoteVideo, 390 fetchRemoteVideo,
302 getOrCreateVideoAndAccountAndChannel, 391 getOrCreateVideoAndAccountAndChannel,
@@ -308,10 +397,23 @@ export {
308 397
309// --------------------------------------------------------------------------- 398// ---------------------------------------------------------------------------
310 399
311function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { 400function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
312 const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) 401 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
313 402
314 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') 403 const urlMediaType = url.mediaType || url.mimeType
404 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
405}
406
407function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
408 const urlMediaType = url.mediaType || url.mimeType
409
410 return urlMediaType === 'application/x-mpegURL'
411}
412
413function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
414 const urlMediaType = tag.mediaType || tag.mimeType
415
416 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
315} 417}
316 418
317async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 419async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
@@ -334,8 +436,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
334 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 436 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
335 await Promise.all(videoFilePromises) 437 await Promise.all(videoFilePromises)
336 438
439 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
440 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
441 await Promise.all(playlistPromises)
442
337 // Process tags 443 // Process tags
338 const tags = videoObject.tag.map(t => t.name) 444 const tags = videoObject.tag
445 .filter(t => t.type === 'Hashtag')
446 .map(t => t.name)
339 const tagInstances = await TagModel.findOrCreateTags(tags, t) 447 const tagInstances = await TagModel.findOrCreateTags(tags, t)
340 await videoCreated.$set('Tags', tagInstances, sequelizeOptions) 448 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
341 449
@@ -359,52 +467,6 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
359 return videoCreated 467 return videoCreated
360} 468}
361 469
362async function refreshVideoIfNeeded (options: {
363 video: VideoModel,
364 fetchedType: VideoFetchByUrlType,
365 syncParam: SyncParam,
366 refreshViews: boolean
367}): Promise<VideoModel> {
368 if (!options.video.isOutdated()) return options.video
369
370 // We need more attributes if the argument video was fetched with not enough joints
371 const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
372
373 try {
374 const { response, videoObject } = await fetchRemoteVideo(video.url)
375 if (response.statusCode === 404) {
376 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
377
378 // Video does not exist anymore
379 await video.destroy()
380 return undefined
381 }
382
383 if (videoObject === undefined) {
384 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
385 return video
386 }
387
388 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
389 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
390
391 const updateOptions = {
392 video,
393 videoObject,
394 account,
395 channel: channelActor.VideoChannel,
396 updateViews: options.refreshViews
397 }
398 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
399 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
400
401 return video
402 } catch (err) {
403 logger.warn('Cannot refresh video %s.', options.video.url, { err })
404 return video
405 }
406}
407
408async function videoActivityObjectToDBAttributes ( 470async function videoActivityObjectToDBAttributes (
409 videoChannel: VideoChannelModel, 471 videoChannel: VideoChannelModel,
410 videoObject: VideoTorrentObject, 472 videoObject: VideoTorrentObject,
@@ -460,17 +522,18 @@ async function videoActivityObjectToDBAttributes (
460} 522}
461 523
462function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { 524function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
463 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] 525 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
464 526
465 if (fileUrls.length === 0) { 527 if (fileUrls.length === 0) {
466 throw new Error('Cannot find video files for ' + video.url) 528 throw new Error('Cannot find video files for ' + video.url)
467 } 529 }
468 530
469 const attributes: VideoFileModel[] = [] 531 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
470 for (const fileUrl of fileUrls) { 532 for (const fileUrl of fileUrls) {
471 // Fetch associated magnet uri 533 // Fetch associated magnet uri
472 const magnet = videoObject.url.find(u => { 534 const magnet = videoObject.url.find(u => {
473 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height 535 const mediaType = u.mediaType || u.mimeType
536 return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
474 }) 537 })
475 538
476 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) 539 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
@@ -480,14 +543,53 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
480 throw new Error('Cannot parse magnet URI ' + magnet.href) 543 throw new Error('Cannot parse magnet URI ' + magnet.href)
481 } 544 }
482 545
546 const mediaType = fileUrl.mediaType || fileUrl.mimeType
483 const attribute = { 547 const attribute = {
484 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], 548 extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
485 infoHash: parsed.infoHash, 549 infoHash: parsed.infoHash,
486 resolution: fileUrl.height, 550 resolution: fileUrl.height,
487 size: fileUrl.size, 551 size: fileUrl.size,
488 videoId: video.id, 552 videoId: video.id,
489 fps: fileUrl.fps || -1 553 fps: fileUrl.fps || -1
490 } as VideoFileModel 554 }
555
556 attributes.push(attribute)
557 }
558
559 return attributes
560}
561
562function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
563 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
564 if (playlistUrls.length === 0) return []
565
566 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
567 for (const playlistUrlObject of playlistUrls) {
568 const p2pMediaLoaderInfohashes = playlistUrlObject.tag
569 .filter(t => t.type === 'Infohash')
570 .map(t => t.name)
571 if (p2pMediaLoaderInfohashes.length === 0) {
572 logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
573 continue
574 }
575
576 const segmentsSha256UrlObject = playlistUrlObject.tag
577 .find(t => {
578 return isAPPlaylistSegmentHashesUrlObject(t)
579 }) as ActivityPlaylistSegmentHashesObject
580 if (!segmentsSha256UrlObject) {
581 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
582 continue
583 }
584
585 const attribute = {
586 type: VideoStreamingPlaylistType.HLS,
587 playlistUrl: playlistUrlObject.href,
588 segmentsSha256Url: segmentsSha256UrlObject.href,
589 p2pMediaLoaderInfohashes,
590 videoId: video.id
591 }
592
491 attributes.push(attribute) 593 attributes.push(attribute)
492 } 594 }
493 595
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 4b6bc3185..021426a1a 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -7,10 +7,11 @@ import { AccountModel } from '../models/account/account'
7import { VideoChannelModel } from '../models/video/video-channel' 7import { VideoChannelModel } from '../models/video/video-channel'
8import { extname, join } from 'path' 8import { extname, join } from 'path'
9import { retryTransactionWrapper } from '../helpers/database-utils' 9import { retryTransactionWrapper } from '../helpers/database-utils'
10import * as uuidv4 from 'uuid/v4'
10 11
11async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { 12async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
12 const extension = extname(avatarPhysicalFile.filename) 13 const extension = extname(avatarPhysicalFile.filename)
13 const avatarName = accountOrChannel.Actor.uuid + extension 14 const avatarName = uuidv4() + extension
14 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) 15 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
15 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) 16 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE)
16 17
diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts
new file mode 100644
index 000000000..1633e500c
--- /dev/null
+++ b/server/lib/blocklist.ts
@@ -0,0 +1,40 @@
1import { sequelizeTypescript } from '../initializers'
2import { AccountBlocklistModel } from '../models/account/account-blocklist'
3import { ServerBlocklistModel } from '../models/server/server-blocklist'
4
5function addAccountInBlocklist (byAccountId: number, targetAccountId: number) {
6 return sequelizeTypescript.transaction(async t => {
7 return AccountBlocklistModel.upsert({
8 accountId: byAccountId,
9 targetAccountId: targetAccountId
10 }, { transaction: t })
11 })
12}
13
14function addServerInBlocklist (byAccountId: number, targetServerId: number) {
15 return sequelizeTypescript.transaction(async t => {
16 return ServerBlocklistModel.upsert({
17 accountId: byAccountId,
18 targetServerId
19 }, { transaction: t })
20 })
21}
22
23function removeAccountFromBlocklist (accountBlock: AccountBlocklistModel) {
24 return sequelizeTypescript.transaction(async t => {
25 return accountBlock.destroy({ transaction: t })
26 })
27}
28
29function removeServerFromBlocklist (serverBlock: ServerBlocklistModel) {
30 return sequelizeTypescript.transaction(async t => {
31 return serverBlock.destroy({ transaction: t })
32 })
33}
34
35export {
36 addAccountInBlocklist,
37 addServerInBlocklist,
38 removeAccountFromBlocklist,
39 removeServerFromBlocklist
40}
diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/cache/actor-follow-score-cache.ts
new file mode 100644
index 000000000..d070bde09
--- /dev/null
+++ b/server/lib/cache/actor-follow-score-cache.ts
@@ -0,0 +1,46 @@
1import { ACTOR_FOLLOW_SCORE } from '../../initializers'
2import { logger } from '../../helpers/logger'
3
4// Cache follows scores, instead of writing them too often in database
5// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores
6class ActorFollowScoreCache {
7
8 private static instance: ActorFollowScoreCache
9 private pendingFollowsScore: { [ url: string ]: number } = {}
10
11 private constructor () {}
12
13 static get Instance () {
14 return this.instance || (this.instance = new this())
15 }
16
17 updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) {
18 if (goodInboxes.length === 0 && badInboxes.length === 0) return
19
20 logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length)
21
22 for (const goodInbox of goodInboxes) {
23 if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0
24
25 this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS
26 }
27
28 for (const badInbox of badInboxes) {
29 if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0
30
31 this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY
32 }
33 }
34
35 getPendingFollowsScoreCopy () {
36 return this.pendingFollowsScore
37 }
38
39 clearPendingFollowsScore () {
40 this.pendingFollowsScore = {}
41 }
42}
43
44export {
45 ActorFollowScoreCache
46}
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts
index 54eb983fa..e921d04a7 100644
--- a/server/lib/cache/index.ts
+++ b/server/lib/cache/index.ts
@@ -1,2 +1,3 @@
1export * from './actor-follow-score-cache'
1export * from './videos-preview-cache' 2export * from './videos-preview-cache'
2export * from './videos-caption-cache' 3export * from './videos-caption-cache'
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index fc013e0c3..b2c376e20 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import * as Bluebird from 'bluebird' 2import * as Bluebird from 'bluebird'
3import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' 3import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n'
4import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, STATIC_PATHS } from '../initializers' 4import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers'
5import { join } from 'path' 5import { join } from 'path'
6import { escapeHTML } from '../helpers/core-utils' 6import { escapeHTML } from '../helpers/core-utils'
7import { VideoModel } from '../models/video/video' 7import { VideoModel } from '../models/video/video'
@@ -18,21 +18,13 @@ export class ClientHtml {
18 ClientHtml.htmlCache = {} 18 ClientHtml.htmlCache = {}
19 } 19 }
20 20
21 static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { 21 static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) {
22 const path = ClientHtml.getIndexPath(req, res, paramLang) 22 const html = await ClientHtml.getIndexHTML(req, res, paramLang)
23 if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path]
24
25 const buffer = await readFile(path)
26 23
27 let html = buffer.toString() 24 let customHtml = ClientHtml.addTitleTag(html)
28 25 customHtml = ClientHtml.addDescriptionTag(customHtml)
29 html = ClientHtml.addTitleTag(html)
30 html = ClientHtml.addDescriptionTag(html)
31 html = ClientHtml.addCustomCSS(html)
32 26
33 ClientHtml.htmlCache[path] = html 27 return customHtml
34
35 return html
36 } 28 }
37 29
38 static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { 30 static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) {
@@ -55,7 +47,26 @@ export class ClientHtml {
55 return ClientHtml.getIndexHTML(req, res) 47 return ClientHtml.getIndexHTML(req, res)
56 } 48 }
57 49
58 return ClientHtml.addOpenGraphAndOEmbedTags(html, video) 50 let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name))
51 customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description))
52 customHtml = ClientHtml.addOpenGraphAndOEmbedTags(customHtml, video)
53
54 return customHtml
55 }
56
57 private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) {
58 const path = ClientHtml.getIndexPath(req, res, paramLang)
59 if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path]
60
61 const buffer = await readFile(path)
62
63 let html = buffer.toString()
64
65 html = ClientHtml.addCustomCSS(html)
66
67 ClientHtml.htmlCache[path] = html
68
69 return html
59 } 70 }
60 71
61 private static getIndexPath (req: express.Request, res: express.Response, paramLang?: string) { 72 private static getIndexPath (req: express.Request, res: express.Response, paramLang?: string) {
@@ -81,14 +92,18 @@ export class ClientHtml {
81 return join(__dirname, '../../../client/dist/' + buildFileLocale(lang) + '/index.html') 92 return join(__dirname, '../../../client/dist/' + buildFileLocale(lang) + '/index.html')
82 } 93 }
83 94
84 private static addTitleTag (htmlStringPage: string) { 95 private static addTitleTag (htmlStringPage: string, title?: string) {
85 const titleTag = '<title>' + CONFIG.INSTANCE.NAME + '</title>' 96 let text = title || CONFIG.INSTANCE.NAME
97 if (title) text += ` - ${CONFIG.INSTANCE.NAME}`
98
99 const titleTag = `<title>${text}</title>`
86 100
87 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) 101 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag)
88 } 102 }
89 103
90 private static addDescriptionTag (htmlStringPage: string) { 104 private static addDescriptionTag (htmlStringPage: string, description?: string) {
91 const descriptionTag = `<meta name="description" content="${CONFIG.INSTANCE.SHORT_DESCRIPTION}" />` 105 const content = description || CONFIG.INSTANCE.SHORT_DESCRIPTION
106 const descriptionTag = `<meta name="description" content="${content}" />`
92 107
93 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) 108 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag)
94 } 109 }
@@ -100,8 +115,8 @@ export class ClientHtml {
100 } 115 }
101 116
102 private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { 117 private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) {
103 const previewUrl = CONFIG.WEBSERVER.URL + STATIC_PATHS.PREVIEWS + video.getPreviewName() 118 const previewUrl = CONFIG.WEBSERVER.URL + video.getPreviewStaticPath()
104 const videoUrl = CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 119 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath()
105 120
106 const videoNameEscaped = escapeHTML(video.name) 121 const videoNameEscaped = escapeHTML(video.name)
107 const videoDescriptionEscaped = escapeHTML(video.description) 122 const videoDescriptionEscaped = escapeHTML(video.description)
@@ -172,8 +187,8 @@ export class ClientHtml {
172 // Schema.org 187 // Schema.org
173 tagsString += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>` 188 tagsString += `<script type="application/ld+json">${JSON.stringify(schemaTags)}</script>`
174 189
175 // SEO 190 // SEO, use origin video url so Google does not index remote videos
176 tagsString += `<link rel="canonical" href="${videoUrl}" />` 191 tagsString += `<link rel="canonical" href="${video.url}" />`
177 192
178 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) 193 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString)
179 } 194 }
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts
index 9327792fb..672414cc0 100644
--- a/server/lib/emailer.ts
+++ b/server/lib/emailer.ts
@@ -1,5 +1,4 @@
1import { createTransport, Transporter } from 'nodemailer' 1import { createTransport, Transporter } from 'nodemailer'
2import { UserRight } from '../../shared/models/users'
3import { isTestInstance } from '../helpers/core-utils' 2import { isTestInstance } from '../helpers/core-utils'
4import { bunyanLogger, logger } from '../helpers/logger' 3import { bunyanLogger, logger } from '../helpers/logger'
5import { CONFIG } from '../initializers' 4import { CONFIG } from '../initializers'
@@ -8,6 +7,11 @@ import { VideoModel } from '../models/video/video'
8import { JobQueue } from './job-queue' 7import { JobQueue } from './job-queue'
9import { EmailPayload } from './job-queue/handlers/email' 8import { EmailPayload } from './job-queue/handlers/email'
10import { readFileSync } from 'fs-extra' 9import { readFileSync } from 'fs-extra'
10import { VideoCommentModel } from '../models/video/video-comment'
11import { VideoAbuseModel } from '../models/video/video-abuse'
12import { VideoBlacklistModel } from '../models/video/video-blacklist'
13import { VideoImportModel } from '../models/video/video-import'
14import { ActorFollowModel } from '../models/activitypub/actor-follow'
11 15
12class Emailer { 16class Emailer {
13 17
@@ -22,7 +26,7 @@ class Emailer {
22 if (this.initialized === true) return 26 if (this.initialized === true) return
23 this.initialized = true 27 this.initialized = true
24 28
25 if (CONFIG.SMTP.HOSTNAME && CONFIG.SMTP.PORT) { 29 if (Emailer.isEnabled()) {
26 logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) 30 logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT)
27 31
28 let tls 32 let tls
@@ -57,6 +61,10 @@ class Emailer {
57 } 61 }
58 } 62 }
59 63
64 static isEnabled () {
65 return !!CONFIG.SMTP.HOSTNAME && !!CONFIG.SMTP.PORT
66 }
67
60 async checkConnectionOrDie () { 68 async checkConnectionOrDie () {
61 if (!this.transporter) return 69 if (!this.transporter) return
62 70
@@ -72,50 +80,158 @@ class Emailer {
72 } 80 }
73 } 81 }
74 82
75 addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { 83 addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) {
84 const channelName = video.VideoChannel.getDisplayName()
85 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath()
86
76 const text = `Hi dear user,\n\n` + 87 const text = `Hi dear user,\n\n` +
77 `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + 88 `Your subscription ${channelName} just published a new video: ${video.name}` +
78 `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + 89 `\n\n` +
79 `If you are not the person who initiated this request, please ignore this email.\n\n` + 90 `You can view it on ${videoUrl} ` +
91 `\n\n` +
80 `Cheers,\n` + 92 `Cheers,\n` +
81 `PeerTube.` 93 `PeerTube.`
82 94
83 const emailPayload: EmailPayload = { 95 const emailPayload: EmailPayload = {
84 to: [ to ], 96 to,
85 subject: 'Reset your PeerTube password', 97 subject: channelName + ' just published a new video',
86 text 98 text
87 } 99 }
88 100
89 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 101 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
90 } 102 }
91 103
92 addVerifyEmailJob (to: string, verifyEmailUrl: string) { 104 addNewFollowNotification (to: string[], actorFollow: ActorFollowModel, followType: 'account' | 'channel') {
93 const text = `Welcome to PeerTube,\n\n` + 105 const followerName = actorFollow.ActorFollower.Account.getDisplayName()
94 `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + 106 const followingName = (actorFollow.ActorFollowing.VideoChannel || actorFollow.ActorFollowing.Account).getDisplayName()
95 `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + 107
96 `If you are not the person who initiated this request, please ignore this email.\n\n` + 108 const text = `Hi dear user,\n\n` +
109 `Your ${followType} ${followingName} has a new subscriber: ${followerName}` +
110 `\n\n` +
97 `Cheers,\n` + 111 `Cheers,\n` +
98 `PeerTube.` 112 `PeerTube.`
99 113
100 const emailPayload: EmailPayload = { 114 const emailPayload: EmailPayload = {
101 to: [ to ], 115 to,
102 subject: 'Verify your PeerTube email', 116 subject: 'New follower on your channel ' + followingName,
117 text
118 }
119
120 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
121 }
122
123 myVideoPublishedNotification (to: string[], video: VideoModel) {
124 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath()
125
126 const text = `Hi dear user,\n\n` +
127 `Your video ${video.name} has been published.` +
128 `\n\n` +
129 `You can view it on ${videoUrl} ` +
130 `\n\n` +
131 `Cheers,\n` +
132 `PeerTube.`
133
134 const emailPayload: EmailPayload = {
135 to,
136 subject: `Your video ${video.name} is published`,
137 text
138 }
139
140 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
141 }
142
143 myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) {
144 const videoUrl = CONFIG.WEBSERVER.URL + videoImport.Video.getWatchStaticPath()
145
146 const text = `Hi dear user,\n\n` +
147 `Your video import ${videoImport.getTargetIdentifier()} is finished.` +
148 `\n\n` +
149 `You can view the imported video on ${videoUrl} ` +
150 `\n\n` +
151 `Cheers,\n` +
152 `PeerTube.`
153
154 const emailPayload: EmailPayload = {
155 to,
156 subject: `Your video import ${videoImport.getTargetIdentifier()} is finished`,
157 text
158 }
159
160 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
161 }
162
163 myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) {
164 const importUrl = CONFIG.WEBSERVER.URL + '/my-account/video-imports'
165
166 const text = `Hi dear user,\n\n` +
167 `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` +
168 `\n\n` +
169 `See your videos import dashboard for more information: ${importUrl}` +
170 `\n\n` +
171 `Cheers,\n` +
172 `PeerTube.`
173
174 const emailPayload: EmailPayload = {
175 to,
176 subject: `Your video import ${videoImport.getTargetIdentifier()} encountered an error`,
177 text
178 }
179
180 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
181 }
182
183 addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) {
184 const accountName = comment.Account.getDisplayName()
185 const video = comment.Video
186 const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath()
187
188 const text = `Hi dear user,\n\n` +
189 `A new comment has been posted by ${accountName} on your video ${video.name}` +
190 `\n\n` +
191 `You can view it on ${commentUrl} ` +
192 `\n\n` +
193 `Cheers,\n` +
194 `PeerTube.`
195
196 const emailPayload: EmailPayload = {
197 to,
198 subject: 'New comment on your video ' + video.name,
103 text 199 text
104 } 200 }
105 201
106 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 202 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
107 } 203 }
108 204
109 async addVideoAbuseReportJob (videoId: number) { 205 addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) {
110 const video = await VideoModel.load(videoId) 206 const accountName = comment.Account.getDisplayName()
111 if (!video) throw new Error('Unknown Video id during Abuse report.') 207 const video = comment.Video
208 const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath()
209
210 const text = `Hi dear user,\n\n` +
211 `${accountName} mentioned you on video ${video.name}` +
212 `\n\n` +
213 `You can view the comment on ${commentUrl} ` +
214 `\n\n` +
215 `Cheers,\n` +
216 `PeerTube.`
217
218 const emailPayload: EmailPayload = {
219 to,
220 subject: 'Mention on video ' + video.name,
221 text
222 }
223
224 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
225 }
226
227 addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) {
228 const videoUrl = CONFIG.WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath()
112 229
113 const text = `Hi,\n\n` + 230 const text = `Hi,\n\n` +
114 `Your instance received an abuse for the following video ${video.url}\n\n` + 231 `${CONFIG.WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` +
115 `Cheers,\n` + 232 `Cheers,\n` +
116 `PeerTube.` 233 `PeerTube.`
117 234
118 const to = await UserModel.listEmailsWithRight(UserRight.MANAGE_VIDEO_ABUSES)
119 const emailPayload: EmailPayload = { 235 const emailPayload: EmailPayload = {
120 to, 236 to,
121 subject: '[PeerTube] Received a video abuse', 237 subject: '[PeerTube] Received a video abuse',
@@ -125,16 +241,27 @@ class Emailer {
125 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 241 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
126 } 242 }
127 243
128 async addVideoBlacklistReportJob (videoId: number, reason?: string) { 244 addNewUserRegistrationNotification (to: string[], user: UserModel) {
129 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) 245 const text = `Hi,\n\n` +
130 if (!video) throw new Error('Unknown Video id during Blacklist report.') 246 `User ${user.username} just registered on ${CONFIG.WEBSERVER.HOST} PeerTube instance.\n\n` +
131 // It's not our user 247 `Cheers,\n` +
132 if (video.remote === true) return 248 `PeerTube.`
249
250 const emailPayload: EmailPayload = {
251 to,
252 subject: '[PeerTube] New user registration on ' + CONFIG.WEBSERVER.HOST,
253 text
254 }
255
256 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
257 }
133 258
134 const user = await UserModel.loadById(video.VideoChannel.Account.userId) 259 addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) {
260 const videoName = videoBlacklist.Video.name
261 const videoUrl = CONFIG.WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath()
135 262
136 const reasonString = reason ? ` for the following reason: ${reason}` : '' 263 const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : ''
137 const blockedString = `Your video ${video.name} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` 264 const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.`
138 265
139 const text = 'Hi,\n\n' + 266 const text = 'Hi,\n\n' +
140 blockedString + 267 blockedString +
@@ -142,33 +269,26 @@ class Emailer {
142 'Cheers,\n' + 269 'Cheers,\n' +
143 `PeerTube.` 270 `PeerTube.`
144 271
145 const to = user.email
146 const emailPayload: EmailPayload = { 272 const emailPayload: EmailPayload = {
147 to: [ to ], 273 to,
148 subject: `[PeerTube] Video ${video.name} blacklisted`, 274 subject: `[PeerTube] Video ${videoName} blacklisted`,
149 text 275 text
150 } 276 }
151 277
152 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 278 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
153 } 279 }
154 280
155 async addVideoUnblacklistReportJob (videoId: number) { 281 addVideoUnblacklistNotification (to: string[], video: VideoModel) {
156 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) 282 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath()
157 if (!video) throw new Error('Unknown Video id during Blacklist report.')
158 // It's not our user
159 if (video.remote === true) return
160
161 const user = await UserModel.loadById(video.VideoChannel.Account.userId)
162 283
163 const text = 'Hi,\n\n' + 284 const text = 'Hi,\n\n' +
164 `Your video ${video.name} on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + 285 `Your video ${video.name} (${videoUrl}) on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` +
165 '\n\n' + 286 '\n\n' +
166 'Cheers,\n' + 287 'Cheers,\n' +
167 `PeerTube.` 288 `PeerTube.`
168 289
169 const to = user.email
170 const emailPayload: EmailPayload = { 290 const emailPayload: EmailPayload = {
171 to: [ to ], 291 to,
172 subject: `[PeerTube] Video ${video.name} unblacklisted`, 292 subject: `[PeerTube] Video ${video.name} unblacklisted`,
173 text 293 text
174 } 294 }
@@ -176,6 +296,40 @@ class Emailer {
176 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 296 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
177 } 297 }
178 298
299 addPasswordResetEmailJob (to: string, resetPasswordUrl: string) {
300 const text = `Hi dear user,\n\n` +
301 `A reset password procedure for your account ${to} has been requested on ${CONFIG.WEBSERVER.HOST} ` +
302 `Please follow this link to reset it: ${resetPasswordUrl}\n\n` +
303 `If you are not the person who initiated this request, please ignore this email.\n\n` +
304 `Cheers,\n` +
305 `PeerTube.`
306
307 const emailPayload: EmailPayload = {
308 to: [ to ],
309 subject: 'Reset your PeerTube password',
310 text
311 }
312
313 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
314 }
315
316 addVerifyEmailJob (to: string, verifyEmailUrl: string) {
317 const text = `Welcome to PeerTube,\n\n` +
318 `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` +
319 `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` +
320 `If you are not the person who initiated this request, please ignore this email.\n\n` +
321 `Cheers,\n` +
322 `PeerTube.`
323
324 const emailPayload: EmailPayload = {
325 to: [ to ],
326 subject: 'Verify your PeerTube email',
327 text
328 }
329
330 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
331 }
332
179 addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { 333 addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) {
180 const reasonString = reason ? ` for the following reason: ${reason}` : '' 334 const reasonString = reason ? ` for the following reason: ${reason}` : ''
181 const blockedWord = blocked ? 'blocked' : 'unblocked' 335 const blockedWord = blocked ? 'blocked' : 'unblocked'
@@ -197,13 +351,32 @@ class Emailer {
197 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 351 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
198 } 352 }
199 353
200 sendMail (to: string[], subject: string, text: string) { 354 addContactFormJob (fromEmail: string, fromName: string, body: string) {
201 if (!this.transporter) { 355 const text = 'Hello dear admin,\n\n' +
356 fromName + ' sent you a message' +
357 '\n\n---------------------------------------\n\n' +
358 body +
359 '\n\n---------------------------------------\n\n' +
360 'Cheers,\n' +
361 'PeerTube.'
362
363 const emailPayload: EmailPayload = {
364 from: fromEmail,
365 to: [ CONFIG.ADMIN.EMAIL ],
366 subject: '[PeerTube] Contact form submitted',
367 text
368 }
369
370 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
371 }
372
373 sendMail (to: string[], subject: string, text: string, from?: string) {
374 if (!Emailer.isEnabled()) {
202 throw new Error('Cannot send mail because SMTP is not configured.') 375 throw new Error('Cannot send mail because SMTP is not configured.')
203 } 376 }
204 377
205 return this.transporter.sendMail({ 378 return this.transporter.sendMail({
206 from: CONFIG.SMTP.FROM_ADDRESS, 379 from: from || CONFIG.SMTP.FROM_ADDRESS,
207 to: to.join(','), 380 to: to.join(','),
208 subject, 381 subject,
209 text 382 text
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
new file mode 100644
index 000000000..3575981f4
--- /dev/null
+++ b/server/lib/hls.ts
@@ -0,0 +1,164 @@
1import { VideoModel } from '../models/video/video'
2import { basename, join, dirname } from 'path'
3import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import { logger } from '../helpers/logger'
9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
10import { generateRandomString } from '../helpers/utils'
11import { flatten, uniq } from 'lodash'
12
13async function updateMasterHLSPlaylist (video: VideoModel) {
14 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
15 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
16 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
17
18 for (const file of video.VideoFiles) {
19 // If we did not generated a playlist for this resolution, skip
20 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
21 if (await pathExists(filePlaylistPath) === false) continue
22
23 const videoFilePath = video.getVideoFilePath(file)
24
25 const size = await getVideoFileSize(videoFilePath)
26
27 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
28 const resolution = `RESOLUTION=${size.width}x${size.height}`
29
30 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
31 if (file.fps) line += ',FRAME-RATE=' + file.fps
32
33 masterPlaylists.push(line)
34 masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
35 }
36
37 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
38}
39
40async function updateSha256Segments (video: VideoModel) {
41 const json: { [filename: string]: { [range: string]: string } } = {}
42
43 const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
44
45 // For all the resolutions available for this video
46 for (const file of video.VideoFiles) {
47 const rangeHashes: { [range: string]: string } = {}
48
49 const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
50 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
51
52 // Maybe the playlist is not generated for this resolution yet
53 if (!await pathExists(playlistPath)) continue
54
55 const playlistContent = await readFile(playlistPath)
56 const ranges = getRangesFromPlaylist(playlistContent.toString())
57
58 const fd = await open(videoPath, 'r')
59 for (const range of ranges) {
60 const buf = Buffer.alloc(range.length)
61 await read(fd, buf, 0, range.length, range.offset)
62
63 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
64 }
65 await close(fd)
66
67 const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
68 json[videoFilename] = rangeHashes
69 }
70
71 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
72 await outputJSON(outputPath, json)
73}
74
75function getRangesFromPlaylist (playlistContent: string) {
76 const ranges: { offset: number, length: number }[] = []
77 const lines = playlistContent.split('\n')
78 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
79
80 for (const line of lines) {
81 const captured = regex.exec(line)
82
83 if (captured) {
84 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
85 }
86 }
87
88 return ranges
89}
90
91function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
92 let timer
93
94 logger.info('Importing HLS playlist %s', playlistUrl)
95
96 return new Promise<string>(async (res, rej) => {
97 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
98
99 await ensureDir(tmpDirectory)
100
101 timer = setTimeout(() => {
102 deleteTmpDirectory(tmpDirectory)
103
104 return rej(new Error('HLS download timeout.'))
105 }, timeout)
106
107 try {
108 // Fetch master playlist
109 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
110
111 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
112 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
113
114 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
115
116 for (const fileUrl of fileUrls) {
117 const destPath = join(tmpDirectory, basename(fileUrl))
118
119 await doRequestAndSaveToFile({ uri: fileUrl }, destPath)
120 }
121
122 clearTimeout(timer)
123
124 await move(tmpDirectory, destinationDir, { overwrite: true })
125
126 return res()
127 } catch (err) {
128 deleteTmpDirectory(tmpDirectory)
129
130 return rej(err)
131 }
132 })
133
134 function deleteTmpDirectory (directory: string) {
135 remove(directory)
136 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
137 }
138
139 async function fetchUniqUrls (playlistUrl: string) {
140 const { body } = await doRequest<string>({ uri: playlistUrl })
141
142 if (!body) return []
143
144 const urls = body.split('\n')
145 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
146 .map(url => {
147 if (url.startsWith('http://') || url.startsWith('https://')) return url
148
149 return `${dirname(playlistUrl)}/${url}`
150 })
151
152 return uniq(urls)
153 }
154}
155
156// ---------------------------------------------------------------------------
157
158export {
159 updateMasterHLSPlaylist,
160 updateSha256Segments,
161 downloadPlaylistSegments
162}
163
164// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts
index 36d0f237b..b4d381062 100644
--- a/server/lib/job-queue/handlers/activitypub-follow.ts
+++ b/server/lib/job-queue/handlers/activitypub-follow.ts
@@ -8,6 +8,7 @@ import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor'
8import { retryTransactionWrapper } from '../../../helpers/database-utils' 8import { retryTransactionWrapper } from '../../../helpers/database-utils'
9import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 9import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
10import { ActorModel } from '../../../models/activitypub/actor' 10import { ActorModel } from '../../../models/activitypub/actor'
11import { Notifier } from '../../notifier'
11 12
12export type ActivitypubFollowPayload = { 13export type ActivitypubFollowPayload = {
13 followerActorId: number 14 followerActorId: number
@@ -42,7 +43,7 @@ export {
42 43
43// --------------------------------------------------------------------------- 44// ---------------------------------------------------------------------------
44 45
45function follow (fromActor: ActorModel, targetActor: ActorModel) { 46async function follow (fromActor: ActorModel, targetActor: ActorModel) {
46 if (fromActor.id === targetActor.id) { 47 if (fromActor.id === targetActor.id) {
47 throw new Error('Follower is the same than target actor.') 48 throw new Error('Follower is the same than target actor.')
48 } 49 }
@@ -50,7 +51,7 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) {
50 // Same server, direct accept 51 // Same server, direct accept
51 const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' 52 const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending'
52 53
53 return sequelizeTypescript.transaction(async t => { 54 const actorFollow = await sequelizeTypescript.transaction(async t => {
54 const [ actorFollow ] = await ActorFollowModel.findOrCreate({ 55 const [ actorFollow ] = await ActorFollowModel.findOrCreate({
55 where: { 56 where: {
56 actorId: fromActor.id, 57 actorId: fromActor.id,
@@ -68,5 +69,9 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) {
68 69
69 // Send a notification to remote server if our follow is not already accepted 70 // Send a notification to remote server if our follow is not already accepted
70 if (actorFollow.state !== 'accepted') await sendFollow(actorFollow) 71 if (actorFollow.state !== 'accepted') await sendFollow(actorFollow)
72
73 return actorFollow
71 }) 74 })
75
76 if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewFollow(actorFollow)
72} 77}
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 03a9e12a4..9493945ff 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -3,8 +3,9 @@ import * as Bluebird from 'bluebird'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { doRequest } from '../../../helpers/requests' 4import { doRequest } from '../../../helpers/requests'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 6import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
7import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' 7import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers'
8import { ActorFollowScoreCache } from '../../cache'
8 9
9export type ActivitypubHttpBroadcastPayload = { 10export type ActivitypubHttpBroadcastPayload = {
10 uris: string[] 11 uris: string[]
@@ -25,7 +26,8 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
25 uri: '', 26 uri: '',
26 json: body, 27 json: body,
27 httpSignature: httpSignatureOptions, 28 httpSignature: httpSignatureOptions,
28 timeout: JOB_REQUEST_TIMEOUT 29 timeout: JOB_REQUEST_TIMEOUT,
30 headers: buildGlobalHeaders(body)
29 } 31 }
30 32
31 const badUrls: string[] = [] 33 const badUrls: string[] = []
@@ -37,7 +39,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
37 .catch(() => badUrls.push(uri)) 39 .catch(() => badUrls.push(uri))
38 }, { concurrency: BROADCAST_CONCURRENCY }) 40 }, { concurrency: BROADCAST_CONCURRENCY })
39 41
40 return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined) 42 return ActorFollowScoreCache.Instance.updateActorFollowsScore(goodUrls, badUrls)
41} 43}
42 44
43// --------------------------------------------------------------------------- 45// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 42217c27c..67ccfa995 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -23,7 +23,7 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) 23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
24 24
25 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { 25 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
26 'activity': items => processActivities(items), 26 'activity': items => processActivities(items, { outboxUrl: payload.uri }),
27 'video-likes': items => createRates(items, video, 'like'), 27 'video-likes': items => createRates(items, video, 'like'),
28 'video-dislikes': items => createRates(items, video, 'dislike'), 28 'video-dislikes': items => createRates(items, video, 'dislike'),
29 'video-shares': items => addVideoShares(items, video), 29 'video-shares': items => addVideoShares(items, video),
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index c90d735f6..3973dcdc8 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -1,9 +1,9 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { doRequest } from '../../../helpers/requests' 3import { doRequest } from '../../../helpers/requests'
4import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 4import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
5import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
6import { JOB_REQUEST_TIMEOUT } from '../../../initializers' 5import { JOB_REQUEST_TIMEOUT } from '../../../initializers'
6import { ActorFollowScoreCache } from '../../cache'
7 7
8export type ActivitypubHttpUnicastPayload = { 8export type ActivitypubHttpUnicastPayload = {
9 uri: string 9 uri: string
@@ -25,14 +25,15 @@ async function processActivityPubHttpUnicast (job: Bull.Job) {
25 uri, 25 uri,
26 json: body, 26 json: body,
27 httpSignature: httpSignatureOptions, 27 httpSignature: httpSignatureOptions,
28 timeout: JOB_REQUEST_TIMEOUT 28 timeout: JOB_REQUEST_TIMEOUT,
29 headers: buildGlobalHeaders(body)
29 } 30 }
30 31
31 try { 32 try {
32 await doRequest(options) 33 await doRequest(options)
33 ActorFollowModel.updateActorFollowsScore([ uri ], [], undefined) 34 ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], [])
34 } catch (err) { 35 } catch (err) {
35 ActorFollowModel.updateActorFollowsScore([], [ uri ], undefined) 36 ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ])
36 37
37 throw err 38 throw err
38 } 39 }
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
new file mode 100644
index 000000000..454b975fe
--- /dev/null
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -0,0 +1,54 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { fetchVideoByUrl } from '../../../helpers/video'
4import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub'
5import { ActorModel } from '../../../models/activitypub/actor'
6
7export type RefreshPayload = {
8 type: 'video' | 'actor'
9 url: string
10}
11
12async function refreshAPObject (job: Bull.Job) {
13 const payload = job.data as RefreshPayload
14
15 logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url)
16
17 if (payload.type === 'video') return refreshVideo(payload.url)
18 if (payload.type === 'actor') return refreshActor(payload.url)
19}
20
21// ---------------------------------------------------------------------------
22
23export {
24 refreshActor,
25 refreshAPObject
26}
27
28// ---------------------------------------------------------------------------
29
30async function refreshVideo (videoUrl: string) {
31 const fetchType = 'all' as 'all'
32 const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true }
33
34 const videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
35 if (videoFromDatabase) {
36 const refreshOptions = {
37 video: videoFromDatabase,
38 fetchedType: fetchType,
39 syncParam
40 }
41
42 await refreshVideoIfNeeded(refreshOptions)
43 }
44}
45
46async function refreshActor (actorUrl: string) {
47 const fetchType = 'all' as 'all'
48 const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl)
49
50 if (actor) {
51 await refreshActorIfNeeded(actor, fetchType)
52 }
53
54}
diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts
index 73d98ae54..220d0af32 100644
--- a/server/lib/job-queue/handlers/email.ts
+++ b/server/lib/job-queue/handlers/email.ts
@@ -6,13 +6,14 @@ export type EmailPayload = {
6 to: string[] 6 to: string[]
7 subject: string 7 subject: string
8 text: string 8 text: string
9 from?: string
9} 10}
10 11
11async function processEmail (job: Bull.Job) { 12async function processEmail (job: Bull.Job) {
12 const payload = job.data as EmailPayload 13 const payload = job.data as EmailPayload
13 logger.info('Processing email in job %d.', job.id) 14 logger.info('Processing email in job %d.', job.id)
14 15
15 return Emailer.Instance.sendMail(payload.to, payload.subject, payload.text) 16 return Emailer.Instance.sendMail(payload.to, payload.subject, payload.text, payload.from)
16} 17}
17 18
18// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
index 36092665e..4961d4502 100644
--- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
+++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
@@ -1,8 +1,12 @@
1import { buildSignedActivity } from '../../../../helpers/activitypub' 1import { buildSignedActivity } from '../../../../helpers/activitypub'
2import { getServerActor } from '../../../../helpers/utils' 2import { getServerActor } from '../../../../helpers/utils'
3import { ActorModel } from '../../../../models/activitypub/actor' 3import { ActorModel } from '../../../../models/activitypub/actor'
4import { sha256 } from '../../../../helpers/core-utils'
5import { HTTP_SIGNATURE } from '../../../../initializers'
4 6
5async function computeBody (payload: { body: any, signatureActorId?: number }) { 7type Payload = { body: any, signatureActorId?: number }
8
9async function computeBody (payload: Payload) {
6 let body = payload.body 10 let body = payload.body
7 11
8 if (payload.signatureActorId) { 12 if (payload.signatureActorId) {
@@ -14,7 +18,7 @@ async function computeBody (payload: { body: any, signatureActorId?: number }) {
14 return body 18 return body
15} 19}
16 20
17async function buildSignedRequestOptions (payload: { signatureActorId?: number }) { 21async function buildSignedRequestOptions (payload: Payload) {
18 let actor: ActorModel | null 22 let actor: ActorModel | null
19 if (payload.signatureActorId) { 23 if (payload.signatureActorId) {
20 actor = await ActorModel.load(payload.signatureActorId) 24 actor = await ActorModel.load(payload.signatureActorId)
@@ -26,14 +30,29 @@ async function buildSignedRequestOptions (payload: { signatureActorId?: number }
26 30
27 const keyId = actor.getWebfingerUrl() 31 const keyId = actor.getWebfingerUrl()
28 return { 32 return {
29 algorithm: 'rsa-sha256', 33 algorithm: HTTP_SIGNATURE.ALGORITHM,
30 authorizationHeaderName: 'Signature', 34 authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME,
31 keyId, 35 keyId,
32 key: actor.privateKey 36 key: actor.privateKey,
37 headers: HTTP_SIGNATURE.HEADERS_TO_SIGN
38 }
39}
40
41function buildGlobalHeaders (body: any) {
42 return {
43 'Digest': buildDigest(body)
33 } 44 }
34} 45}
35 46
47function buildDigest (body: any) {
48 const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
49
50 return 'SHA-256=' + sha256(rawBody, 'base64')
51}
52
36export { 53export {
54 buildDigest,
55 buildGlobalHeaders,
37 computeBody, 56 computeBody,
38 buildSignedRequestOptions 57 buildSignedRequestOptions
39} 58}
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index 1463c93fc..04983155c 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -5,16 +5,18 @@ import { VideoModel } from '../../../models/video/video'
5import { JobQueue } from '../job-queue' 5import { JobQueue } from '../job-queue'
6import { federateVideoIfNeeded } from '../../activitypub' 6import { federateVideoIfNeeded } from '../../activitypub'
7import { retryTransactionWrapper } from '../../../helpers/database-utils' 7import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript, CONFIG } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding' 11import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
12import { Notifier } from '../../notifier'
12 13
13export type VideoFilePayload = { 14export type VideoFilePayload = {
14 videoUUID: string 15 videoUUID: string
15 isNewVideo?: boolean
16 resolution?: VideoResolution 16 resolution?: VideoResolution
17 isNewVideo?: boolean
17 isPortraitMode?: boolean 18 isPortraitMode?: boolean
19 generateHlsPlaylist?: boolean
18} 20}
19 21
20export type VideoFileImportPayload = { 22export type VideoFileImportPayload = {
@@ -50,34 +52,51 @@ async function processVideoFile (job: Bull.Job) {
50 return undefined 52 return undefined
51 } 53 }
52 54
53 // Transcoding in other resolution 55 if (payload.generateHlsPlaylist) {
54 if (payload.resolution) { 56 await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
57
58 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
59 } else if (payload.resolution) { // Transcoding in other resolution
55 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) 60 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
56 61
57 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 62 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload)
58 } else { 63 } else {
59 await optimizeOriginalVideofile(video) 64 await optimizeVideofile(video)
60 65
61 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 66 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
62 } 67 }
63 68
64 return video 69 return video
65} 70}
66 71
67async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { 72async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
73 if (video === undefined) return undefined
74
75 await sequelizeTypescript.transaction(async t => {
76 // Maybe the video changed in database, refresh it
77 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
78 // Video does not exist anymore
79 if (!videoDatabase) return undefined
80
81 // If the video was not published, we consider it is a new one for other instances
82 await federateVideoIfNeeded(videoDatabase, false, t)
83 })
84}
85
86async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) {
68 if (video === undefined) return undefined 87 if (video === undefined) return undefined
69 88
70 return sequelizeTypescript.transaction(async t => { 89 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
71 // Maybe the video changed in database, refresh it 90 // Maybe the video changed in database, refresh it
72 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 91 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
73 // Video does not exist anymore 92 // Video does not exist anymore
74 if (!videoDatabase) return undefined 93 if (!videoDatabase) return undefined
75 94
76 let isNewVideo = false 95 let videoPublished = false
77 96
78 // We transcoded the video file in another format, now we can publish it 97 // We transcoded the video file in another format, now we can publish it
79 if (videoDatabase.state !== VideoState.PUBLISHED) { 98 if (videoDatabase.state !== VideoState.PUBLISHED) {
80 isNewVideo = true 99 videoPublished = true
81 100
82 videoDatabase.state = VideoState.PUBLISHED 101 videoDatabase.state = VideoState.PUBLISHED
83 videoDatabase.publishedAt = new Date() 102 videoDatabase.publishedAt = new Date()
@@ -85,21 +104,29 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
85 } 104 }
86 105
87 // If the video was not published, we consider it is a new one for other instances 106 // If the video was not published, we consider it is a new one for other instances
88 await federateVideoIfNeeded(videoDatabase, isNewVideo, t) 107 await federateVideoIfNeeded(videoDatabase, videoPublished, t)
89 108
90 return undefined 109 return { videoDatabase, videoPublished }
91 }) 110 })
111
112 // don't notify prior to scheduled video update
113 if (videoPublished && !videoDatabase.ScheduleVideoUpdate) {
114 Notifier.Instance.notifyOnNewVideo(videoDatabase)
115 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
116 }
117
118 await createHlsJobIfEnabled(payload)
92} 119}
93 120
94async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boolean) { 121async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) {
95 if (video === undefined) return undefined 122 if (videoArg === undefined) return undefined
96 123
97 // Outside the transaction (IO on disk) 124 // Outside the transaction (IO on disk)
98 const { videoFileResolution } = await video.getOriginalFileResolution() 125 const { videoFileResolution } = await videoArg.getOriginalFileResolution()
99 126
100 return sequelizeTypescript.transaction(async t => { 127 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
101 // Maybe the video changed in database, refresh it 128 // Maybe the video changed in database, refresh it
102 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 129 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t)
103 // Video does not exist anymore 130 // Video does not exist anymore
104 if (!videoDatabase) return undefined 131 if (!videoDatabase) return undefined
105 132
@@ -110,8 +137,10 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
110 { resolutions: resolutionsEnabled } 137 { resolutions: resolutionsEnabled }
111 ) 138 )
112 139
140 let videoPublished = false
141
113 if (resolutionsEnabled.length !== 0) { 142 if (resolutionsEnabled.length !== 0) {
114 const tasks: Bluebird<any>[] = [] 143 const tasks: Bluebird<Bull.Job<any>>[] = []
115 144
116 for (const resolution of resolutionsEnabled) { 145 for (const resolution of resolutionsEnabled) {
117 const dataInput = { 146 const dataInput = {
@@ -127,15 +156,27 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
127 156
128 logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) 157 logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled })
129 } else { 158 } else {
159 videoPublished = true
160
130 // No transcoding to do, it's now published 161 // No transcoding to do, it's now published
131 video.state = VideoState.PUBLISHED 162 videoDatabase.state = VideoState.PUBLISHED
132 video = await video.save({ transaction: t }) 163 videoDatabase = await videoDatabase.save({ transaction: t })
133 164
134 logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid) 165 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
135 } 166 }
136 167
137 return federateVideoIfNeeded(video, isNewVideo, t) 168 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)
169
170 return { videoDatabase, videoPublished }
138 }) 171 })
172
173 // don't notify prior to scheduled video update
174 if (!videoDatabase.ScheduleVideoUpdate) {
175 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
176 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
177 }
178
179 await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
139} 180}
140 181
141// --------------------------------------------------------------------------- 182// ---------------------------------------------------------------------------
@@ -144,3 +185,20 @@ export {
144 processVideoFile, 185 processVideoFile,
145 processVideoFileImport 186 processVideoFileImport
146} 187}
188
189// ---------------------------------------------------------------------------
190
191function createHlsJobIfEnabled (payload?: VideoFilePayload) {
192 // Generate HLS playlist?
193 if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
194 const hlsTranscodingPayload = {
195 videoUUID: payload.videoUUID,
196 resolution: payload.resolution,
197 isPortraitMode: payload.isPortraitMode,
198
199 generateHlsPlaylist: true
200 }
201
202 return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload })
203 }
204}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index e3f2a276c..12004dcd7 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -6,15 +6,16 @@ import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path' 7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file' 8import { VideoFileModel } from '../../../models/video/video-file'
9import { CONFIG, sequelizeTypescript, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' 9import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
10import { doRequestAndSaveToFile } from '../../../helpers/requests' 10import { downloadImage } from '../../../helpers/requests'
11import { VideoState } from '../../../../shared' 11import { VideoState } from '../../../../shared'
12import { JobQueue } from '../index' 12import { JobQueue } from '../index'
13import { federateVideoIfNeeded } from '../../activitypub' 13import { federateVideoIfNeeded } from '../../activitypub'
14import { VideoModel } from '../../../models/video/video' 14import { VideoModel } from '../../../models/video/video'
15import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' 15import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
16import { getSecureTorrentName } from '../../../helpers/utils' 16import { getSecureTorrentName } from '../../../helpers/utils'
17import { remove, rename, stat } from 'fs-extra' 17import { remove, move, stat } from 'fs-extra'
18import { Notifier } from '../../notifier'
18 19
19type VideoImportYoutubeDLPayload = { 20type VideoImportYoutubeDLPayload = {
20 type: 'youtube-dl' 21 type: 'youtube-dl'
@@ -109,6 +110,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
109 let tempVideoPath: string 110 let tempVideoPath: string
110 let videoDestFile: string 111 let videoDestFile: string
111 let videoFile: VideoFileModel 112 let videoFile: VideoFileModel
113
112 try { 114 try {
113 // Download video from youtubeDL 115 // Download video from youtubeDL
114 tempVideoPath = await downloader() 116 tempVideoPath = await downloader()
@@ -133,19 +135,18 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
133 videoId: videoImport.videoId 135 videoId: videoImport.videoId
134 } 136 }
135 videoFile = new VideoFileModel(videoFileData) 137 videoFile = new VideoFileModel(videoFileData)
136 // Import if the import fails, to clean files 138 // To clean files if the import fails
137 videoImport.Video.VideoFiles = [ videoFile ] 139 videoImport.Video.VideoFiles = [ videoFile ]
138 140
139 // Move file 141 // Move file
140 videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) 142 videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
141 await rename(tempVideoPath, videoDestFile) 143 await move(tempVideoPath, videoDestFile)
142 tempVideoPath = null // This path is not used anymore 144 tempVideoPath = null // This path is not used anymore
143 145
144 // Process thumbnail 146 // Process thumbnail
145 if (options.downloadThumbnail) { 147 if (options.downloadThumbnail) {
146 if (options.thumbnailUrl) { 148 if (options.thumbnailUrl) {
147 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) 149 await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE)
148 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
149 } else { 150 } else {
150 await videoImport.Video.createThumbnail(videoFile) 151 await videoImport.Video.createThumbnail(videoFile)
151 } 152 }
@@ -156,8 +157,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
156 // Process preview 157 // Process preview
157 if (options.downloadPreview) { 158 if (options.downloadPreview) {
158 if (options.thumbnailUrl) { 159 if (options.thumbnailUrl) {
159 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) 160 await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE)
160 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
161 } else { 161 } else {
162 await videoImport.Video.createPreview(videoFile) 162 await videoImport.Video.createPreview(videoFile)
163 } 163 }
@@ -180,7 +180,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
180 // Update video DB object 180 // Update video DB object
181 video.duration = duration 181 video.duration = duration
182 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 182 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
183 const videoUpdated = await video.save({ transaction: t }) 183 await video.save({ transaction: t })
184 184
185 // Now we can federate the video (reload from database, we need more attributes) 185 // Now we can federate the video (reload from database, we need more attributes)
186 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 186 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
@@ -192,10 +192,13 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
192 192
193 logger.info('Video %s imported.', video.uuid) 193 logger.info('Video %s imported.', video.uuid)
194 194
195 videoImportUpdated.Video = videoUpdated 195 videoImportUpdated.Video = videoForFederation
196 return videoImportUpdated 196 return videoImportUpdated
197 }) 197 })
198 198
199 Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video)
200 Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true)
201
199 // Create transcoding jobs? 202 // Create transcoding jobs?
200 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { 203 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
201 // Put uuid because we don't have id auto incremented for now 204 // Put uuid because we don't have id auto incremented for now
@@ -218,6 +221,8 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
218 videoImport.state = VideoImportState.FAILED 221 videoImport.state = VideoImportState.FAILED
219 await videoImport.save() 222 await videoImport.save()
220 223
224 Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)
225
221 throw err 226 throw err
222 } 227 }
223} 228}
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index cf180a11a..fa1fd13b3 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -3,8 +3,9 @@ import { logger } from '../../../helpers/logger'
3import { VideoModel } from '../../../models/video/video' 3import { VideoModel } from '../../../models/video/video'
4import { VideoViewModel } from '../../../models/video/video-views' 4import { VideoViewModel } from '../../../models/video/video-views'
5import { isTestInstance } from '../../../helpers/core-utils' 5import { isTestInstance } from '../../../helpers/core-utils'
6import { federateVideoIfNeeded } from '../../activitypub'
6 7
7async function processVideosViewsViews () { 8async function processVideosViews () {
8 const lastHour = new Date() 9 const lastHour = new Date()
9 10
10 // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour 11 // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour
@@ -22,13 +23,9 @@ async function processVideosViewsViews () {
22 for (const videoId of videoIds) { 23 for (const videoId of videoIds) {
23 try { 24 try {
24 const views = await Redis.Instance.getVideoViews(videoId, hour) 25 const views = await Redis.Instance.getVideoViews(videoId, hour)
25 if (isNaN(views)) { 26 if (views) {
26 logger.error('Cannot process videos views of video %d in hour %d: views number is NaN.', videoId, hour)
27 } else {
28 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) 27 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
29 28
30 await VideoModel.incrementViews(videoId, views)
31
32 try { 29 try {
33 await VideoViewModel.create({ 30 await VideoViewModel.create({
34 startDate, 31 startDate,
@@ -36,6 +33,16 @@ async function processVideosViewsViews () {
36 views, 33 views,
37 videoId 34 videoId
38 }) 35 })
36
37 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
38 if (video.isOwned()) {
39 // If this is a remote video, the origin instance will send us an update
40 await VideoModel.incrementViews(videoId, views)
41
42 // Send video update
43 video.views += views
44 await federateVideoIfNeeded(video, false)
45 }
39 } catch (err) { 46 } catch (err) {
40 logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour) 47 logger.debug('Cannot create video views for video %d in hour %d. Maybe the video does not exist anymore?', videoId, hour)
41 } 48 }
@@ -51,5 +58,5 @@ async function processVideosViewsViews () {
51// --------------------------------------------------------------------------- 58// ---------------------------------------------------------------------------
52 59
53export { 60export {
54 processVideosViewsViews 61 processVideosViews
55} 62}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 0696ba43c..ba9cbe0d9 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -10,7 +10,8 @@ import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import' 12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViewsViews } from './handlers/video-views' 13import { processVideosViews } from './handlers/video-views'
14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
14 15
15type CreateJobArgument = 16type CreateJobArgument =
16 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 17 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -21,6 +22,7 @@ type CreateJobArgument =
21 { type: 'video-file', payload: VideoFilePayload } | 22 { type: 'video-file', payload: VideoFilePayload } |
22 { type: 'email', payload: EmailPayload } | 23 { type: 'email', payload: EmailPayload } |
23 { type: 'video-import', payload: VideoImportPayload } | 24 { type: 'video-import', payload: VideoImportPayload } |
25 { type: 'activitypub-refresher', payload: RefreshPayload } |
24 { type: 'videos-views', payload: {} } 26 { type: 'videos-views', payload: {} }
25 27
26const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { 28const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
@@ -32,7 +34,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
32 'video-file': processVideoFile, 34 'video-file': processVideoFile,
33 'email': processEmail, 35 'email': processEmail,
34 'video-import': processVideoImport, 36 'video-import': processVideoImport,
35 'videos-views': processVideosViewsViews 37 'videos-views': processVideosViews,
38 'activitypub-refresher': refreshAPObject
36} 39}
37 40
38const jobTypes: JobType[] = [ 41const jobTypes: JobType[] = [
@@ -44,7 +47,8 @@ const jobTypes: JobType[] = [
44 'video-file', 47 'video-file',
45 'video-file-import', 48 'video-file-import',
46 'video-import', 49 'video-import',
47 'videos-views' 50 'videos-views',
51 'activitypub-refresher'
48] 52]
49 53
50class JobQueue { 54class JobQueue {
@@ -84,7 +88,6 @@ class JobQueue {
84 88
85 queue.on('error', err => { 89 queue.on('error', err => {
86 logger.error('Error in job queue %s.', handlerName, { err }) 90 logger.error('Error in job queue %s.', handlerName, { err })
87 process.exit(-1)
88 }) 91 })
89 92
90 this.queues[handlerName] = queue 93 this.queues[handlerName] = queue
@@ -162,10 +165,10 @@ class JobQueue {
162 return total 165 return total
163 } 166 }
164 167
165 removeOldJobs () { 168 async removeOldJobs () {
166 for (const key of Object.keys(this.queues)) { 169 for (const key of Object.keys(this.queues)) {
167 const queue = this.queues[key] 170 const queue = this.queues[key]
168 queue.clean(JOB_COMPLETED_LIFETIME, 'completed') 171 await queue.clean(JOB_COMPLETED_LIFETIME, 'completed')
169 } 172 }
170 } 173 }
171 174
diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts
new file mode 100644
index 000000000..d1b331346
--- /dev/null
+++ b/server/lib/notifier.ts
@@ -0,0 +1,455 @@
1import { UserNotificationSettingValue, UserNotificationType, UserRight } from '../../shared/models/users'
2import { logger } from '../helpers/logger'
3import { VideoModel } from '../models/video/video'
4import { Emailer } from './emailer'
5import { UserNotificationModel } from '../models/account/user-notification'
6import { VideoCommentModel } from '../models/video/video-comment'
7import { UserModel } from '../models/account/user'
8import { PeerTubeSocket } from './peertube-socket'
9import { CONFIG } from '../initializers/constants'
10import { VideoPrivacy, VideoState } from '../../shared/models/videos'
11import { VideoAbuseModel } from '../models/video/video-abuse'
12import { VideoBlacklistModel } from '../models/video/video-blacklist'
13import * as Bluebird from 'bluebird'
14import { VideoImportModel } from '../models/video/video-import'
15import { AccountBlocklistModel } from '../models/account/account-blocklist'
16import { ActorFollowModel } from '../models/activitypub/actor-follow'
17import { AccountModel } from '../models/account/account'
18
19class Notifier {
20
21 private static instance: Notifier
22
23 private constructor () {}
24
25 notifyOnNewVideo (video: VideoModel): void {
26 // Only notify on public and published videos
27 if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED) return
28
29 this.notifySubscribersOfNewVideo(video)
30 .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err }))
31 }
32
33 notifyOnPendingVideoPublished (video: VideoModel): void {
34 // Only notify on public videos that has been published while the user waited transcoding/scheduled update
35 if (video.waitTranscoding === false && !video.ScheduleVideoUpdate) return
36
37 this.notifyOwnedVideoHasBeenPublished(video)
38 .catch(err => logger.error('Cannot notify owner that its video %s has been published.', video.url, { err }))
39 }
40
41 notifyOnNewComment (comment: VideoCommentModel): void {
42 this.notifyVideoOwnerOfNewComment(comment)
43 .catch(err => logger.error('Cannot notify video owner of new comment %s.', comment.url, { err }))
44
45 this.notifyOfCommentMention(comment)
46 .catch(err => logger.error('Cannot notify mentions of comment %s.', comment.url, { err }))
47 }
48
49 notifyOnNewVideoAbuse (videoAbuse: VideoAbuseModel): void {
50 this.notifyModeratorsOfNewVideoAbuse(videoAbuse)
51 .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err }))
52 }
53
54 notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void {
55 this.notifyVideoOwnerOfBlacklist(videoBlacklist)
56 .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err }))
57 }
58
59 notifyOnVideoUnblacklist (video: VideoModel): void {
60 this.notifyVideoOwnerOfUnblacklist(video)
61 .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err }))
62 }
63
64 notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void {
65 this.notifyOwnerVideoImportIsFinished(videoImport, success)
66 .catch(err => logger.error('Cannot notify owner that its video import %s is finished.', videoImport.getTargetIdentifier(), { err }))
67 }
68
69 notifyOnNewUserRegistration (user: UserModel): void {
70 this.notifyModeratorsOfNewUserRegistration(user)
71 .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err }))
72 }
73
74 notifyOfNewFollow (actorFollow: ActorFollowModel): void {
75 this.notifyUserOfNewActorFollow(actorFollow)
76 .catch(err => {
77 logger.error(
78 'Cannot notify owner of channel %s of a new follow by %s.',
79 actorFollow.ActorFollowing.VideoChannel.getDisplayName(),
80 actorFollow.ActorFollower.Account.getDisplayName(),
81 err
82 )
83 })
84 }
85
86 private async notifySubscribersOfNewVideo (video: VideoModel) {
87 // List all followers that are users
88 const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId)
89
90 logger.info('Notifying %d users of new video %s.', users.length, video.url)
91
92 function settingGetter (user: UserModel) {
93 return user.NotificationSetting.newVideoFromSubscription
94 }
95
96 async function notificationCreator (user: UserModel) {
97 const notification = await UserNotificationModel.create({
98 type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION,
99 userId: user.id,
100 videoId: video.id
101 })
102 notification.Video = video
103
104 return notification
105 }
106
107 function emailSender (emails: string[]) {
108 return Emailer.Instance.addNewVideoFromSubscriberNotification(emails, video)
109 }
110
111 return this.notify({ users, settingGetter, notificationCreator, emailSender })
112 }
113
114 private async notifyVideoOwnerOfNewComment (comment: VideoCommentModel) {
115 if (comment.Video.isOwned() === false) return
116
117 const user = await UserModel.loadByVideoId(comment.videoId)
118
119 // Not our user or user comments its own video
120 if (!user || comment.Account.userId === user.id) return
121
122 const accountMuted = await AccountBlocklistModel.isAccountMutedBy(user.Account.id, comment.accountId)
123 if (accountMuted) return
124
125 logger.info('Notifying user %s of new comment %s.', user.username, comment.url)
126
127 function settingGetter (user: UserModel) {
128 return user.NotificationSetting.newCommentOnMyVideo
129 }
130
131 async function notificationCreator (user: UserModel) {
132 const notification = await UserNotificationModel.create({
133 type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO,
134 userId: user.id,
135 commentId: comment.id
136 })
137 notification.Comment = comment
138
139 return notification
140 }
141
142 function emailSender (emails: string[]) {
143 return Emailer.Instance.addNewCommentOnMyVideoNotification(emails, comment)
144 }
145
146 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
147 }
148
149 private async notifyOfCommentMention (comment: VideoCommentModel) {
150 const usernames = comment.extractMentions()
151 let users = await UserModel.listByUsernames(usernames)
152
153 if (comment.Video.isOwned()) {
154 const userException = await UserModel.loadByVideoId(comment.videoId)
155 users = users.filter(u => u.id !== userException.id)
156 }
157
158 // Don't notify if I mentioned myself
159 users = users.filter(u => u.Account.id !== comment.accountId)
160
161 if (users.length === 0) return
162
163 const accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(users.map(u => u.Account.id), comment.accountId)
164
165 logger.info('Notifying %d users of new comment %s.', users.length, comment.url)
166
167 function settingGetter (user: UserModel) {
168 if (accountMutedHash[user.Account.id] === true) return UserNotificationSettingValue.NONE
169
170 return user.NotificationSetting.commentMention
171 }
172
173 async function notificationCreator (user: UserModel) {
174 const notification = await UserNotificationModel.create({
175 type: UserNotificationType.COMMENT_MENTION,
176 userId: user.id,
177 commentId: comment.id
178 })
179 notification.Comment = comment
180
181 return notification
182 }
183
184 function emailSender (emails: string[]) {
185 return Emailer.Instance.addNewCommentMentionNotification(emails, comment)
186 }
187
188 return this.notify({ users, settingGetter, notificationCreator, emailSender })
189 }
190
191 private async notifyUserOfNewActorFollow (actorFollow: ActorFollowModel) {
192 if (actorFollow.ActorFollowing.isOwned() === false) return
193
194 // Account follows one of our account?
195 let followType: 'account' | 'channel' = 'channel'
196 let user = await UserModel.loadByChannelActorId(actorFollow.ActorFollowing.id)
197
198 // Account follows one of our channel?
199 if (!user) {
200 user = await UserModel.loadByAccountActorId(actorFollow.ActorFollowing.id)
201 followType = 'account'
202 }
203
204 if (!user) return
205
206 if (!actorFollow.ActorFollower.Account || !actorFollow.ActorFollower.Account.name) {
207 actorFollow.ActorFollower.Account = await actorFollow.ActorFollower.$get('Account') as AccountModel
208 }
209 const followerAccount = actorFollow.ActorFollower.Account
210
211 const accountMuted = await AccountBlocklistModel.isAccountMutedBy(user.Account.id, followerAccount.id)
212 if (accountMuted) return
213
214 logger.info('Notifying user %s of new follower: %s.', user.username, followerAccount.getDisplayName())
215
216 function settingGetter (user: UserModel) {
217 return user.NotificationSetting.newFollow
218 }
219
220 async function notificationCreator (user: UserModel) {
221 const notification = await UserNotificationModel.create({
222 type: UserNotificationType.NEW_FOLLOW,
223 userId: user.id,
224 actorFollowId: actorFollow.id
225 })
226 notification.ActorFollow = actorFollow
227
228 return notification
229 }
230
231 function emailSender (emails: string[]) {
232 return Emailer.Instance.addNewFollowNotification(emails, actorFollow, followType)
233 }
234
235 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
236 }
237
238 private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) {
239 const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES)
240 if (moderators.length === 0) return
241
242 logger.info('Notifying %s user/moderators of new video abuse %s.', moderators.length, videoAbuse.Video.url)
243
244 function settingGetter (user: UserModel) {
245 return user.NotificationSetting.videoAbuseAsModerator
246 }
247
248 async function notificationCreator (user: UserModel) {
249 const notification = await UserNotificationModel.create({
250 type: UserNotificationType.NEW_VIDEO_ABUSE_FOR_MODERATORS,
251 userId: user.id,
252 videoAbuseId: videoAbuse.id
253 })
254 notification.VideoAbuse = videoAbuse
255
256 return notification
257 }
258
259 function emailSender (emails: string[]) {
260 return Emailer.Instance.addVideoAbuseModeratorsNotification(emails, videoAbuse)
261 }
262
263 return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender })
264 }
265
266 private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) {
267 const user = await UserModel.loadByVideoId(videoBlacklist.videoId)
268 if (!user) return
269
270 logger.info('Notifying user %s that its video %s has been blacklisted.', user.username, videoBlacklist.Video.url)
271
272 function settingGetter (user: UserModel) {
273 return user.NotificationSetting.blacklistOnMyVideo
274 }
275
276 async function notificationCreator (user: UserModel) {
277 const notification = await UserNotificationModel.create({
278 type: UserNotificationType.BLACKLIST_ON_MY_VIDEO,
279 userId: user.id,
280 videoBlacklistId: videoBlacklist.id
281 })
282 notification.VideoBlacklist = videoBlacklist
283
284 return notification
285 }
286
287 function emailSender (emails: string[]) {
288 return Emailer.Instance.addVideoBlacklistNotification(emails, videoBlacklist)
289 }
290
291 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
292 }
293
294 private async notifyVideoOwnerOfUnblacklist (video: VideoModel) {
295 const user = await UserModel.loadByVideoId(video.id)
296 if (!user) return
297
298 logger.info('Notifying user %s that its video %s has been unblacklisted.', user.username, video.url)
299
300 function settingGetter (user: UserModel) {
301 return user.NotificationSetting.blacklistOnMyVideo
302 }
303
304 async function notificationCreator (user: UserModel) {
305 const notification = await UserNotificationModel.create({
306 type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO,
307 userId: user.id,
308 videoId: video.id
309 })
310 notification.Video = video
311
312 return notification
313 }
314
315 function emailSender (emails: string[]) {
316 return Emailer.Instance.addVideoUnblacklistNotification(emails, video)
317 }
318
319 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
320 }
321
322 private async notifyOwnedVideoHasBeenPublished (video: VideoModel) {
323 const user = await UserModel.loadByVideoId(video.id)
324 if (!user) return
325
326 logger.info('Notifying user %s of the publication of its video %s.', user.username, video.url)
327
328 function settingGetter (user: UserModel) {
329 return user.NotificationSetting.myVideoPublished
330 }
331
332 async function notificationCreator (user: UserModel) {
333 const notification = await UserNotificationModel.create({
334 type: UserNotificationType.MY_VIDEO_PUBLISHED,
335 userId: user.id,
336 videoId: video.id
337 })
338 notification.Video = video
339
340 return notification
341 }
342
343 function emailSender (emails: string[]) {
344 return Emailer.Instance.myVideoPublishedNotification(emails, video)
345 }
346
347 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
348 }
349
350 private async notifyOwnerVideoImportIsFinished (videoImport: VideoImportModel, success: boolean) {
351 const user = await UserModel.loadByVideoImportId(videoImport.id)
352 if (!user) return
353
354 logger.info('Notifying user %s its video import %s is finished.', user.username, videoImport.getTargetIdentifier())
355
356 function settingGetter (user: UserModel) {
357 return user.NotificationSetting.myVideoImportFinished
358 }
359
360 async function notificationCreator (user: UserModel) {
361 const notification = await UserNotificationModel.create({
362 type: success ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS : UserNotificationType.MY_VIDEO_IMPORT_ERROR,
363 userId: user.id,
364 videoImportId: videoImport.id
365 })
366 notification.VideoImport = videoImport
367
368 return notification
369 }
370
371 function emailSender (emails: string[]) {
372 return success
373 ? Emailer.Instance.myVideoImportSuccessNotification(emails, videoImport)
374 : Emailer.Instance.myVideoImportErrorNotification(emails, videoImport)
375 }
376
377 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
378 }
379
380 private async notifyModeratorsOfNewUserRegistration (registeredUser: UserModel) {
381 const moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS)
382 if (moderators.length === 0) return
383
384 logger.info(
385 'Notifying %s moderators of new user registration of %s.',
386 moderators.length, registeredUser.Account.Actor.preferredUsername
387 )
388
389 function settingGetter (user: UserModel) {
390 return user.NotificationSetting.newUserRegistration
391 }
392
393 async function notificationCreator (user: UserModel) {
394 const notification = await UserNotificationModel.create({
395 type: UserNotificationType.NEW_USER_REGISTRATION,
396 userId: user.id,
397 accountId: registeredUser.Account.id
398 })
399 notification.Account = registeredUser.Account
400
401 return notification
402 }
403
404 function emailSender (emails: string[]) {
405 return Emailer.Instance.addNewUserRegistrationNotification(emails, registeredUser)
406 }
407
408 return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender })
409 }
410
411 private async notify (options: {
412 users: UserModel[],
413 notificationCreator: (user: UserModel) => Promise<UserNotificationModel>,
414 emailSender: (emails: string[]) => Promise<any> | Bluebird<any>,
415 settingGetter: (user: UserModel) => UserNotificationSettingValue
416 }) {
417 const emails: string[] = []
418
419 for (const user of options.users) {
420 if (this.isWebNotificationEnabled(options.settingGetter(user))) {
421 const notification = await options.notificationCreator(user)
422
423 PeerTubeSocket.Instance.sendNotification(user.id, notification)
424 }
425
426 if (this.isEmailEnabled(user, options.settingGetter(user))) {
427 emails.push(user.email)
428 }
429 }
430
431 if (emails.length !== 0) {
432 await options.emailSender(emails)
433 }
434 }
435
436 private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) {
437 if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified !== true) return false
438
439 return value & UserNotificationSettingValue.EMAIL
440 }
441
442 private isWebNotificationEnabled (value: UserNotificationSettingValue) {
443 return value & UserNotificationSettingValue.WEB
444 }
445
446 static get Instance () {
447 return this.instance || (this.instance = new this())
448 }
449}
450
451// ---------------------------------------------------------------------------
452
453export {
454 Notifier
455}
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 5cbe60b82..2cd2ae97c 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -1,3 +1,4 @@
1import * as Bluebird from 'bluebird'
1import { AccessDeniedError } from 'oauth2-server' 2import { AccessDeniedError } from 'oauth2-server'
2import { logger } from '../helpers/logger' 3import { logger } from '../helpers/logger'
3import { UserModel } from '../models/account/user' 4import { UserModel } from '../models/account/user'
@@ -37,7 +38,7 @@ function clearCacheByToken (token: string) {
37function getAccessToken (bearerToken: string) { 38function getAccessToken (bearerToken: string) {
38 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') 39 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').')
39 40
40 if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken] 41 if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken])
41 42
42 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 43 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
43 .then(tokenModel => { 44 .then(tokenModel => {
diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts
new file mode 100644
index 000000000..eb84ecd4b
--- /dev/null
+++ b/server/lib/peertube-socket.ts
@@ -0,0 +1,52 @@
1import * as SocketIO from 'socket.io'
2import { authenticateSocket } from '../middlewares'
3import { UserNotificationModel } from '../models/account/user-notification'
4import { logger } from '../helpers/logger'
5import { Server } from 'http'
6
7class PeerTubeSocket {
8
9 private static instance: PeerTubeSocket
10
11 private userNotificationSockets: { [ userId: number ]: SocketIO.Socket } = {}
12
13 private constructor () {}
14
15 init (server: Server) {
16 const io = SocketIO(server)
17
18 io.of('/user-notifications')
19 .use(authenticateSocket)
20 .on('connection', socket => {
21 const userId = socket.handshake.query.user.id
22
23 logger.debug('User %d connected on the notification system.', userId)
24
25 this.userNotificationSockets[userId] = socket
26
27 socket.on('disconnect', () => {
28 logger.debug('User %d disconnected from SocketIO notifications.', userId)
29
30 delete this.userNotificationSockets[userId]
31 })
32 })
33 }
34
35 sendNotification (userId: number, notification: UserNotificationModel) {
36 const socket = this.userNotificationSockets[userId]
37
38 if (!socket) return
39
40 socket.emit('new-notification', notification.toFormattedJSON())
41 }
42
43 static get Instance () {
44 return this.instance || (this.instance = new this())
45 }
46}
47
48// ---------------------------------------------------------------------------
49
50export {
51 PeerTubeSocket
52}
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index abd75d512..3628c0583 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -2,7 +2,13 @@ import * as express from 'express'
2import { createClient, RedisClient } from 'redis' 2import { createClient, RedisClient } from 'redis'
3import { logger } from '../helpers/logger' 3import { logger } from '../helpers/logger'
4import { generateRandomString } from '../helpers/utils' 4import { generateRandomString } from '../helpers/utils'
5import { CONFIG, USER_PASSWORD_RESET_LIFETIME, USER_EMAIL_VERIFY_LIFETIME, VIDEO_VIEW_LIFETIME } from '../initializers' 5import {
6 CONFIG,
7 CONTACT_FORM_LIFETIME,
8 USER_EMAIL_VERIFY_LIFETIME,
9 USER_PASSWORD_RESET_LIFETIME,
10 VIDEO_VIEW_LIFETIME
11} from '../initializers'
6 12
7type CachedRoute = { 13type CachedRoute = {
8 body: string, 14 body: string,
@@ -76,6 +82,16 @@ class Redis {
76 return this.getValue(this.generateVerifyEmailKey(userId)) 82 return this.getValue(this.generateVerifyEmailKey(userId))
77 } 83 }
78 84
85 /************* Contact form per IP *************/
86
87 async setContactFormIp (ip: string) {
88 return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME)
89 }
90
91 async isContactFormIpExists (ip: string) {
92 return this.exists(this.generateContactFormKey(ip))
93 }
94
79 /************* Views per IP *************/ 95 /************* Views per IP *************/
80 96
81 setIPVideoView (ip: string, videoUUID: string) { 97 setIPVideoView (ip: string, videoUUID: string) {
@@ -121,7 +137,14 @@ class Redis {
121 const key = this.generateVideoViewKey(videoId, hour) 137 const key = this.generateVideoViewKey(videoId, hour)
122 138
123 const valueString = await this.getValue(key) 139 const valueString = await this.getValue(key)
124 return parseInt(valueString, 10) 140 const valueInt = parseInt(valueString, 10)
141
142 if (isNaN(valueInt)) {
143 logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
144 return undefined
145 }
146
147 return valueInt
125 } 148 }
126 149
127 async getVideosIdViewed (hour: number) { 150 async getVideosIdViewed (hour: number) {
@@ -168,7 +191,11 @@ class Redis {
168 } 191 }
169 192
170 private generateViewKey (ip: string, videoUUID: string) { 193 private generateViewKey (ip: string, videoUUID: string) {
171 return videoUUID + '-' + ip 194 return `views-${videoUUID}-${ip}`
195 }
196
197 private generateContactFormKey (ip: string) {
198 return 'contact-form-' + ip
172 } 199 }
173 200
174 /************* Redis helpers *************/ 201 /************* Redis helpers *************/
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts
index b9d0a4d17..86ea7aa38 100644
--- a/server/lib/schedulers/abstract-scheduler.ts
+++ b/server/lib/schedulers/abstract-scheduler.ts
@@ -1,8 +1,11 @@
1import { logger } from '../../helpers/logger'
2
1export abstract class AbstractScheduler { 3export abstract class AbstractScheduler {
2 4
3 protected abstract schedulerIntervalMs: number 5 protected abstract schedulerIntervalMs: number
4 6
5 private interval: NodeJS.Timer 7 private interval: NodeJS.Timer
8 private isRunning = false
6 9
7 enable () { 10 enable () {
8 if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') 11 if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.')
@@ -14,5 +17,18 @@ export abstract class AbstractScheduler {
14 clearInterval(this.interval) 17 clearInterval(this.interval)
15 } 18 }
16 19
17 abstract execute () 20 async execute () {
21 if (this.isRunning === true) return
22 this.isRunning = true
23
24 try {
25 await this.internalExecute()
26 } catch (err) {
27 logger.error('Cannot execute %s scheduler.', this.constructor.name, { err })
28 } finally {
29 this.isRunning = false
30 }
31 }
32
33 protected abstract internalExecute (): Promise<any>
18} 34}
diff --git a/server/lib/schedulers/bad-actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts
index 617149aaf..3967be7f8 100644
--- a/server/lib/schedulers/bad-actor-follow-scheduler.ts
+++ b/server/lib/schedulers/actor-follow-scheduler.ts
@@ -3,18 +3,35 @@ import { logger } from '../../helpers/logger'
3import { ActorFollowModel } from '../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../models/activitypub/actor-follow'
4import { AbstractScheduler } from './abstract-scheduler' 4import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers'
6import { ActorFollowScoreCache } from '../cache'
6 7
7export class BadActorFollowScheduler extends AbstractScheduler { 8export class ActorFollowScheduler extends AbstractScheduler {
8 9
9 private static instance: AbstractScheduler 10 private static instance: AbstractScheduler
10 11
11 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.badActorFollow 12 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.actorFollowScores
12 13
13 private constructor () { 14 private constructor () {
14 super() 15 super()
15 } 16 }
16 17
17 async execute () { 18 protected async internalExecute () {
19 await this.processPendingScores()
20
21 await this.removeBadActorFollows()
22 }
23
24 private async processPendingScores () {
25 const pendingScores = ActorFollowScoreCache.Instance.getPendingFollowsScoreCopy()
26
27 ActorFollowScoreCache.Instance.clearPendingFollowsScore()
28
29 for (const inbox of Object.keys(pendingScores)) {
30 await ActorFollowModel.updateFollowScore(inbox, pendingScores[inbox])
31 }
32 }
33
34 private async removeBadActorFollows () {
18 if (!isTestInstance()) logger.info('Removing bad actor follows (scheduler).') 35 if (!isTestInstance()) logger.info('Removing bad actor follows (scheduler).')
19 36
20 try { 37 try {
diff --git a/server/lib/schedulers/remove-old-jobs-scheduler.ts b/server/lib/schedulers/remove-old-jobs-scheduler.ts
index a29a6b800..4a4341ba9 100644
--- a/server/lib/schedulers/remove-old-jobs-scheduler.ts
+++ b/server/lib/schedulers/remove-old-jobs-scheduler.ts
@@ -14,10 +14,10 @@ export class RemoveOldJobsScheduler extends AbstractScheduler {
14 super() 14 super()
15 } 15 }
16 16
17 async execute () { 17 protected internalExecute () {
18 if (!isTestInstance()) logger.info('Removing old jobs (scheduler).') 18 if (!isTestInstance()) logger.info('Removing old jobs in scheduler.')
19 19
20 JobQueue.Instance.removeOldJobs() 20 return JobQueue.Instance.removeOldJobs()
21 } 21 }
22 22
23 static get Instance () { 23 static get Instance () {
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index fd2edfd17..2618a5857 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -5,6 +5,8 @@ import { retryTransactionWrapper } from '../../helpers/database-utils'
5import { federateVideoIfNeeded } from '../activitypub' 5import { federateVideoIfNeeded } from '../activitypub'
6import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' 6import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers'
7import { VideoPrivacy } from '../../../shared/models/videos' 7import { VideoPrivacy } from '../../../shared/models/videos'
8import { Notifier } from '../notifier'
9import { VideoModel } from '../../models/video/video'
8 10
9export class UpdateVideosScheduler extends AbstractScheduler { 11export class UpdateVideosScheduler extends AbstractScheduler {
10 12
@@ -12,30 +14,20 @@ export class UpdateVideosScheduler extends AbstractScheduler {
12 14
13 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.updateVideos 15 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.updateVideos
14 16
15 private isRunning = false
16
17 private constructor () { 17 private constructor () {
18 super() 18 super()
19 } 19 }
20 20
21 async execute () { 21 protected async internalExecute () {
22 if (this.isRunning === true) return 22 return retryTransactionWrapper(this.updateVideos.bind(this))
23 this.isRunning = true
24
25 try {
26 await retryTransactionWrapper(this.updateVideos.bind(this))
27 } catch (err) {
28 logger.error('Cannot execute update videos scheduler.', { err })
29 } finally {
30 this.isRunning = false
31 }
32 } 23 }
33 24
34 private async updateVideos () { 25 private async updateVideos () {
35 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined 26 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
36 27
37 return sequelizeTypescript.transaction(async t => { 28 const publishedVideos = await sequelizeTypescript.transaction(async t => {
38 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) 29 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t)
30 const publishedVideos: VideoModel[] = []
39 31
40 for (const schedule of schedules) { 32 for (const schedule of schedules) {
41 const video = schedule.Video 33 const video = schedule.Video
@@ -50,11 +42,23 @@ export class UpdateVideosScheduler extends AbstractScheduler {
50 42
51 await video.save({ transaction: t }) 43 await video.save({ transaction: t })
52 await federateVideoIfNeeded(video, isNewVideo, t) 44 await federateVideoIfNeeded(video, isNewVideo, t)
45
46 if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) {
47 video.ScheduleVideoUpdate = schedule
48 publishedVideos.push(video)
49 }
53 } 50 }
54 51
55 await schedule.destroy({ transaction: t }) 52 await schedule.destroy({ transaction: t })
56 } 53 }
54
55 return publishedVideos
57 }) 56 })
57
58 for (const v of publishedVideos) {
59 Notifier.Instance.notifyOnNewVideo(v)
60 Notifier.Instance.notifyOnPendingVideoPublished(v)
61 }
58 } 62 }
59 63
60 static get Instance () { 64 static get Instance () {
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c49a8c89a..1a48f2bd0 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,22 +1,31 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' 2import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 7import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
8import { join } from 'path' 8import { join } from 'path'
9import { rename } from 'fs-extra' 9import { move } from 'fs-extra'
10import { getServerActor } from '../../helpers/utils' 10import { getServerActor } from '../../helpers/utils'
11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 12import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
13import { removeVideoRedundancy } from '../redundancy' 13import { removeVideoRedundancy } from '../redundancy'
14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { VideoModel } from '../../models/video/video'
17import { downloadPlaylistSegments } from '../hls'
18
19type CandidateToDuplicate = {
20 redundancy: VideosRedundancy,
21 video: VideoModel,
22 files: VideoFileModel[],
23 streamingPlaylists: VideoStreamingPlaylistModel[]
24}
15 25
16export class VideosRedundancyScheduler extends AbstractScheduler { 26export class VideosRedundancyScheduler extends AbstractScheduler {
17 27
18 private static instance: AbstractScheduler 28 private static instance: AbstractScheduler
19 private executing = false
20 29
21 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL 30 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
22 31
@@ -24,41 +33,39 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
24 super() 33 super()
25 } 34 }
26 35
27 async execute () { 36 protected async internalExecute () {
28 if (this.executing) return 37 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
29 38 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
30 this.executing = true
31
32 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
33 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
34 39
35 try { 40 try {
36 const videoToDuplicate = await this.findVideoToDuplicate(obj) 41 const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
37 if (!videoToDuplicate) continue 42 if (!videoToDuplicate) continue
38 43
39 const videoFiles = videoToDuplicate.VideoFiles 44 const candidateToDuplicate = {
40 videoFiles.forEach(f => f.Video = videoToDuplicate) 45 video: videoToDuplicate,
46 redundancy: redundancyConfig,
47 files: videoToDuplicate.VideoFiles,
48 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
49 }
41 50
42 await this.purgeCacheIfNeeded(obj, videoFiles) 51 await this.purgeCacheIfNeeded(candidateToDuplicate)
43 52
44 if (await this.isTooHeavy(obj, videoFiles)) { 53 if (await this.isTooHeavy(candidateToDuplicate)) {
45 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 54 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
46 continue 55 continue
47 } 56 }
48 57
49 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 58 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)
50 59
51 await this.createVideoRedundancy(obj, videoFiles) 60 await this.createVideoRedundancies(candidateToDuplicate)
52 } catch (err) { 61 } catch (err) {
53 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 62 logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
54 } 63 }
55 } 64 }
56 65
57 await this.extendsLocalExpiration() 66 await this.extendsLocalExpiration()
58 67
59 await this.purgeRemoteExpired() 68 await this.purgeRemoteExpired()
60
61 this.executing = false
62 } 69 }
63 70
64 static get Instance () { 71 static get Instance () {
@@ -70,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
70 77
71 for (const redundancyModel of expired) { 78 for (const redundancyModel of expired) {
72 try { 79 try {
73 await this.extendsOrDeleteRedundancy(redundancyModel) 80 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
81 const candidate = {
82 redundancy: redundancyConfig,
83 video: null,
84 files: [],
85 streamingPlaylists: []
86 }
87
88 // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
89 if (!redundancyConfig || await this.isTooHeavy(candidate)) {
90 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
91 await removeVideoRedundancy(redundancyModel)
92 } else {
93 await this.extendsRedundancy(redundancyModel)
94 }
74 } catch (err) { 95 } catch (err) {
75 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) 96 logger.error(
97 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
98 { err }
99 )
76 } 100 }
77 } 101 }
78 } 102 }
79 103
80 private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { 104 private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
81 // Refresh the video, maybe it was deleted
82 const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
83
84 if (!video) {
85 logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
86
87 await redundancyModel.destroy()
88 return
89 }
90
91 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 105 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
106 // Redundancy strategy disabled, remove our redundancy instead of extending expiration
107 if (!redundancy) await removeVideoRedundancy(redundancyModel)
108
92 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) 109 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
93 } 110 }
94 111
@@ -119,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
119 } 136 }
120 } 137 }
121 138
122 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 139 private async createVideoRedundancies (data: CandidateToDuplicate) {
123 const serverActor = await getServerActor() 140 const video = await this.loadAndRefreshVideo(data.video.url)
124 141
125 for (const file of filesToDuplicate) { 142 if (!video) {
126 const video = await this.loadAndRefreshVideo(file.Video.url) 143 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)
127 144
145 return
146 }
147
148 for (const file of data.files) {
128 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) 149 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
129 if (existingRedundancy) { 150 if (existingRedundancy) {
130 await this.extendsOrDeleteRedundancy(existingRedundancy) 151 await this.extendsRedundancy(existingRedundancy)
131 152
132 continue 153 continue
133 } 154 }
134 155
135 if (!video) { 156 await this.createVideoFileRedundancy(data.redundancy, video, file)
136 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) 157 }
158
159 for (const streamingPlaylist of data.streamingPlaylists) {
160 const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
161 if (existingRedundancy) {
162 await this.extendsRedundancy(existingRedundancy)
137 163
138 continue 164 continue
139 } 165 }
140 166
141 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 167 await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
168 }
169 }
142 170
143 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 171 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
144 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 172 file.Video = video
145 173
146 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) 174 const serverActor = await getServerActor()
147 175
148 const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) 176 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
149 await rename(tmpPath, destPath)
150 177
151 const createdModel = await VideoRedundancyModel.create({ 178 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
152 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 179 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
153 url: getVideoCacheFileActivityPubUrl(file),
154 fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
155 strategy: redundancy.strategy,
156 videoFileId: file.id,
157 actorId: serverActor.id
158 })
159 createdModel.VideoFile = file
160 180
161 await sendCreateCacheFile(serverActor, createdModel) 181 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
162 182
163 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) 183 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
164 } 184 await move(tmpPath, destPath)
185
186 const createdModel = await VideoRedundancyModel.create({
187 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
188 url: getVideoCacheFileActivityPubUrl(file),
189 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
190 strategy: redundancy.strategy,
191 videoFileId: file.id,
192 actorId: serverActor.id
193 })
194
195 createdModel.VideoFile = file
196
197 await sendCreateCacheFile(serverActor, video, createdModel)
198
199 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
200 }
201
202 private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
203 playlist.Video = video
204
205 const serverActor = await getServerActor()
206
207 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
208
209 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
210 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
211
212 const createdModel = await VideoRedundancyModel.create({
213 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
214 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
215 fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL),
216 strategy: redundancy.strategy,
217 videoStreamingPlaylistId: playlist.id,
218 actorId: serverActor.id
219 })
220
221 createdModel.VideoStreamingPlaylist = playlist
222
223 await sendCreateCacheFile(serverActor, video, createdModel)
224
225 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
165 } 226 }
166 227
167 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { 228 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@@ -175,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
175 await sendUpdateCacheFile(serverActor, redundancy) 236 await sendUpdateCacheFile(serverActor, redundancy)
176 } 237 }
177 238
178 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 239 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
179 while (this.isTooHeavy(redundancy, filesToDuplicate)) { 240 while (this.isTooHeavy(candidateToDuplicate)) {
241 const redundancy = candidateToDuplicate.redundancy
180 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) 242 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
181 if (!toDelete) return 243 if (!toDelete) return
182 244
@@ -184,12 +246,13 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
184 } 246 }
185 } 247 }
186 248
187 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 249 private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
188 const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate) 250 const maxSize = candidateToDuplicate.redundancy.size
189 251
190 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) 252 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
253 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
191 254
192 return totalDuplicated > maxSize 255 return totalWillDuplicate > maxSize
193 } 256 }
194 257
195 private buildNewExpiration (expiresAfterMs: number) { 258 private buildNewExpiration (expiresAfterMs: number) {
@@ -197,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
197 } 260 }
198 261
199 private buildEntryLogId (object: VideoRedundancyModel) { 262 private buildEntryLogId (object: VideoRedundancyModel) {
200 return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 263 if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
264
265 return `${object.VideoStreamingPlaylist.playlistUrl}`
201 } 266 }
202 267
203 private getTotalFileSizes (files: VideoFileModel[]) { 268 private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
204 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size 269 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
205 270
206 return files.reduce(fileReducer, 0) 271 return files.reduce(fileReducer, 0) * playlists.length
207 } 272 }
208 273
209 private async loadAndRefreshVideo (videoUrl: string) { 274 private async loadAndRefreshVideo (videoUrl: string) {
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
index 461cd045e..aa027116d 100644
--- a/server/lib/schedulers/youtube-dl-update-scheduler.ts
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -12,7 +12,7 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler {
12 super() 12 super()
13 } 13 }
14 14
15 execute () { 15 protected internalExecute () {
16 return updateYoutubeDLBinary() 16 return updateYoutubeDLBinary()
17 } 17 }
18 18
diff --git a/server/lib/user.ts b/server/lib/user.ts
index db29469eb..a39ef6c3d 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -9,6 +9,8 @@ import { createVideoChannel } from './video-channel'
9import { VideoChannelModel } from '../models/video/video-channel' 9import { VideoChannelModel } from '../models/video/video-channel'
10import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' 10import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
11import { ActorModel } from '../models/activitypub/actor' 11import { ActorModel } from '../models/activitypub/actor'
12import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
13import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users'
12 14
13async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { 15async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) {
14 const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { 16 const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => {
@@ -18,7 +20,9 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse
18 } 20 }
19 21
20 const userCreated = await userToCreate.save(userOptions) 22 const userCreated = await userToCreate.save(userOptions)
21 const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) 23 userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t)
24
25 const accountCreated = await createLocalAccountWithoutKeys(userCreated.username, userCreated.id, null, t)
22 userCreated.Account = accountCreated 26 userCreated.Account = accountCreated
23 27
24 let channelName = userCreated.username + '_channel' 28 let channelName = userCreated.username + '_channel'
@@ -37,8 +41,13 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse
37 return { user: userCreated, account: accountCreated, videoChannel } 41 return { user: userCreated, account: accountCreated, videoChannel }
38 }) 42 })
39 43
40 account.Actor = await setAsyncActorKeys(account.Actor) 44 const [ accountKeys, channelKeys ] = await Promise.all([
41 videoChannel.Actor = await setAsyncActorKeys(videoChannel.Actor) 45 setAsyncActorKeys(account.Actor),
46 setAsyncActorKeys(videoChannel.Actor)
47 ])
48
49 account.Actor = accountKeys
50 videoChannel.Actor = channelKeys
42 51
43 return { user, account, videoChannel } as { user: UserModel, account: AccountModel, videoChannel: VideoChannelModel } 52 return { user, account, videoChannel } as { user: UserModel, account: AccountModel, videoChannel: VideoChannelModel }
44} 53}
@@ -83,3 +92,22 @@ export {
83 createUserAccountAndChannel, 92 createUserAccountAndChannel,
84 createLocalAccountWithoutKeys 93 createLocalAccountWithoutKeys
85} 94}
95
96// ---------------------------------------------------------------------------
97
98function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Transaction | undefined) {
99 const values: UserNotificationSetting & { userId: number } = {
100 userId: user.id,
101 newVideoFromSubscription: UserNotificationSettingValue.WEB,
102 newCommentOnMyVideo: UserNotificationSettingValue.WEB,
103 myVideoImportFinished: UserNotificationSettingValue.WEB,
104 myVideoPublished: UserNotificationSettingValue.WEB,
105 videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL,
106 blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL,
107 newUserRegistration: UserNotificationSettingValue.WEB,
108 commentMention: UserNotificationSettingValue.WEB,
109 newFollow: UserNotificationSettingValue.WEB
110 }
111
112 return UserNotificationSettingModel.create(values, { transaction: t })
113}
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts
index 70ba7c303..59bce7520 100644
--- a/server/lib/video-comment.ts
+++ b/server/lib/video-comment.ts
@@ -64,10 +64,8 @@ function buildFormattedCommentTree (resultList: ResultList<VideoCommentModel>):
64 } 64 }
65 65
66 const parentCommentThread = idx[childComment.inReplyToCommentId] 66 const parentCommentThread = idx[childComment.inReplyToCommentId]
67 if (!parentCommentThread) { 67 // Maybe the parent comment was blocked by the admin/user
68 const msg = `Cannot format video thread tree, parent ${childComment.inReplyToCommentId} not found for child ${childComment.id}` 68 if (!parentCommentThread) continue
69 throw new Error(msg)
70 }
71 69
72 parentCommentThread.children.push(childCommentThread) 70 parentCommentThread.children.push(childCommentThread)
73 idx[childComment.id] = childCommentThread 71 idx[childComment.id] = childCommentThread
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index bf3ff78c2..086b860a2 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,22 +1,27 @@
1import { CONFIG } from '../initializers' 1import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
2import { join, extname } from 'path' 2import { extname, join } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' 3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, rename, stat } from 'fs-extra' 4import { copy, ensureDir, move, remove, stat } from 'fs-extra'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos' 6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file' 7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video' 8import { VideoModel } from '../models/video/video'
9import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
11import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
9 12
10async function optimizeOriginalVideofile (video: VideoModel) { 13async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 14 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
12 const newExtname = '.mp4' 15 const newExtname = '.mp4'
13 const inputVideoFile = video.getOriginalFile() 16
17 const inputVideoFile = inputVideoFileArg ? inputVideoFileArg : video.getOriginalFile()
14 const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) 18 const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile))
15 const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) 19 const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname)
16 20
17 const transcodeOptions = { 21 const transcodeOptions = {
18 inputPath: videoInputPath, 22 inputPath: videoInputPath,
19 outputPath: videoTranscodedPath 23 outputPath: videoTranscodedPath,
24 resolution: inputVideoFile.resolution
20 } 25 }
21 26
22 // Could be very long! 27 // Could be very long!
@@ -29,7 +34,7 @@ async function optimizeOriginalVideofile (video: VideoModel) {
29 inputVideoFile.set('extname', newExtname) 34 inputVideoFile.set('extname', newExtname)
30 35
31 const videoOutputPath = video.getVideoFilePath(inputVideoFile) 36 const videoOutputPath = video.getVideoFilePath(inputVideoFile)
32 await rename(videoTranscodedPath, videoOutputPath) 37 await move(videoTranscodedPath, videoOutputPath)
33 const stats = await stat(videoOutputPath) 38 const stats = await stat(videoOutputPath)
34 const fps = await getVideoFileFPS(videoOutputPath) 39 const fps = await getVideoFileFPS(videoOutputPath)
35 40
@@ -46,7 +51,7 @@ async function optimizeOriginalVideofile (video: VideoModel) {
46 } 51 }
47} 52}
48 53
49async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { 54async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
50 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 55 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
51 const extname = '.mp4' 56 const extname = '.mp4'
52 57
@@ -59,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
59 size: 0, 64 size: 0,
60 videoId: video.id 65 videoId: video.id
61 }) 66 })
62 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) 67 const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
63 68
64 const transcodeOptions = { 69 const transcodeOptions = {
65 inputPath: videoInputPath, 70 inputPath: videoInputPath,
66 outputPath: videoOutputPath, 71 outputPath: videoOutputPath,
67 resolution, 72 resolution,
68 isPortraitMode 73 isPortraitMode: isPortrait
69 } 74 }
70 75
71 await transcode(transcodeOptions) 76 await transcode(transcodeOptions)
@@ -83,6 +88,41 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
83 video.VideoFiles.push(newVideoFile) 88 video.VideoFiles.push(newVideoFile)
84} 89}
85 90
91async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
92 const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
93 await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid))
94
95 const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
96 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
97
98 const transcodeOptions = {
99 inputPath: videoInputPath,
100 outputPath,
101 resolution,
102 isPortraitMode,
103
104 hlsPlaylist: {
105 videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution)
106 }
107 }
108
109 await transcode(transcodeOptions)
110
111 await updateMasterHLSPlaylist(video)
112 await updateSha256Segments(video)
113
114 const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
115
116 await VideoStreamingPlaylistModel.upsert({
117 videoId: video.id,
118 playlistUrl,
119 segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
120 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
121
122 type: VideoStreamingPlaylistType.HLS
123 })
124}
125
86async function importVideoFile (video: VideoModel, inputFilePath: string) { 126async function importVideoFile (video: VideoModel, inputFilePath: string) {
87 const { videoFileResolution } = await getVideoFileResolution(inputFilePath) 127 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
88 const { size } = await stat(inputFilePath) 128 const { size } = await stat(inputFilePath)
@@ -124,7 +164,8 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) {
124} 164}
125 165
126export { 166export {
127 optimizeOriginalVideofile, 167 generateHlsPlaylist,
168 optimizeVideofile,
128 transcodeOriginalVideofile, 169 transcodeOriginalVideofile,
129 importVideoFile 170 importVideoFile
130} 171}