aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts30
-rw-r--r--server/lib/activitypub/audience.ts2
-rw-r--r--server/lib/activitypub/cache-file.ts25
-rw-r--r--server/lib/activitypub/crawl.ts16
-rw-r--r--server/lib/activitypub/index.ts1
-rw-r--r--server/lib/activitypub/playlist.ts213
-rw-r--r--server/lib/activitypub/process/process-create.ts19
-rw-r--r--server/lib/activitypub/process/process-delete.ts24
-rw-r--r--server/lib/activitypub/process/process-follow.ts36
-rw-r--r--server/lib/activitypub/process/process-undo.ts5
-rw-r--r--server/lib/activitypub/process/process-update.ts23
-rw-r--r--server/lib/activitypub/send/index.ts2
-rw-r--r--server/lib/activitypub/send/send-accept.ts2
-rw-r--r--server/lib/activitypub/send/send-create.ts99
-rw-r--r--server/lib/activitypub/send/send-delete.ts28
-rw-r--r--server/lib/activitypub/send/send-dislike.ts41
-rw-r--r--server/lib/activitypub/send/send-flag.ts39
-rw-r--r--server/lib/activitypub/send/send-follow.ts2
-rw-r--r--server/lib/activitypub/send/send-reject.ts40
-rw-r--r--server/lib/activitypub/send/send-undo.ts17
-rw-r--r--server/lib/activitypub/send/send-update.ts34
-rw-r--r--server/lib/activitypub/send/send-view.ts40
-rw-r--r--server/lib/activitypub/share.ts9
-rw-r--r--server/lib/activitypub/url.ts43
-rw-r--r--server/lib/activitypub/video-comments.ts12
-rw-r--r--server/lib/activitypub/video-rates.ts33
-rw-r--r--server/lib/activitypub/videos.ts212
-rw-r--r--server/lib/avatar.ts6
-rw-r--r--server/lib/cache/abstract-video-static-file-cache.ts52
-rw-r--r--server/lib/client-html.ts80
-rw-r--r--server/lib/emailer.ts94
-rw-r--r--server/lib/files-cache/abstract-video-static-file-cache.ts30
-rw-r--r--server/lib/files-cache/actor-follow-score-cache.ts (renamed from server/lib/cache/actor-follow-score-cache.ts)2
-rw-r--r--server/lib/files-cache/index.ts (renamed from server/lib/cache/index.ts)0
-rw-r--r--server/lib/files-cache/videos-caption-cache.ts (renamed from server/lib/cache/videos-caption-cache.ts)20
-rw-r--r--server/lib/files-cache/videos-preview-cache.ts (renamed from server/lib/cache/videos-preview-cache.ts)19
-rw-r--r--server/lib/hls.ts184
-rw-r--r--server/lib/job-queue/handlers/activitypub-follow.ts7
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-broadcast.ts5
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts24
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-unicast.ts4
-rw-r--r--server/lib/job-queue/handlers/activitypub-refresher.ts14
-rw-r--r--server/lib/job-queue/handlers/email.ts11
-rw-r--r--server/lib/job-queue/handlers/utils/activitypub-http-utils.ts4
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts78
-rw-r--r--server/lib/job-queue/handlers/video-import.ts48
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts (renamed from server/lib/job-queue/handlers/video-file.ts)98
-rw-r--r--server/lib/job-queue/job-queue.ts18
-rw-r--r--server/lib/notifier.ts114
-rw-r--r--server/lib/oauth-model.ts15
-rw-r--r--server/lib/redis.ts25
-rw-r--r--server/lib/schedulers/abstract-scheduler.ts3
-rw-r--r--server/lib/schedulers/actor-follow-scheduler.ts4
-rw-r--r--server/lib/schedulers/remove-old-history-scheduler.ts32
-rw-r--r--server/lib/schedulers/remove-old-jobs-scheduler.ts2
-rw-r--r--server/lib/schedulers/remove-old-views-scheduler.ts33
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts5
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts190
-rw-r--r--server/lib/schedulers/youtube-dl-update-scheduler.ts2
-rw-r--r--server/lib/thumbnail.ts151
-rw-r--r--server/lib/user.ts19
-rw-r--r--server/lib/video-blacklist.ts33
-rw-r--r--server/lib/video-comment.ts2
-rw-r--r--server/lib/video-playlist.ts29
-rw-r--r--server/lib/video-transcoding.ts81
65 files changed, 2039 insertions, 546 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 8215840da..25cd40905 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -12,7 +12,7 @@ import { logger } from '../../helpers/logger'
12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' 12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13import { doRequest, downloadImage } from '../../helpers/requests' 13import { doRequest, downloadImage } from '../../helpers/requests'
14import { getUrlFromWebfinger } from '../../helpers/webfinger' 14import { getUrlFromWebfinger } from '../../helpers/webfinger'
15import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers' 15import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants'
16import { AccountModel } from '../../models/account/account' 16import { AccountModel } from '../../models/account/account'
17import { ActorModel } from '../../models/activitypub/actor' 17import { ActorModel } from '../../models/activitypub/actor'
18import { AvatarModel } from '../../models/avatar/avatar' 18import { AvatarModel } from '../../models/avatar/avatar'
@@ -21,6 +21,8 @@ import { VideoChannelModel } from '../../models/video/video-channel'
21import { JobQueue } from '../job-queue' 21import { JobQueue } from '../job-queue'
22import { getServerActor } from '../../helpers/utils' 22import { getServerActor } from '../../helpers/utils'
23import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' 23import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24import { CONFIG } from '../../initializers/config'
25import { sequelizeTypescript } from '../../initializers/database'
24 26
25// Set account keys, this could be long so process after the account creation and do not block the client 27// Set account keys, this could be long so process after the account creation and do not block the client
26function setAsyncActorKeys (actor: ActorModel) { 28function setAsyncActorKeys (actor: ActorModel) {
@@ -44,6 +46,7 @@ async function getOrCreateActorAndServerAndModel (
44) { 46) {
45 const actorUrl = getAPId(activityActor) 47 const actorUrl = getAPId(activityActor)
46 let created = false 48 let created = false
49 let accountPlaylistsUrl: string
47 50
48 let actor = await fetchActorByUrl(actorUrl, fetchType) 51 let actor = await fetchActorByUrl(actorUrl, fetchType)
49 // Orphan actor (not associated to an account of channel) so recreate it 52 // Orphan actor (not associated to an account of channel) so recreate it
@@ -70,7 +73,8 @@ async function getOrCreateActorAndServerAndModel (
70 73
71 try { 74 try {
72 // Don't recurse another time 75 // Don't recurse another time
73 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) 76 const recurseIfNeeded = false
77 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
74 } catch (err) { 78 } catch (err) {
75 logger.error('Cannot get or create account attributed to video channel ' + actor.url) 79 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
76 throw new Error(err) 80 throw new Error(err)
@@ -79,6 +83,7 @@ async function getOrCreateActorAndServerAndModel (
79 83
80 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor) 84 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
81 created = true 85 created = true
86 accountPlaylistsUrl = result.playlists
82 } 87 }
83 88
84 if (actor.Account) actor.Account.Actor = actor 89 if (actor.Account) actor.Account.Actor = actor
@@ -92,6 +97,12 @@ async function getOrCreateActorAndServerAndModel (
92 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) 97 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
93 } 98 }
94 99
100 // We created a new account: fetch the playlists
101 if (created === true && actor.Account && accountPlaylistsUrl) {
102 const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
103 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
104 }
105
95 return actorRefreshed 106 return actorRefreshed
96} 107}
97 108
@@ -107,7 +118,7 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU
107 followingCount: 0, 118 followingCount: 0,
108 inboxUrl: url + '/inbox', 119 inboxUrl: url + '/inbox',
109 outboxUrl: url + '/outbox', 120 outboxUrl: url + '/outbox',
110 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox', 121 sharedInboxUrl: WEBSERVER.URL + '/inbox',
111 followersUrl: url + '/followers', 122 followersUrl: url + '/followers',
112 followingUrl: url + '/following' 123 followingUrl: url + '/following'
113 }) 124 })
@@ -259,7 +270,7 @@ async function refreshActorIfNeeded (
259 return { refreshed: true, actor } 270 return { refreshed: true, actor }
260 }) 271 })
261 } catch (err) { 272 } catch (err) {
262 logger.warn('Cannot refresh actor.', { err }) 273 logger.warn('Cannot refresh actor %s.', actor.url, { err })
263 return { actor, refreshed: false } 274 return { actor, refreshed: false }
264 } 275 }
265} 276}
@@ -333,6 +344,8 @@ function saveActorAndServerAndModelIfNotExist (
333 actorCreated.VideoChannel.Account = ownerActor.Account 344 actorCreated.VideoChannel.Account = ownerActor.Account
334 } 345 }
335 346
347 actorCreated.Server = server
348
336 return actorCreated 349 return actorCreated
337 } 350 }
338} 351}
@@ -342,6 +355,7 @@ type FetchRemoteActorResult = {
342 name: string 355 name: string
343 summary: string 356 summary: string
344 support?: string 357 support?: string
358 playlists?: string
345 avatarName?: string 359 avatarName?: string
346 attributedTo: ActivityPubAttributedTo[] 360 attributedTo: ActivityPubAttributedTo[]
347} 361}
@@ -355,17 +369,18 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
355 369
356 logger.info('Fetching remote actor %s.', actorUrl) 370 logger.info('Fetching remote actor %s.', actorUrl)
357 371
358 const requestResult = await doRequest(options) 372 const requestResult = await doRequest<ActivityPubActor>(options)
359 normalizeActor(requestResult.body) 373 normalizeActor(requestResult.body)
360 374
361 const actorJSON: ActivityPubActor = requestResult.body 375 const actorJSON = requestResult.body
362 if (isActorObjectValid(actorJSON) === false) { 376 if (isActorObjectValid(actorJSON) === false) {
363 logger.debug('Remote actor JSON is not valid.', { actorJSON }) 377 logger.debug('Remote actor JSON is not valid.', { actorJSON })
364 return { result: undefined, statusCode: requestResult.response.statusCode } 378 return { result: undefined, statusCode: requestResult.response.statusCode }
365 } 379 }
366 380
367 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { 381 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
368 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) 382 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
383 return { result: undefined, statusCode: requestResult.response.statusCode }
369 } 384 }
370 385
371 const followersCount = await fetchActorTotalItems(actorJSON.followers) 386 const followersCount = await fetchActorTotalItems(actorJSON.followers)
@@ -398,6 +413,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
398 avatarName, 413 avatarName,
399 summary: actorJSON.summary, 414 summary: actorJSON.summary,
400 support: actorJSON.support, 415 support: actorJSON.support,
416 playlists: actorJSON.playlists,
401 attributedTo: actorJSON.attributedTo 417 attributedTo: actorJSON.attributedTo
402 } 418 }
403 } 419 }
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index 10277eca7..771a01366 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -1,6 +1,6 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience } from '../../../shared/models/activitypub' 2import { ActivityAudience } from '../../../shared/models/activitypub'
3import { ACTIVITY_PUB } from '../../initializers' 3import { ACTIVITY_PUB } from '../../initializers/constants'
4import { ActorModel } from '../../models/activitypub/actor' 4import { ActorModel } from '../../models/activitypub/actor'
5import { VideoModel } from '../../models/video/video' 5import { VideoModel } from '../../models/video/video'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index f6f068b45..de5cc54ac 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -2,10 +2,27 @@ import { CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize' 4import { Transaction } from 'sequelize'
5import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5 6
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url
8 8
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14
15 return {
16 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 }
24
25 const url = cacheFileObject.url
9 const videoFile = video.VideoFiles.find(f => { 26 const videoFile = video.VideoFiles.find(f => {
10 return f.resolution === url.height && f.fps === url.fps 27 return f.resolution === url.height && f.fps === url.fps
11 }) 28 })
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
15 return { 32 return {
16 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id, 34 url: cacheFileObject.id,
18 fileUrl: cacheFileObject.url.href, 35 fileUrl: url.href,
19 strategy: null, 36 strategy: null,
20 videoFileId: videoFile.id, 37 videoFileId: videoFile.id,
21 actorId: byActor.id 38 actorId: byActor.id
@@ -51,8 +68,8 @@ function updateCacheFile (
51 68
52 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) 69 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
53 70
54 redundancyModel.set('expires', attributes.expiresOn) 71 redundancyModel.expiresOn = attributes.expiresOn
55 redundancyModel.set('fileUrl', attributes.fileUrl) 72 redundancyModel.fileUrl = attributes.fileUrl
56 73
57 return redundancyModel.save({ transaction: t }) 74 return redundancyModel.save({ transaction: t })
58} 75}
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 1b9b14c2e..686eef04d 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -1,10 +1,14 @@
1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' 1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
2import { doRequest } from '../../helpers/requests' 2import { doRequest } from '../../helpers/requests'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import * as Bluebird from 'bluebird' 4import * as Bluebird from 'bluebird'
5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' 5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
6import { parse } from 'url'
6 7
7async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { 8type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
9type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
10
11async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
8 logger.info('Crawling ActivityPub data on %s.', uri) 12 logger.info('Crawling ActivityPub data on %s.', uri)
9 13
10 const options = { 14 const options = {
@@ -15,6 +19,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
15 timeout: JOB_REQUEST_TIMEOUT 19 timeout: JOB_REQUEST_TIMEOUT
16 } 20 }
17 21
22 const startDate = new Date()
23
18 const response = await doRequest<ActivityPubOrderedCollection<T>>(options) 24 const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
19 const firstBody = response.body 25 const firstBody = response.body
20 26
@@ -22,6 +28,10 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
22 let i = 0 28 let i = 0
23 let nextLink = firstBody.first 29 let nextLink = firstBody.first
24 while (nextLink && i < limit) { 30 while (nextLink && i < limit) {
31 // Don't crawl ourselves
32 const remoteHost = parse(nextLink).host
33 if (remoteHost === WEBSERVER.HOST) continue
34
25 options.uri = nextLink 35 options.uri = nextLink
26 36
27 const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options) 37 const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options)
@@ -35,6 +45,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr
35 await handler(items) 45 await handler(items)
36 } 46 }
37 } 47 }
48
49 if (cleaner) await cleaner(startDate)
38} 50}
39 51
40export { 52export {
diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts
index 6906bf9d3..d8c7d83b7 100644
--- a/server/lib/activitypub/index.ts
+++ b/server/lib/activitypub/index.ts
@@ -2,6 +2,7 @@ export * from './process'
2export * from './send' 2export * from './send'
3export * from './actor' 3export * from './actor'
4export * from './share' 4export * from './share'
5export * from './playlist'
5export * from './videos' 6export * from './videos'
6export * from './video-comments' 7export * from './video-comments'
7export * from './video-rates' 8export * from './video-rates'
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
new file mode 100644
index 000000000..36a91faec
--- /dev/null
+++ b/server/lib/activitypub/playlist.ts
@@ -0,0 +1,213 @@
1import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
2import { crawlCollectionPage } from './crawl'
3import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
4import { AccountModel } from '../../models/account/account'
5import { isArray } from '../../helpers/custom-validators/misc'
6import { getOrCreateActorAndServerAndModel } from './actor'
7import { logger } from '../../helpers/logger'
8import { VideoPlaylistModel } from '../../models/video/video-playlist'
9import { doRequest } from '../../helpers/requests'
10import { checkUrlsSameHost } from '../../helpers/activitypub'
11import * as Bluebird from 'bluebird'
12import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
13import { getOrCreateVideoAndAccountAndChannel } from './videos'
14import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
15import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
16import { VideoModel } from '../../models/video/video'
17import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
18import { sequelizeTypescript } from '../../initializers/database'
19import { createPlaylistMiniatureFromUrl } from '../thumbnail'
20import { FilteredModelAttributes } from '../../typings/sequelize'
21
22function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
23 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
24
25 return {
26 name: playlistObject.name,
27 description: playlistObject.content,
28 privacy,
29 url: playlistObject.id,
30 uuid: playlistObject.uuid,
31 ownerAccountId: byAccount.id,
32 videoChannelId: null,
33 createdAt: new Date(playlistObject.published),
34 updatedAt: new Date(playlistObject.updated)
35 }
36}
37
38function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) {
39 return {
40 position: elementObject.position,
41 url: elementObject.id,
42 startTimestamp: elementObject.startTimestamp || null,
43 stopTimestamp: elementObject.stopTimestamp || null,
44 videoPlaylistId: videoPlaylist.id,
45 videoId: video.id
46 }
47}
48
49async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) {
50 await Bluebird.map(playlistUrls, async playlistUrl => {
51 try {
52 const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
53 if (exists === true) return
54
55 // Fetch url
56 const { body } = await doRequest<PlaylistObject>({
57 uri: playlistUrl,
58 json: true,
59 activityPub: true
60 })
61
62 if (!isPlaylistObjectValid(body)) {
63 throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
64 }
65
66 if (!isArray(body.to)) {
67 throw new Error('Playlist does not have an audience.')
68 }
69
70 return createOrUpdateVideoPlaylist(body, account, body.to)
71 } catch (err) {
72 logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
73 }
74 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
75}
76
77async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
78 const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
79
80 if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
81 const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
82
83 if (actor.VideoChannel) {
84 playlistAttributes.videoChannelId = actor.VideoChannel.id
85 } else {
86 logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
87 }
88 }
89
90 const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true })
91
92 let accItems: string[] = []
93 await crawlCollectionPage<string>(playlistObject.id, items => {
94 accItems = accItems.concat(items)
95
96 return Promise.resolve()
97 })
98
99 const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
100
101 if (playlistObject.icon) {
102 try {
103 const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist)
104 await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
105 } catch (err) {
106 logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
107 }
108 }
109
110 return resetVideoPlaylistElements(accItems, refreshedPlaylist)
111}
112
113async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
114 if (!videoPlaylist.isOutdated()) return videoPlaylist
115
116 try {
117 const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
118 if (statusCode === 404) {
119 logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
120
121 await videoPlaylist.destroy()
122 return undefined
123 }
124
125 if (playlistObject === undefined) {
126 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
127
128 await videoPlaylist.setAsRefreshed()
129 return videoPlaylist
130 }
131
132 const byAccount = videoPlaylist.OwnerAccount
133 await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
134
135 return videoPlaylist
136 } catch (err) {
137 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
138
139 await videoPlaylist.setAsRefreshed()
140 return videoPlaylist
141 }
142}
143
144// ---------------------------------------------------------------------------
145
146export {
147 createAccountPlaylists,
148 playlistObjectToDBAttributes,
149 playlistElementObjectToDBAttributes,
150 createOrUpdateVideoPlaylist,
151 refreshVideoPlaylistIfNeeded
152}
153
154// ---------------------------------------------------------------------------
155
156async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) {
157 const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
158
159 await Bluebird.map(elementUrls, async elementUrl => {
160 try {
161 // Fetch url
162 const { body } = await doRequest<PlaylistElementObject>({
163 uri: elementUrl,
164 json: true,
165 activityPub: true
166 })
167
168 if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
169
170 if (checkUrlsSameHost(body.id, elementUrl) !== true) {
171 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
172 }
173
174 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
175
176 elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
177 } catch (err) {
178 logger.warn('Cannot add playlist element %s.', elementUrl, { err })
179 }
180 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
181
182 await sequelizeTypescript.transaction(async t => {
183 await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
184
185 for (const element of elementsToCreate) {
186 await VideoPlaylistElementModel.create(element, { transaction: t })
187 }
188 })
189
190 logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
191
192 return undefined
193}
194
195async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
196 const options = {
197 uri: playlistUrl,
198 method: 'GET',
199 json: true,
200 activityPub: true
201 }
202
203 logger.info('Fetching remote playlist %s.', playlistUrl)
204
205 const { response, body } = await doRequest(options)
206
207 if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
208 logger.debug('Remote video playlist JSON is not valid.', { body })
209 return { statusCode: response.statusCode, playlistObject: undefined }
210 }
211
212 return { statusCode: response.statusCode, playlistObject: body }
213}
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 5f4d793a5..e882669ce 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -12,6 +12,8 @@ import { Notifier } from '../../notifier'
12import { processViewActivity } from './process-view' 12import { processViewActivity } from './process-view'
13import { processDislikeActivity } from './process-dislike' 13import { processDislikeActivity } from './process-dislike'
14import { processFlagActivity } from './process-flag' 14import { processFlagActivity } from './process-flag'
15import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
16import { createOrUpdateVideoPlaylist } from '../playlist'
15 17
16async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { 18async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
17 const activityObject = activity.object 19 const activityObject = activity.object
@@ -38,7 +40,11 @@ async function processCreateActivity (activity: ActivityCreate, byActor: ActorMo
38 } 40 }
39 41
40 if (activityType === 'CacheFile') { 42 if (activityType === 'CacheFile') {
41 return retryTransactionWrapper(processCacheFile, activity, byActor) 43 return retryTransactionWrapper(processCreateCacheFile, activity, byActor)
44 }
45
46 if (activityType === 'Playlist') {
47 return retryTransactionWrapper(processCreatePlaylist, activity, byActor)
42 } 48 }
43 49
44 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) 50 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -63,7 +69,7 @@ async function processCreateVideo (activity: ActivityCreate) {
63 return video 69 return video
64} 70}
65 71
66async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) { 72async function processCreateCacheFile (activity: ActivityCreate, byActor: ActorModel) {
67 const cacheFile = activity.object as CacheFileObject 73 const cacheFile = activity.object as CacheFileObject
68 74
69 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 75 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
@@ -98,3 +104,12 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act
98 104
99 if (created === true) Notifier.Instance.notifyOnNewComment(comment) 105 if (created === true) Notifier.Instance.notifyOnNewComment(comment)
100} 106}
107
108async function processCreatePlaylist (activity: ActivityCreate, byActor: ActorModel) {
109 const playlistObject = activity.object as PlaylistObject
110 const byAccount = byActor.Account
111
112 if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
113
114 await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to)
115}
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts
index 155d2ffcc..76f07fd8a 100644
--- a/server/lib/activitypub/process/process-delete.ts
+++ b/server/lib/activitypub/process/process-delete.ts
@@ -8,6 +8,7 @@ import { VideoModel } from '../../../models/video/video'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { VideoCommentModel } from '../../../models/video/video-comment' 9import { VideoCommentModel } from '../../../models/video/video-comment'
10import { forwardVideoRelatedActivity } from '../send/utils' 10import { forwardVideoRelatedActivity } from '../send/utils'
11import { VideoPlaylistModel } from '../../../models/video/video-playlist'
11 12
12async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) { 13async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) {
13 const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id 14 const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id
@@ -45,6 +46,15 @@ async function processDeleteActivity (activity: ActivityDelete, byActor: ActorMo
45 } 46 }
46 } 47 }
47 48
49 {
50 const videoPlaylist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(objectUrl)
51 if (videoPlaylist) {
52 if (videoPlaylist.isOwned()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`)
53
54 return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist)
55 }
56 }
57
48 return undefined 58 return undefined
49} 59}
50 60
@@ -70,6 +80,20 @@ async function processDeleteVideo (actor: ActorModel, videoToDelete: VideoModel)
70 logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) 80 logger.info('Remote video with uuid %s removed.', videoToDelete.uuid)
71} 81}
72 82
83async function processDeleteVideoPlaylist (actor: ActorModel, playlistToDelete: VideoPlaylistModel) {
84 logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid)
85
86 await sequelizeTypescript.transaction(async t => {
87 if (playlistToDelete.OwnerAccount.Actor.id !== actor.id) {
88 throw new Error('Account ' + actor.url + ' does not own video playlist ' + playlistToDelete.url)
89 }
90
91 await playlistToDelete.destroy({ transaction: t })
92 })
93
94 logger.info('Remote video playlist with uuid %s removed.', playlistToDelete.uuid)
95}
96
73async function processDeleteAccount (accountToRemove: AccountModel) { 97async function processDeleteAccount (accountToRemove: AccountModel) {
74 logger.debug('Removing remote account "%s".', accountToRemove.Actor.uuid) 98 logger.debug('Removing remote account "%s".', accountToRemove.Actor.uuid)
75 99
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index 0cd537187..ed16ba172 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -4,9 +4,11 @@ import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers' 4import { sequelizeTypescript } from '../../../initializers'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 6import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { sendAccept } from '../send' 7import { sendAccept, sendReject } from '../send'
8import { Notifier } from '../../notifier' 8import { Notifier } from '../../notifier'
9import { getAPId } from '../../../helpers/activitypub' 9import { getAPId } from '../../../helpers/activitypub'
10import { getServerActor } from '../../../helpers/utils'
11import { CONFIG } from '../../../initializers/config'
10 12
11async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { 13async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) {
12 const activityObject = getAPId(activity.object) 14 const activityObject = getAPId(activity.object)
@@ -23,12 +25,23 @@ export {
23// --------------------------------------------------------------------------- 25// ---------------------------------------------------------------------------
24 26
25async function processFollow (actor: ActorModel, targetActorURL: string) { 27async function processFollow (actor: ActorModel, targetActorURL: string) {
26 const { actorFollow, created } = await sequelizeTypescript.transaction(async t => { 28 const { actorFollow, created, isFollowingInstance } = await sequelizeTypescript.transaction(async t => {
27 const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) 29 const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t)
28 30
29 if (!targetActor) throw new Error('Unknown actor') 31 if (!targetActor) throw new Error('Unknown actor')
30 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') 32 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.')
31 33
34 const serverActor = await getServerActor()
35 const isFollowingInstance = targetActor.id === serverActor.id
36
37 if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) {
38 logger.info('Rejecting %s because instance followers are disabled.', targetActor.url)
39
40 await sendReject(actor, targetActor)
41
42 return { actorFollow: undefined }
43 }
44
32 const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({ 45 const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({
33 where: { 46 where: {
34 actorId: actor.id, 47 actorId: actor.id,
@@ -37,15 +50,12 @@ async function processFollow (actor: ActorModel, targetActorURL: string) {
37 defaults: { 50 defaults: {
38 actorId: actor.id, 51 actorId: actor.id,
39 targetActorId: targetActor.id, 52 targetActorId: targetActor.id,
40 state: 'accepted' 53 state: CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL ? 'pending' : 'accepted'
41 }, 54 },
42 transaction: t 55 transaction: t
43 }) 56 })
44 57
45 actorFollow.ActorFollower = actor 58 if (actorFollow.state !== 'accepted' && CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL === false) {
46 actorFollow.ActorFollowing = targetActor
47
48 if (actorFollow.state !== 'accepted') {
49 actorFollow.state = 'accepted' 59 actorFollow.state = 'accepted'
50 await actorFollow.save({ transaction: t }) 60 await actorFollow.save({ transaction: t })
51 } 61 }
@@ -54,12 +64,18 @@ async function processFollow (actor: ActorModel, targetActorURL: string) {
54 actorFollow.ActorFollowing = targetActor 64 actorFollow.ActorFollowing = targetActor
55 65
56 // Target sends to actor he accepted the follow request 66 // Target sends to actor he accepted the follow request
57 await sendAccept(actorFollow) 67 if (actorFollow.state === 'accepted') await sendAccept(actorFollow)
58 68
59 return { actorFollow, created } 69 return { actorFollow, created, isFollowingInstance }
60 }) 70 })
61 71
62 if (created) Notifier.Instance.notifyOfNewFollow(actorFollow) 72 // Rejected
73 if (!actorFollow) return
74
75 if (created) {
76 if (isFollowingInstance) Notifier.Instance.notifyOfNewInstanceFollow(actorFollow)
77 else Notifier.Instance.notifyOfNewUserFollow(actorFollow)
78 }
63 79
64 logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) 80 logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url)
65} 81}
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index ed0177a67..2d48848fe 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -108,7 +108,10 @@ async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo
108 108
109 return sequelizeTypescript.transaction(async t => { 109 return sequelizeTypescript.transaction(async t => {
110 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 110 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
111 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFileObject.id) 111 if (!cacheFile) {
112 logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id)
113 return
114 }
112 115
113 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') 116 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
114 117
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index c6b42d846..54a9234bb 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -12,6 +12,8 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-vali
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { createOrUpdateCacheFile } from '../cache-file' 13import { createOrUpdateCacheFile } from '../cache-file'
14import { forwardVideoRelatedActivity } from '../send/utils' 14import { forwardVideoRelatedActivity } from '../send/utils'
15import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
16import { createOrUpdateVideoPlaylist } from '../playlist'
15 17
16async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { 18async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
17 const objectType = activity.object.type 19 const objectType = activity.object.type
@@ -32,6 +34,10 @@ async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorMo
32 return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity) 34 return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity)
33 } 35 }
34 36
37 if (objectType === 'Playlist') {
38 return retryTransactionWrapper(processUpdatePlaylist, byActor, activity)
39 }
40
35 return undefined 41 return undefined
36} 42}
37 43
@@ -114,9 +120,11 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
114 120
115 await actor.save({ transaction: t }) 121 await actor.save({ transaction: t })
116 122
117 accountOrChannelInstance.set('name', actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername) 123 accountOrChannelInstance.name = actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername
118 accountOrChannelInstance.set('description', actorAttributesToUpdate.summary) 124 accountOrChannelInstance.description = actorAttributesToUpdate.summary
119 accountOrChannelInstance.set('support', actorAttributesToUpdate.support) 125
126 if (accountOrChannelInstance instanceof VideoChannelModel) accountOrChannelInstance.support = actorAttributesToUpdate.support
127
120 await accountOrChannelInstance.save({ transaction: t }) 128 await accountOrChannelInstance.save({ transaction: t })
121 }) 129 })
122 130
@@ -135,3 +143,12 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
135 throw err 143 throw err
136 } 144 }
137} 145}
146
147async function processUpdatePlaylist (byActor: ActorModel, activity: ActivityUpdate) {
148 const playlistObject = activity.object as PlaylistObject
149 const byAccount = byActor.Account
150
151 if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
152
153 await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to)
154}
diff --git a/server/lib/activitypub/send/index.ts b/server/lib/activitypub/send/index.ts
index 79ba6c7fe..028936810 100644
--- a/server/lib/activitypub/send/index.ts
+++ b/server/lib/activitypub/send/index.ts
@@ -1,8 +1,10 @@
1export * from './send-accept' 1export * from './send-accept'
2export * from './send-accept'
2export * from './send-announce' 3export * from './send-announce'
3export * from './send-create' 4export * from './send-create'
4export * from './send-delete' 5export * from './send-delete'
5export * from './send-follow' 6export * from './send-follow'
6export * from './send-like' 7export * from './send-like'
8export * from './send-reject'
7export * from './send-undo' 9export * from './send-undo'
8export * from './send-update' 10export * from './send-update'
diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts
index b6abde13d..388a9ed23 100644
--- a/server/lib/activitypub/send/send-accept.ts
+++ b/server/lib/activitypub/send/send-accept.ts
@@ -17,7 +17,7 @@ async function sendAccept (actorFollow: ActorFollowModel) {
17 17
18 logger.info('Creating job to accept follower %s.', follower.url) 18 logger.info('Creating job to accept follower %s.', follower.url)
19 19
20 const followUrl = getActorFollowActivityPubUrl(actorFollow) 20 const followUrl = getActorFollowActivityPubUrl(follower, me)
21 const followData = buildFollowActivity(followUrl, follower, me) 21 const followData = buildFollowActivity(followUrl, follower, me)
22 22
23 const url = getActorFollowAcceptActivityPubUrl(actorFollow) 23 const url = getActorFollowAcceptActivityPubUrl(actorFollow)
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index e3fca0a17..28f18595b 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -3,13 +3,14 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti
3import { VideoPrivacy } from '../../../../shared/models/videos' 3import { VideoPrivacy } from '../../../../shared/models/videos'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
6import { VideoAbuseModel } from '../../../models/video/video-abuse'
7import { VideoCommentModel } from '../../../models/video/video-comment' 6import { VideoCommentModel } from '../../../models/video/video-comment'
8import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url'
9import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 7import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
10import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' 8import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
11import { logger } from '../../../helpers/logger' 9import { logger } from '../../../helpers/logger'
12import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 10import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
11import { VideoPlaylistModel } from '../../../models/video/video-playlist'
12import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
13import { getServerActor } from '../../../helpers/utils'
13 14
14async function sendCreateVideo (video: VideoModel, t: Transaction) { 15async function sendCreateVideo (video: VideoModel, t: Transaction) {
15 if (video.privacy === VideoPrivacy.PRIVATE) return undefined 16 if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -25,34 +26,36 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) {
25 return broadcastToFollowers(createActivity, byActor, [ byActor ], t) 26 return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
26} 27}
27 28
28async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { 29async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
29 if (!video.VideoChannel.Account.Actor.serverId) return // Local
30
31 const url = getVideoAbuseActivityPubUrl(videoAbuse)
32
33 logger.info('Creating job to send video abuse %s.', url)
34
35 // Custom audience, we only send the abuse to the origin instance
36 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
37 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
38
39 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
40}
41
42async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 30 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
44 31
45 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
46 const redundancyObject = fileRedundancy.toActivityPubObject()
47
48 return sendVideoRelatedCreateActivity({ 32 return sendVideoRelatedCreateActivity({
49 byActor, 33 byActor,
50 video, 34 video,
51 url: fileRedundancy.url, 35 url: fileRedundancy.url,
52 object: redundancyObject 36 object: fileRedundancy.toActivityPubObject()
53 }) 37 })
54} 38}
55 39
40async function sendCreateVideoPlaylist (playlist: VideoPlaylistModel, t: Transaction) {
41 if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
42
43 logger.info('Creating job to send create video playlist of %s.', playlist.url)
44
45 const byActor = playlist.OwnerAccount.Actor
46 const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC)
47
48 const object = await playlist.toActivityPubObject(null, t)
49 const createActivity = buildCreateActivity(playlist.url, byActor, object, audience)
50
51 const serverActor = await getServerActor()
52 const toFollowersOf = [ byActor, serverActor ]
53
54 if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor)
55
56 return broadcastToFollowers(createActivity, byActor, toFollowersOf, t)
57}
58
56async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { 59async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
57 logger.info('Creating job to send comment %s.', comment.url) 60 logger.info('Creating job to send comment %s.', comment.url)
58 61
@@ -91,37 +94,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
91 return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) 94 return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
92} 95}
93 96
94async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) {
95 logger.info('Creating job to send view of %s.', video.url)
96
97 const url = getVideoViewActivityPubUrl(byActor, video)
98 const viewActivity = buildViewActivity(url, byActor, video)
99
100 return sendVideoRelatedCreateActivity({
101 // Use the server actor to send the view
102 byActor,
103 video,
104 url,
105 object: viewActivity,
106 transaction: t
107 })
108}
109
110async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
111 logger.info('Creating job to dislike %s.', video.url)
112
113 const url = getVideoDislikeActivityPubUrl(byActor, video)
114 const dislikeActivity = buildDislikeActivity(url, byActor, video)
115
116 return sendVideoRelatedCreateActivity({
117 byActor,
118 video,
119 url,
120 object: dislikeActivity,
121 transaction: t
122 })
123}
124
125function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { 97function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
126 if (!audience) audience = getAudience(byActor) 98 if (!audience) audience = getAudience(byActor)
127 99
@@ -136,34 +108,13 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud
136 ) 108 )
137} 109}
138 110
139function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) {
140 return {
141 id: url,
142 type: 'Dislike',
143 actor: byActor.url,
144 object: video.url
145 }
146}
147
148function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) {
149 return {
150 id: url,
151 type: 'View',
152 actor: byActor.url,
153 object: video.url
154 }
155}
156
157// --------------------------------------------------------------------------- 111// ---------------------------------------------------------------------------
158 112
159export { 113export {
160 sendCreateVideo, 114 sendCreateVideo,
161 sendVideoAbuse,
162 buildCreateActivity, 115 buildCreateActivity,
163 sendCreateView,
164 sendCreateDislike,
165 buildDislikeActivity,
166 sendCreateVideoComment, 116 sendCreateVideoComment,
117 sendCreateVideoPlaylist,
167 sendCreateCacheFile 118 sendCreateCacheFile
168} 119}
169 120
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 18969433a..7bf5ca520 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -8,6 +8,8 @@ import { getDeleteActivityPubUrl } from '../url'
8import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 8import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
9import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' 9import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience'
10import { logger } from '../../../helpers/logger' 10import { logger } from '../../../helpers/logger'
11import { VideoPlaylistModel } from '../../../models/video/video-playlist'
12import { getServerActor } from '../../../helpers/utils'
11 13
12async function sendDeleteVideo (video: VideoModel, transaction: Transaction) { 14async function sendDeleteVideo (video: VideoModel, transaction: Transaction) {
13 logger.info('Creating job to broadcast delete of video %s.', video.url) 15 logger.info('Creating job to broadcast delete of video %s.', video.url)
@@ -29,7 +31,12 @@ async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
29 const url = getDeleteActivityPubUrl(byActor.url) 31 const url = getDeleteActivityPubUrl(byActor.url)
30 const activity = buildDeleteActivity(url, byActor.url, byActor) 32 const activity = buildDeleteActivity(url, byActor.url, byActor)
31 33
32 const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) 34 const actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, t)
35
36 // In case the actor did not have any videos
37 const serverActor = await getServerActor()
38 actorsInvolved.push(serverActor)
39
33 actorsInvolved.push(byActor) 40 actorsInvolved.push(byActor)
34 41
35 return broadcastToFollowers(activity, byActor, actorsInvolved, t) 42 return broadcastToFollowers(activity, byActor, actorsInvolved, t)
@@ -64,12 +71,29 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
64 return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) 71 return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
65} 72}
66 73
74async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
75 logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url)
76
77 const byActor = videoPlaylist.OwnerAccount.Actor
78
79 const url = getDeleteActivityPubUrl(videoPlaylist.url)
80 const activity = buildDeleteActivity(url, videoPlaylist.url, byActor)
81
82 const serverActor = await getServerActor()
83 const toFollowersOf = [ byActor, serverActor ]
84
85 if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor)
86
87 return broadcastToFollowers(activity, byActor, toFollowersOf, t)
88}
89
67// --------------------------------------------------------------------------- 90// ---------------------------------------------------------------------------
68 91
69export { 92export {
70 sendDeleteVideo, 93 sendDeleteVideo,
71 sendDeleteActor, 94 sendDeleteActor,
72 sendDeleteVideoComment 95 sendDeleteVideoComment,
96 sendDeleteVideoPlaylist
73} 97}
74 98
75// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts
new file mode 100644
index 000000000..a88436f2c
--- /dev/null
+++ b/server/lib/activitypub/send/send-dislike.ts
@@ -0,0 +1,41 @@
1import { Transaction } from 'sequelize'
2import { ActorModel } from '../../../models/activitypub/actor'
3import { VideoModel } from '../../../models/video/video'
4import { getVideoDislikeActivityPubUrl } from '../url'
5import { logger } from '../../../helpers/logger'
6import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub'
7import { sendVideoRelatedActivity } from './utils'
8import { audiencify, getAudience } from '../audience'
9
10async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
11 logger.info('Creating job to dislike %s.', video.url)
12
13 const activityBuilder = (audience: ActivityAudience) => {
14 const url = getVideoDislikeActivityPubUrl(byActor, video)
15
16 return buildDislikeActivity(url, byActor, video, audience)
17 }
18
19 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
20}
21
22function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike {
23 if (!audience) audience = getAudience(byActor)
24
25 return audiencify(
26 {
27 id: url,
28 type: 'Dislike' as 'Dislike',
29 actor: byActor.url,
30 object: video.url
31 },
32 audience
33 )
34}
35
36// ---------------------------------------------------------------------------
37
38export {
39 sendDislike,
40 buildDislikeActivity
41}
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts
new file mode 100644
index 000000000..96a7311b9
--- /dev/null
+++ b/server/lib/activitypub/send/send-flag.ts
@@ -0,0 +1,39 @@
1import { ActorModel } from '../../../models/activitypub/actor'
2import { VideoModel } from '../../../models/video/video'
3import { VideoAbuseModel } from '../../../models/video/video-abuse'
4import { getVideoAbuseActivityPubUrl } from '../url'
5import { unicastTo } from './utils'
6import { logger } from '../../../helpers/logger'
7import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub'
8import { audiencify, getAudience } from '../audience'
9
10async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
11 if (!video.VideoChannel.Account.Actor.serverId) return // Local user
12
13 const url = getVideoAbuseActivityPubUrl(videoAbuse)
14
15 logger.info('Creating job to send video abuse %s.', url)
16
17 // Custom audience, we only send the abuse to the origin instance
18 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
19 const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
20
21 return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
22}
23
24function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag {
25 if (!audience) audience = getAudience(byActor)
26
27 const activity = Object.assign(
28 { id: url, actor: byActor.url },
29 videoAbuse.toActivityPubObject()
30 )
31
32 return audiencify(activity, audience)
33}
34
35// ---------------------------------------------------------------------------
36
37export {
38 sendVideoAbuse
39}
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts
index 170b46b48..2c3d02014 100644
--- a/server/lib/activitypub/send/send-follow.ts
+++ b/server/lib/activitypub/send/send-follow.ts
@@ -14,7 +14,7 @@ function sendFollow (actorFollow: ActorFollowModel) {
14 14
15 logger.info('Creating job to send follow request to %s.', following.url) 15 logger.info('Creating job to send follow request to %s.', following.url)
16 16
17 const url = getActorFollowActivityPubUrl(actorFollow) 17 const url = getActorFollowActivityPubUrl(me, following)
18 const data = buildFollowActivity(url, me, following) 18 const data = buildFollowActivity(url, me, following)
19 19
20 return unicastTo(data, me, following.inboxUrl) 20 return unicastTo(data, me, following.inboxUrl)
diff --git a/server/lib/activitypub/send/send-reject.ts b/server/lib/activitypub/send/send-reject.ts
new file mode 100644
index 000000000..bac7ff556
--- /dev/null
+++ b/server/lib/activitypub/send/send-reject.ts
@@ -0,0 +1,40 @@
1import { ActivityFollow, ActivityReject } from '../../../../shared/models/activitypub'
2import { ActorModel } from '../../../models/activitypub/actor'
3import { getActorFollowActivityPubUrl, getActorFollowRejectActivityPubUrl } from '../url'
4import { unicastTo } from './utils'
5import { buildFollowActivity } from './send-follow'
6import { logger } from '../../../helpers/logger'
7
8async function sendReject (follower: ActorModel, following: ActorModel) {
9 if (!follower.serverId) { // This should never happen
10 logger.warn('Do not sending reject to local follower.')
11 return
12 }
13
14 logger.info('Creating job to reject follower %s.', follower.url)
15
16 const followUrl = getActorFollowActivityPubUrl(follower, following)
17 const followData = buildFollowActivity(followUrl, follower, following)
18
19 const url = getActorFollowRejectActivityPubUrl(follower, following)
20 const data = buildRejectActivity(url, following, followData)
21
22 return unicastTo(data, following, follower.inboxUrl)
23}
24
25// ---------------------------------------------------------------------------
26
27export {
28 sendReject
29}
30
31// ---------------------------------------------------------------------------
32
33function buildRejectActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityReject {
34 return {
35 type: 'Reject',
36 id: url,
37 actor: byActor.url,
38 object: followActivityData
39 }
40}
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index bf1b6e117..8727a121e 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize'
2import { 2import {
3 ActivityAnnounce, 3 ActivityAnnounce,
4 ActivityAudience, 4 ActivityAudience,
5 ActivityCreate, 5 ActivityCreate, ActivityDislike,
6 ActivityFollow, 6 ActivityFollow,
7 ActivityLike, 7 ActivityLike,
8 ActivityUndo 8 ActivityUndo
@@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video'
13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' 13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
14import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 14import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
15import { audiencify, getAudience } from '../audience' 15import { audiencify, getAudience } from '../audience'
16import { buildCreateActivity, buildDislikeActivity } from './send-create' 16import { buildCreateActivity } from './send-create'
17import { buildFollowActivity } from './send-follow' 17import { buildFollowActivity } from './send-follow'
18import { buildLikeActivity } from './send-like' 18import { buildLikeActivity } from './send-like'
19import { VideoShareModel } from '../../../models/video/video-share' 19import { VideoShareModel } from '../../../models/video/video-share'
20import { buildAnnounceWithVideoAudience } from './send-announce' 20import { buildAnnounceWithVideoAudience } from './send-announce'
21import { logger } from '../../../helpers/logger' 21import { logger } from '../../../helpers/logger'
22import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 22import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
23import { buildDislikeActivity } from './send-dislike'
23 24
24async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { 25async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
25 const me = actorFollow.ActorFollower 26 const me = actorFollow.ActorFollower
@@ -30,7 +31,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
30 31
31 logger.info('Creating job to send an unfollow request to %s.', following.url) 32 logger.info('Creating job to send an unfollow request to %s.', following.url)
32 33
33 const followUrl = getActorFollowActivityPubUrl(actorFollow) 34 const followUrl = getActorFollowActivityPubUrl(me, following)
34 const undoUrl = getUndoActivityPubUrl(followUrl) 35 const undoUrl = getUndoActivityPubUrl(followUrl)
35 36
36 const followActivity = buildFollowActivity(followUrl, me, following) 37 const followActivity = buildFollowActivity(followUrl, me, following)
@@ -65,15 +66,15 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
65 66
66 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) 67 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
67 const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) 68 const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video)
68 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
69 69
70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) 70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t })
71} 71}
72 72
73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
74 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
75 75
76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const videoId = redundancyModel.getVideo().id
77 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 78 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
78 79
79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) 80 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
@@ -94,7 +95,7 @@ export {
94function undoActivityData ( 95function undoActivityData (
95 url: string, 96 url: string,
96 byActor: ActorModel, 97 byActor: ActorModel,
97 object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, 98 object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce,
98 audience?: ActivityAudience 99 audience?: ActivityAudience
99): ActivityUndo { 100): ActivityUndo {
100 if (!audience) audience = getAudience(byActor) 101 if (!audience) audience = getAudience(byActor)
@@ -114,7 +115,7 @@ async function sendUndoVideoRelatedActivity (options: {
114 byActor: ActorModel, 115 byActor: ActorModel,
115 video: VideoModel, 116 video: VideoModel,
116 url: string, 117 url: string,
117 activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, 118 activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce,
118 transaction: Transaction 119 transaction: Transaction
119}) { 120}) {
120 const activityBuilder = (audience: ActivityAudience) => { 121 const activityBuilder = (audience: ActivityAudience) => {
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index a68f03edf..7411c08d5 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -12,8 +12,13 @@ import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience'
12import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
13import { VideoCaptionModel } from '../../../models/video/video-caption' 13import { VideoCaptionModel } from '../../../models/video/video-caption'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
15import { VideoPlaylistModel } from '../../../models/video/video-playlist'
16import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
17import { getServerActor } from '../../../helpers/utils'
15 18
16async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { 19async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) {
20 if (video.privacy === VideoPrivacy.PRIVATE) return undefined
21
17 logger.info('Creating job to update video %s.', video.url) 22 logger.info('Creating job to update video %s.', video.url)
18 23
19 const byActor = overrodeByActor ? overrodeByActor : video.VideoChannel.Account.Actor 24 const byActor = overrodeByActor ? overrodeByActor : video.VideoChannel.Account.Actor
@@ -47,7 +52,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
47 let actorsInvolved: ActorModel[] 52 let actorsInvolved: ActorModel[]
48 if (accountOrChannel instanceof AccountModel) { 53 if (accountOrChannel instanceof AccountModel) {
49 // Actors that shared my videos are involved too 54 // Actors that shared my videos are involved too
50 actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) 55 actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, t)
51 } else { 56 } else {
52 // Actors that shared videos of my channel are involved too 57 // Actors that shared videos of my channel are involved too
53 actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, t) 58 actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, t)
@@ -61,7 +66,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 66async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 67 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 68
64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 69 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)
65 70
66 const activityBuilder = (audience: ActivityAudience) => { 71 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject() 72 const redundancyObject = redundancyModel.toActivityPubObject()
@@ -73,12 +78,35 @@ async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoR
73 return sendVideoRelatedActivity(activityBuilder, { byActor, video }) 78 return sendVideoRelatedActivity(activityBuilder, { byActor, video })
74} 79}
75 80
81async function sendUpdateVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
82 if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
83
84 const byActor = videoPlaylist.OwnerAccount.Actor
85
86 logger.info('Creating job to update video playlist %s.', videoPlaylist.url)
87
88 const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString())
89
90 const object = await videoPlaylist.toActivityPubObject(null, t)
91 const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC)
92
93 const updateActivity = buildUpdateActivity(url, byActor, object, audience)
94
95 const serverActor = await getServerActor()
96 const toFollowersOf = [ byActor, serverActor ]
97
98 if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor)
99
100 return broadcastToFollowers(updateActivity, byActor, toFollowersOf, t)
101}
102
76// --------------------------------------------------------------------------- 103// ---------------------------------------------------------------------------
77 104
78export { 105export {
79 sendUpdateActor, 106 sendUpdateActor,
80 sendUpdateVideo, 107 sendUpdateVideo,
81 sendUpdateCacheFile 108 sendUpdateCacheFile,
109 sendUpdateVideoPlaylist
82} 110}
83 111
84// --------------------------------------------------------------------------- 112// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts
new file mode 100644
index 000000000..8ad126be0
--- /dev/null
+++ b/server/lib/activitypub/send/send-view.ts
@@ -0,0 +1,40 @@
1import { Transaction } from 'sequelize'
2import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub'
3import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video'
5import { getVideoLikeActivityPubUrl } from '../url'
6import { sendVideoRelatedActivity } from './utils'
7import { audiencify, getAudience } from '../audience'
8import { logger } from '../../../helpers/logger'
9
10async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) {
11 logger.info('Creating job to send view of %s.', video.url)
12
13 const activityBuilder = (audience: ActivityAudience) => {
14 const url = getVideoLikeActivityPubUrl(byActor, video)
15
16 return buildViewActivity(url, byActor, video, audience)
17 }
18
19 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
20}
21
22function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView {
23 if (!audience) audience = getAudience(byActor)
24
25 return audiencify(
26 {
27 id: url,
28 type: 'View' as 'View',
29 actor: byActor.url,
30 object: video.url
31 },
32 audience
33 )
34}
35
36// ---------------------------------------------------------------------------
37
38export {
39 sendView
40}
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 1767df0ae..7f38402b6 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -10,7 +10,7 @@ import * as Bluebird from 'bluebird'
10import { doRequest } from '../../helpers/requests' 10import { doRequest } from '../../helpers/requests'
11import { getOrCreateActorAndServerAndModel } from './actor' 11import { getOrCreateActorAndServerAndModel } from './actor'
12import { logger } from '../../helpers/logger' 12import { logger } from '../../helpers/logger'
13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 13import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
14import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 14import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
15 15
16async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { 16async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
@@ -54,12 +54,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) {
54 url: shareUrl 54 url: shareUrl
55 } 55 }
56 56
57 await VideoShareModel.findOrCreate({ 57 await VideoShareModel.upsert(entry)
58 where: {
59 url: shareUrl
60 },
61 defaults: entry
62 })
63 } catch (err) { 58 } catch (err) {
64 logger.warn('Cannot add share %s.', shareUrl, { err }) 59 logger.warn('Cannot add share %s.', shareUrl, { err })
65 } 60 }
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 38f15448c..bcb7a4ee2 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -1,35 +1,49 @@
1import { CONFIG } from '../../initializers' 1import { WEBSERVER } from '../../initializers/constants'
2import { ActorModel } from '../../models/activitypub/actor' 2import { ActorModel } from '../../models/activitypub/actor'
3import { ActorFollowModel } from '../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../models/activitypub/actor-follow'
4import { VideoModel } from '../../models/video/video' 4import { VideoModel } from '../../models/video/video'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseModel } from '../../models/video/video-abuse'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoFileModel } from '../../models/video/video-file' 7import { VideoFileModel } from '../../models/video/video-file'
8import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
9import { VideoPlaylistModel } from '../../models/video/video-playlist'
8 10
9function getVideoActivityPubUrl (video: VideoModel) { 11function getVideoActivityPubUrl (video: VideoModel) {
10 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 12 return WEBSERVER.URL + '/videos/watch/' + video.uuid
13}
14
15function getVideoPlaylistActivityPubUrl (videoPlaylist: VideoPlaylistModel) {
16 return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid
17}
18
19function getVideoPlaylistElementActivityPubUrl (videoPlaylist: VideoPlaylistModel, video: VideoModel) {
20 return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/' + video.uuid
11} 21}
12 22
13function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { 23function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
14 const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' 24 const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : ''
15 25
16 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` 26 return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
27}
28
29function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
30 return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}`
17} 31}
18 32
19function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { 33function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
20 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id 34 return WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
21} 35}
22 36
23function getVideoChannelActivityPubUrl (videoChannelName: string) { 37function getVideoChannelActivityPubUrl (videoChannelName: string) {
24 return CONFIG.WEBSERVER.URL + '/video-channels/' + videoChannelName 38 return WEBSERVER.URL + '/video-channels/' + videoChannelName
25} 39}
26 40
27function getAccountActivityPubUrl (accountName: string) { 41function getAccountActivityPubUrl (accountName: string) {
28 return CONFIG.WEBSERVER.URL + '/accounts/' + accountName 42 return WEBSERVER.URL + '/accounts/' + accountName
29} 43}
30 44
31function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) { 45function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) {
32 return CONFIG.WEBSERVER.URL + '/admin/video-abuses/' + videoAbuse.id 46 return WEBSERVER.URL + '/admin/video-abuses/' + videoAbuse.id
33} 47}
34 48
35function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { 49function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) {
@@ -60,11 +74,8 @@ function getVideoDislikesActivityPubUrl (video: VideoModel) {
60 return video.url + '/dislikes' 74 return video.url + '/dislikes'
61} 75}
62 76
63function getActorFollowActivityPubUrl (actorFollow: ActorFollowModel) { 77function getActorFollowActivityPubUrl (follower: ActorModel, following: ActorModel) {
64 const me = actorFollow.ActorFollower 78 return follower.url + '/follows/' + following.id
65 const following = actorFollow.ActorFollowing
66
67 return me.url + '/follows/' + following.id
68} 79}
69 80
70function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) { 81function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) {
@@ -74,6 +85,10 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) {
74 return follower.url + '/accepts/follows/' + me.id 85 return follower.url + '/accepts/follows/' + me.id
75} 86}
76 87
88function getActorFollowRejectActivityPubUrl (follower: ActorModel, following: ActorModel) {
89 return follower.url + '/rejects/follows/' + following.id
90}
91
77function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) { 92function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) {
78 return video.url + '/announces/' + byActor.id 93 return video.url + '/announces/' + byActor.id
79} 94}
@@ -92,6 +107,9 @@ function getUndoActivityPubUrl (originalUrl: string) {
92 107
93export { 108export {
94 getVideoActivityPubUrl, 109 getVideoActivityPubUrl,
110 getVideoPlaylistElementActivityPubUrl,
111 getVideoPlaylistActivityPubUrl,
112 getVideoCacheStreamingPlaylistActivityPubUrl,
95 getVideoChannelActivityPubUrl, 113 getVideoChannelActivityPubUrl,
96 getAccountActivityPubUrl, 114 getAccountActivityPubUrl,
97 getVideoAbuseActivityPubUrl, 115 getVideoAbuseActivityPubUrl,
@@ -103,6 +121,7 @@ export {
103 getVideoViewActivityPubUrl, 121 getVideoViewActivityPubUrl,
104 getVideoLikeActivityPubUrl, 122 getVideoLikeActivityPubUrl,
105 getVideoDislikeActivityPubUrl, 123 getVideoDislikeActivityPubUrl,
124 getActorFollowRejectActivityPubUrl,
106 getVideoCommentActivityPubUrl, 125 getVideoCommentActivityPubUrl,
107 getDeleteActivityPubUrl, 126 getDeleteActivityPubUrl,
108 getVideoSharesActivityPubUrl, 127 getVideoSharesActivityPubUrl,
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index e87301fe7..18f44d50e 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -2,7 +2,7 @@ import { VideoCommentObject } from '../../../shared/models/activitypub/objects/v
2import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' 2import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { doRequest } from '../../helpers/requests' 4import { doRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
6import { ActorModel } from '../../models/activitypub/actor' 6import { ActorModel } from '../../models/activitypub/actor'
7import { VideoModel } from '../../models/video/video' 7import { VideoModel } from '../../models/video/video'
8import { VideoCommentModel } from '../../models/video/video-comment' 8import { VideoCommentModel } from '../../models/video/video-comment'
@@ -34,8 +34,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
34 accountId: actor.Account.id, 34 accountId: actor.Account.id,
35 inReplyToCommentId, 35 inReplyToCommentId,
36 originCommentId, 36 originCommentId,
37 createdAt: new Date(comment.published), 37 createdAt: new Date(comment.published)
38 updatedAt: new Date(comment.updated)
39 } 38 }
40} 39}
41 40
@@ -74,12 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
74 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) 73 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
75 if (!entry) return { created: false } 74 if (!entry) return { created: false }
76 75
77 const [ comment, created ] = await VideoCommentModel.findOrCreate({ 76 const [ comment, created ] = await VideoCommentModel.upsert<VideoCommentModel>(entry, { returning: true })
78 where: {
79 url: body.id
80 },
81 defaults: entry
82 })
83 comment.Account = actor.Account 77 comment.Account = actor.Account
84 comment.Video = videoInstance 78 comment.Video = videoInstance
85 79
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 45a2b22ea..cda5b2981 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -1,17 +1,18 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { AccountModel } from '../../models/account/account' 2import { AccountModel } from '../../models/account/account'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' 4import { sendLike, sendUndoDislike, sendUndoLike } from './send'
5import { VideoRateType } from '../../../shared/models/videos' 5import { VideoRateType } from '../../../shared/models/videos'
6import * as Bluebird from 'bluebird' 6import * as Bluebird from 'bluebird'
7import { getOrCreateActorAndServerAndModel } from './actor' 7import { getOrCreateActorAndServerAndModel } from './actor'
8import { AccountVideoRateModel } from '../../models/account/account-video-rate' 8import { AccountVideoRateModel } from '../../models/account/account-video-rate'
9import { logger } from '../../helpers/logger' 9import { logger } from '../../helpers/logger'
10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' 10import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
11import { doRequest } from '../../helpers/requests' 11import { doRequest } from '../../helpers/requests'
12import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 12import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
13import { ActorModel } from '../../models/activitypub/actor' 13import { ActorModel } from '../../models/activitypub/actor'
14import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' 14import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url'
15import { sendDislike } from './send/send-dislike'
15 16
16async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { 17async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) {
17 let rateCounts = 0 18 let rateCounts = 0
@@ -37,19 +38,14 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa
37 38
38 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 39 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
39 40
40 const [ , created ] = await AccountVideoRateModel 41 const entry = {
41 .findOrCreate({ 42 videoId: video.id,
42 where: { 43 accountId: actor.Account.id,
43 videoId: video.id, 44 type: rate,
44 accountId: actor.Account.id 45 url: body.id
45 }, 46 }
46 defaults: { 47
47 videoId: video.id, 48 const created = await AccountVideoRateModel.upsert(entry)
48 accountId: actor.Account.id,
49 type: rate,
50 url: body.id
51 }
52 })
53 49
54 if (created) rateCounts += 1 50 if (created) rateCounts += 1
55 } catch (err) { 51 } catch (err) {
@@ -60,7 +56,10 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa
60 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) 56 logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
61 57
62 // This is "likes" and "dislikes" 58 // This is "likes" and "dislikes"
63 if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) 59 if (rateCounts !== 0) {
60 const field = rate === 'like' ? 'likes' : 'dislikes'
61 await video.increment(field, { by: rateCounts })
62 }
64 63
65 return 64 return
66} 65}
@@ -82,7 +81,7 @@ async function sendVideoRateChange (account: AccountModel,
82 // Like 81 // Like
83 if (likes > 0) await sendLike(actor, video, t) 82 if (likes > 0) await sendLike(actor, video, t)
84 // Dislike 83 // Dislike
85 if (dislikes > 0) await sendCreateDislike(actor, video, t) 84 if (dislikes > 0) await sendDislike(actor, video, t)
86} 85}
87 86
88function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { 87function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index e1e523499..4f26cb6be 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -2,15 +2,28 @@ import * as Bluebird from 'bluebird'
2import * as sequelize from 'sequelize' 2import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as request from 'request' 4import * as request from 'request'
5import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' 5import {
6 ActivityPlaylistSegmentHashesObject,
7 ActivityPlaylistUrlObject,
8 ActivityUrlObject,
9 ActivityVideoUrlObject,
10 VideoState
11} from '../../../shared/index'
6import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 12import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
7import { VideoPrivacy } from '../../../shared/models/videos' 13import { VideoPrivacy } from '../../../shared/models/videos'
8import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 14import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
9import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 15import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
10import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 16import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
11import { logger } from '../../helpers/logger' 17import { logger } from '../../helpers/logger'
12import { doRequest, downloadImage } from '../../helpers/requests' 18import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
13import { ACTIVITY_PUB, CONFIG, MIMETYPES, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers' 19import {
20 ACTIVITY_PUB,
21 MIMETYPES,
22 P2P_MEDIA_LOADER_PEER_VERSION,
23 PREVIEWS_SIZE,
24 REMOTE_SCHEME,
25 STATIC_PATHS
26} from '../../initializers/constants'
14import { ActorModel } from '../../models/activitypub/actor' 27import { ActorModel } from '../../models/activitypub/actor'
15import { TagModel } from '../../models/video/tag' 28import { TagModel } from '../../models/video/tag'
16import { VideoModel } from '../../models/video/video' 29import { VideoModel } from '../../models/video/video'
@@ -30,9 +43,20 @@ import { AccountModel } from '../../models/account/account'
30import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 43import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 44import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
32import { Notifier } from '../notifier' 45import { Notifier } from '../notifier'
46import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
47import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
48import { AccountVideoRateModel } from '../../models/account/account-video-rate'
49import { VideoShareModel } from '../../models/video/video-share'
50import { VideoCommentModel } from '../../models/video/video-comment'
51import { sequelizeTypescript } from '../../initializers/database'
52import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
53import { ThumbnailModel } from '../../models/video/thumbnail'
54import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
55import { join } from 'path'
56import { FilteredModelAttributes } from '../../typings/sequelize'
33 57
34async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 58async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
35 // If the video is not private and published, we federate it 59 // If the video is not private and is published, we federate it
36 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) { 60 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
37 // Fetch more attributes that we will need to serialize in AP object 61 // Fetch more attributes that we will need to serialize in AP object
38 if (isArray(video.VideoCaptions) === false) { 62 if (isArray(video.VideoCaptions) === false) {
@@ -84,19 +108,17 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
84 return body.description ? body.description : '' 108 return body.description ? body.description : ''
85} 109}
86 110
87function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { 111function fetchRemoteVideoStaticFile (video: VideoModel, path: string, destPath: string) {
88 const host = video.VideoChannel.Account.Actor.Server.host 112 const url = buildRemoteBaseUrl(video, path)
89 113
90 // We need to provide a callback, if no we could have an uncaught exception 114 // We need to provide a callback, if no we could have an uncaught exception
91 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { 115 return doRequestAndSaveToFile({ uri: url }, destPath)
92 if (err) reject(err)
93 })
94} 116}
95 117
96function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { 118function buildRemoteBaseUrl (video: VideoModel, path: string) {
97 const thumbnailName = video.getThumbnailName() 119 const host = video.VideoChannel.Account.Actor.Server.host
98 120
99 return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) 121 return REMOTE_SCHEME.HTTP + '://' + host + path
100} 122}
101 123
102function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 124function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
@@ -124,31 +146,43 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
124 const jobPayloads: ActivitypubHttpFetcherPayload[] = [] 146 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
125 147
126 if (syncParam.likes === true) { 148 if (syncParam.likes === true) {
127 await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like')) 149 const handler = items => createRates(items, video, 'like')
150 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
151
152 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
128 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) 153 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
129 } else { 154 } else {
130 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) 155 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
131 } 156 }
132 157
133 if (syncParam.dislikes === true) { 158 if (syncParam.dislikes === true) {
134 await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike')) 159 const handler = items => createRates(items, video, 'dislike')
160 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
161
162 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
135 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) 163 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
136 } else { 164 } else {
137 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) 165 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
138 } 166 }
139 167
140 if (syncParam.shares === true) { 168 if (syncParam.shares === true) {
141 await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video)) 169 const handler = items => addVideoShares(items, video)
170 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
171
172 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
142 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) 173 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
143 } else { 174 } else {
144 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) 175 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
145 } 176 }
146 177
147 if (syncParam.comments === true) { 178 if (syncParam.comments === true) {
148 await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video)) 179 const handler = items => addVideoComments(items, video)
180 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
181
182 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
149 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) 183 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
150 } else { 184 } else {
151 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) 185 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
152 } 186 }
153 187
154 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) 188 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
@@ -170,8 +204,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
170 204
171 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) 205 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
172 if (videoFromDatabase) { 206 if (videoFromDatabase) {
173 207 if (videoFromDatabase.isOutdated() && allowRefresh === true) {
174 if (allowRefresh === true) {
175 const refreshOptions = { 208 const refreshOptions = {
176 video: videoFromDatabase, 209 video: videoFromDatabase,
177 fetchedType: fetchType, 210 fetchedType: fetchType,
@@ -210,6 +243,14 @@ async function updateVideoFromAP (options: {
210 const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED 243 const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED
211 244
212 try { 245 try {
246 let thumbnailModel: ThumbnailModel
247
248 try {
249 thumbnailModel = await createVideoMiniatureFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.MINIATURE)
250 } catch (err) {
251 logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
252 }
253
213 await sequelizeTypescript.transaction(async t => { 254 await sequelizeTypescript.transaction(async t => {
214 const sequelizeOptions = { transaction: t } 255 const sequelizeOptions = { transaction: t }
215 256
@@ -233,17 +274,26 @@ async function updateVideoFromAP (options: {
233 options.video.set('support', videoData.support) 274 options.video.set('support', videoData.support)
234 options.video.set('nsfw', videoData.nsfw) 275 options.video.set('nsfw', videoData.nsfw)
235 options.video.set('commentsEnabled', videoData.commentsEnabled) 276 options.video.set('commentsEnabled', videoData.commentsEnabled)
277 options.video.set('downloadEnabled', videoData.downloadEnabled)
236 options.video.set('waitTranscoding', videoData.waitTranscoding) 278 options.video.set('waitTranscoding', videoData.waitTranscoding)
237 options.video.set('state', videoData.state) 279 options.video.set('state', videoData.state)
238 options.video.set('duration', videoData.duration) 280 options.video.set('duration', videoData.duration)
239 options.video.set('createdAt', videoData.createdAt) 281 options.video.set('createdAt', videoData.createdAt)
240 options.video.set('publishedAt', videoData.publishedAt) 282 options.video.set('publishedAt', videoData.publishedAt)
283 options.video.set('originallyPublishedAt', videoData.originallyPublishedAt)
241 options.video.set('privacy', videoData.privacy) 284 options.video.set('privacy', videoData.privacy)
242 options.video.set('channelId', videoData.channelId) 285 options.video.set('channelId', videoData.channelId)
243 options.video.set('views', videoData.views) 286 options.video.set('views', videoData.views)
244 287
245 await options.video.save(sequelizeOptions) 288 await options.video.save(sequelizeOptions)
246 289
290 if (thumbnailModel) if (thumbnailModel) await options.video.addAndSaveThumbnail(thumbnailModel, t)
291
292 // FIXME: use icon URL instead
293 const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename))
294 const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
295 await options.video.addAndSaveThumbnail(previewModel, t)
296
247 { 297 {
248 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) 298 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
249 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) 299 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
@@ -264,6 +314,29 @@ async function updateVideoFromAP (options: {
264 } 314 }
265 315
266 { 316 {
317 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
318 options.video,
319 options.videoObject,
320 options.video.VideoFiles
321 )
322 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
323
324 // Remove video files that do not exist anymore
325 const destroyTasks = options.video.VideoStreamingPlaylists
326 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
327 .map(f => f.destroy(sequelizeOptions))
328 await Promise.all(destroyTasks)
329
330 // Update or add other one
331 const upsertTasks = streamingPlaylistAttributes.map(a => {
332 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
333 .then(([ streamingPlaylist ]) => streamingPlaylist)
334 })
335
336 options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
337 }
338
339 {
267 // Update Tags 340 // Update Tags
268 const tags = options.videoObject.tag.map(tag => tag.name) 341 const tags = options.videoObject.tag.map(tag => tag.name)
269 const tagInstances = await TagModel.findOrCreateTags(tags, t) 342 const tagInstances = await TagModel.findOrCreateTags(tags, t)
@@ -296,12 +369,6 @@ async function updateVideoFromAP (options: {
296 logger.debug('Cannot update the remote video.', { err }) 369 logger.debug('Cannot update the remote video.', { err })
297 throw err 370 throw err
298 } 371 }
299
300 try {
301 await generateThumbnailFromUrl(options.video, options.videoObject.icon)
302 } catch (err) {
303 logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
304 }
305} 372}
306 373
307async function refreshVideoIfNeeded (options: { 374async function refreshVideoIfNeeded (options: {
@@ -361,29 +428,55 @@ export {
361 getOrCreateVideoAndAccountAndChannel, 428 getOrCreateVideoAndAccountAndChannel,
362 fetchRemoteVideoStaticFile, 429 fetchRemoteVideoStaticFile,
363 fetchRemoteVideoDescription, 430 fetchRemoteVideoDescription,
364 generateThumbnailFromUrl,
365 getOrCreateVideoChannelFromVideoObject 431 getOrCreateVideoChannelFromVideoObject
366} 432}
367 433
368// --------------------------------------------------------------------------- 434// ---------------------------------------------------------------------------
369 435
370function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { 436function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
371 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 437 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
372 438
373 const urlMediaType = url.mediaType || url.mimeType 439 const urlMediaType = url.mediaType || url.mimeType
374 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') 440 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
375} 441}
376 442
443function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
444 const urlMediaType = url.mediaType || url.mimeType
445
446 return urlMediaType === 'application/x-mpegURL'
447}
448
449function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
450 const urlMediaType = tag.mediaType || tag.mimeType
451
452 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
453}
454
377async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 455async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
378 logger.debug('Adding remote video %s.', videoObject.id) 456 logger.debug('Adding remote video %s.', videoObject.id)
379 457
458 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
459 const video = VideoModel.build(videoData)
460
461 const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
462
463 let thumbnailModel: ThumbnailModel
464 if (waitThumbnail === true) {
465 thumbnailModel = await promiseThumbnail
466 }
467
380 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { 468 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
381 const sequelizeOptions = { transaction: t } 469 const sequelizeOptions = { transaction: t }
382 470
383 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
384 const video = VideoModel.build(videoData)
385
386 const videoCreated = await video.save(sequelizeOptions) 471 const videoCreated = await video.save(sequelizeOptions)
472 videoCreated.VideoChannel = channelActor.VideoChannel
473
474 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
475
476 // FIXME: use icon URL instead
477 const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
478 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
479 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
387 480
388 // Process files 481 // Process files
389 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) 482 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
@@ -392,10 +485,16 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
392 } 485 }
393 486
394 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 487 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
395 await Promise.all(videoFilePromises) 488 const videoFiles = await Promise.all(videoFilePromises)
489
490 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
491 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
492 await Promise.all(playlistPromises)
396 493
397 // Process tags 494 // Process tags
398 const tags = videoObject.tag.map(t => t.name) 495 const tags = videoObject.tag
496 .filter(t => t.type === 'Hashtag')
497 .map(t => t.name)
399 const tagInstances = await TagModel.findOrCreateTags(tags, t) 498 const tagInstances = await TagModel.findOrCreateTags(tags, t)
400 await videoCreated.$set('Tags', tagInstances, sequelizeOptions) 499 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
401 500
@@ -407,14 +506,16 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
407 506
408 logger.info('Remote video with uuid %s inserted.', videoObject.uuid) 507 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
409 508
410 videoCreated.VideoChannel = channelActor.VideoChannel
411 return videoCreated 509 return videoCreated
412 }) 510 })
413 511
414 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) 512 if (waitThumbnail === false) {
415 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) 513 promiseThumbnail.then(thumbnailModel => {
514 thumbnailModel = videoCreated.id
416 515
417 if (waitThumbnail === true) await p 516 return thumbnailModel.save()
517 })
518 }
418 519
419 return videoCreated 520 return videoCreated
420} 521}
@@ -456,12 +557,14 @@ async function videoActivityObjectToDBAttributes (
456 support, 557 support,
457 nsfw: videoObject.sensitive, 558 nsfw: videoObject.sensitive,
458 commentsEnabled: videoObject.commentsEnabled, 559 commentsEnabled: videoObject.commentsEnabled,
560 downloadEnabled: videoObject.downloadEnabled,
459 waitTranscoding: videoObject.waitTranscoding, 561 waitTranscoding: videoObject.waitTranscoding,
460 state: videoObject.state, 562 state: videoObject.state,
461 channelId: videoChannel.id, 563 channelId: videoChannel.id,
462 duration: parseInt(duration, 10), 564 duration: parseInt(duration, 10),
463 createdAt: new Date(videoObject.published), 565 createdAt: new Date(videoObject.published),
464 publishedAt: new Date(videoObject.published), 566 publishedAt: new Date(videoObject.published),
567 originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
465 // FIXME: updatedAt does not seems to be considered by Sequelize 568 // FIXME: updatedAt does not seems to be considered by Sequelize
466 updatedAt: new Date(videoObject.updated), 569 updatedAt: new Date(videoObject.updated),
467 views: videoObject.views, 570 views: videoObject.views,
@@ -473,13 +576,13 @@ async function videoActivityObjectToDBAttributes (
473} 576}
474 577
475function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { 578function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
476 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] 579 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
477 580
478 if (fileUrls.length === 0) { 581 if (fileUrls.length === 0) {
479 throw new Error('Cannot find video files for ' + video.url) 582 throw new Error('Cannot find video files for ' + video.url)
480 } 583 }
481 584
482 const attributes: VideoFileModel[] = [] 585 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
483 for (const fileUrl of fileUrls) { 586 for (const fileUrl of fileUrls) {
484 // Fetch associated magnet uri 587 // Fetch associated magnet uri
485 const magnet = videoObject.url.find(u => { 588 const magnet = videoObject.url.find(u => {
@@ -502,7 +605,38 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
502 size: fileUrl.size, 605 size: fileUrl.size,
503 videoId: video.id, 606 videoId: video.id,
504 fps: fileUrl.fps || -1 607 fps: fileUrl.fps || -1
505 } as VideoFileModel 608 }
609
610 attributes.push(attribute)
611 }
612
613 return attributes
614}
615
616function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
617 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
618 if (playlistUrls.length === 0) return []
619
620 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
621 for (const playlistUrlObject of playlistUrls) {
622 const segmentsSha256UrlObject = playlistUrlObject.tag
623 .find(t => {
624 return isAPPlaylistSegmentHashesUrlObject(t)
625 }) as ActivityPlaylistSegmentHashesObject
626 if (!segmentsSha256UrlObject) {
627 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
628 continue
629 }
630
631 const attribute = {
632 type: VideoStreamingPlaylistType.HLS,
633 playlistUrl: playlistUrlObject.href,
634 segmentsSha256Url: segmentsSha256UrlObject.href,
635 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
636 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
637 videoId: video.id
638 }
639
506 attributes.push(attribute) 640 attributes.push(attribute)
507 } 641 }
508 642
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 021426a1a..09b4e38ca 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -1,6 +1,6 @@
1import 'multer' 1import 'multer'
2import { sendUpdateActor } from './activitypub/send' 2import { sendUpdateActor } from './activitypub/send'
3import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' 3import { AVATARS_SIZE } from '../initializers/constants'
4import { updateActorAvatarInstance } from './activitypub' 4import { updateActorAvatarInstance } from './activitypub'
5import { processImage } from '../helpers/image-utils' 5import { processImage } from '../helpers/image-utils'
6import { AccountModel } from '../models/account/account' 6import { AccountModel } from '../models/account/account'
@@ -8,12 +8,14 @@ import { VideoChannelModel } from '../models/video/video-channel'
8import { extname, join } from 'path' 8import { extname, join } from 'path'
9import { retryTransactionWrapper } from '../helpers/database-utils' 9import { retryTransactionWrapper } from '../helpers/database-utils'
10import * as uuidv4 from 'uuid/v4' 10import * as uuidv4 from 'uuid/v4'
11import { CONFIG } from '../initializers/config'
12import { sequelizeTypescript } from '../initializers/database'
11 13
12async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { 14async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
13 const extension = extname(avatarPhysicalFile.filename) 15 const extension = extname(avatarPhysicalFile.filename)
14 const avatarName = uuidv4() + extension 16 const avatarName = uuidv4() + extension
15 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) 17 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
16 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) 18 await processImage(avatarPhysicalFile.path, destination, AVATARS_SIZE)
17 19
18 return retryTransactionWrapper(() => { 20 return retryTransactionWrapper(() => {
19 return sequelizeTypescript.transaction(async t => { 21 return sequelizeTypescript.transaction(async t => {
diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/cache/abstract-video-static-file-cache.ts
deleted file mode 100644
index 7512f2b9d..000000000
--- a/server/lib/cache/abstract-video-static-file-cache.ts
+++ /dev/null
@@ -1,52 +0,0 @@
1import * as AsyncLRU from 'async-lru'
2import { createWriteStream, remove } from 'fs-extra'
3import { logger } from '../../helpers/logger'
4import { VideoModel } from '../../models/video/video'
5import { fetchRemoteVideoStaticFile } from '../activitypub'
6
7export abstract class AbstractVideoStaticFileCache <T> {
8
9 protected lru
10
11 abstract getFilePath (params: T): Promise<string>
12
13 // Load and save the remote file, then return the local path from filesystem
14 protected abstract loadRemoteFile (key: string): Promise<string>
15
16 init (max: number, maxAge: number) {
17 this.lru = new AsyncLRU({
18 max,
19 maxAge,
20 load: (key, cb) => {
21 this.loadRemoteFile(key)
22 .then(res => cb(null, res))
23 .catch(err => cb(err))
24 }
25 })
26
27 this.lru.on('evict', (obj: { key: string, value: string }) => {
28 remove(obj.value)
29 .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name))
30 })
31 }
32
33 protected loadFromLRU (key: string) {
34 return new Promise<string>((res, rej) => {
35 this.lru.get(key, (err, value) => {
36 err ? rej(err) : res(value)
37 })
38 })
39 }
40
41 protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) {
42 return new Promise<string>((res, rej) => {
43 const req = fetchRemoteVideoStaticFile(video, remoteStaticPath, rej)
44
45 const stream = createWriteStream(destPath)
46
47 req.pipe(stream)
48 .on('error', (err) => rej(err))
49 .on('finish', () => res(destPath))
50 })
51 }
52}
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index b2c376e20..516827a05 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -1,7 +1,6 @@
1import * as express from 'express' 1import * as express from 'express'
2import * as Bluebird from 'bluebird'
3import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' 2import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n'
4import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers' 3import { CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, WEBSERVER } from '../initializers/constants'
5import { join } from 'path' 4import { join } from 'path'
6import { escapeHTML } from '../helpers/core-utils' 5import { escapeHTML } from '../helpers/core-utils'
7import { VideoModel } from '../models/video/video' 6import { VideoModel } from '../models/video/video'
@@ -9,10 +8,14 @@ import * as validator from 'validator'
9import { VideoPrivacy } from '../../shared/models/videos' 8import { VideoPrivacy } from '../../shared/models/videos'
10import { readFile } from 'fs-extra' 9import { readFile } from 'fs-extra'
11import { getActivityStreamDuration } from '../models/video/video-format-utils' 10import { getActivityStreamDuration } from '../models/video/video-format-utils'
11import { AccountModel } from '../models/account/account'
12import { VideoChannelModel } from '../models/video/video-channel'
13import * as Bluebird from 'bluebird'
14import { CONFIG } from '../initializers/config'
12 15
13export class ClientHtml { 16export class ClientHtml {
14 17
15 private static htmlCache: { [path: string]: string } = {} 18 private static htmlCache: { [ path: string ]: string } = {}
16 19
17 static invalidCache () { 20 static invalidCache () {
18 ClientHtml.htmlCache = {} 21 ClientHtml.htmlCache = {}
@@ -28,18 +31,14 @@ export class ClientHtml {
28 } 31 }
29 32
30 static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { 33 static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) {
31 let videoPromise: Bluebird<VideoModel>
32
33 // Let Angular application handle errors 34 // Let Angular application handle errors
34 if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) { 35 if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) {
35 videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
36 } else {
37 return ClientHtml.getIndexHTML(req, res) 36 return ClientHtml.getIndexHTML(req, res)
38 } 37 }
39 38
40 const [ html, video ] = await Promise.all([ 39 const [ html, video ] = await Promise.all([
41 ClientHtml.getIndexHTML(req, res), 40 ClientHtml.getIndexHTML(req, res),
42 videoPromise 41 VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
43 ]) 42 ])
44 43
45 // Let Angular application handle errors 44 // Let Angular application handle errors
@@ -49,14 +48,44 @@ export class ClientHtml {
49 48
50 let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name)) 49 let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name))
51 customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description)) 50 customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description))
52 customHtml = ClientHtml.addOpenGraphAndOEmbedTags(customHtml, video) 51 customHtml = ClientHtml.addVideoOpenGraphAndOEmbedTags(customHtml, video)
52
53 return customHtml
54 }
55
56 static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) {
57 return this.getAccountOrChannelHTMLPage(() => AccountModel.loadByNameWithHost(nameWithHost), req, res)
58 }
59
60 static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) {
61 return this.getAccountOrChannelHTMLPage(() => VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost), req, res)
62 }
63
64 private static async getAccountOrChannelHTMLPage (
65 loader: () => Bluebird<AccountModel | VideoChannelModel>,
66 req: express.Request,
67 res: express.Response
68 ) {
69 const [ html, entity ] = await Promise.all([
70 ClientHtml.getIndexHTML(req, res),
71 loader()
72 ])
73
74 // Let Angular application handle errors
75 if (!entity) {
76 return ClientHtml.getIndexHTML(req, res)
77 }
78
79 let customHtml = ClientHtml.addTitleTag(html, escapeHTML(entity.getDisplayName()))
80 customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(entity.description))
81 customHtml = ClientHtml.addAccountOrChannelMetaTags(customHtml, entity)
53 82
54 return customHtml 83 return customHtml
55 } 84 }
56 85
57 private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { 86 private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) {
58 const path = ClientHtml.getIndexPath(req, res, paramLang) 87 const path = ClientHtml.getIndexPath(req, res, paramLang)
59 if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] 88 if (ClientHtml.htmlCache[ path ]) return ClientHtml.htmlCache[ path ]
60 89
61 const buffer = await readFile(path) 90 const buffer = await readFile(path)
62 91
@@ -64,7 +93,7 @@ export class ClientHtml {
64 93
65 html = ClientHtml.addCustomCSS(html) 94 html = ClientHtml.addCustomCSS(html)
66 95
67 ClientHtml.htmlCache[path] = html 96 ClientHtml.htmlCache[ path ] = html
68 97
69 return html 98 return html
70 } 99 }
@@ -78,7 +107,7 @@ export class ClientHtml {
78 107
79 // Save locale in cookies 108 // Save locale in cookies
80 res.cookie('clientLanguage', lang, { 109 res.cookie('clientLanguage', lang, {
81 secure: CONFIG.WEBSERVER.SCHEME === 'https', 110 secure: WEBSERVER.SCHEME === 'https',
82 sameSite: true, 111 sameSite: true,
83 maxAge: 1000 * 3600 * 24 * 90 // 3 months 112 maxAge: 1000 * 3600 * 24 * 90 // 3 months
84 }) 113 })
@@ -114,13 +143,13 @@ export class ClientHtml {
114 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) 143 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag)
115 } 144 }
116 145
117 private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { 146 private static addVideoOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) {
118 const previewUrl = CONFIG.WEBSERVER.URL + video.getPreviewStaticPath() 147 const previewUrl = WEBSERVER.URL + video.getPreviewStaticPath()
119 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() 148 const videoUrl = WEBSERVER.URL + video.getWatchStaticPath()
120 149
121 const videoNameEscaped = escapeHTML(video.name) 150 const videoNameEscaped = escapeHTML(video.name)
122 const videoDescriptionEscaped = escapeHTML(video.description) 151 const videoDescriptionEscaped = escapeHTML(video.description)
123 const embedUrl = CONFIG.WEBSERVER.URL + video.getEmbedStaticPath() 152 const embedUrl = WEBSERVER.URL + video.getEmbedStaticPath()
124 153
125 const openGraphMetaTags = { 154 const openGraphMetaTags = {
126 'og:type': 'video', 155 'og:type': 'video',
@@ -152,7 +181,7 @@ export class ClientHtml {
152 const oembedLinkTags = [ 181 const oembedLinkTags = [
153 { 182 {
154 type: 'application/json+oembed', 183 type: 'application/json+oembed',
155 href: CONFIG.WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(videoUrl), 184 href: WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(videoUrl),
156 title: videoNameEscaped 185 title: videoNameEscaped
157 } 186 }
158 ] 187 ]
@@ -174,7 +203,7 @@ export class ClientHtml {
174 203
175 // Opengraph 204 // Opengraph
176 Object.keys(openGraphMetaTags).forEach(tagName => { 205 Object.keys(openGraphMetaTags).forEach(tagName => {
177 const tagValue = openGraphMetaTags[tagName] 206 const tagValue = openGraphMetaTags[ tagName ]
178 207
179 tagsString += `<meta property="${tagName}" content="${tagValue}" />` 208 tagsString += `<meta property="${tagName}" content="${tagValue}" />`
180 }) 209 })
@@ -190,6 +219,17 @@ export class ClientHtml {
190 // SEO, use origin video url so Google does not index remote videos 219 // SEO, use origin video url so Google does not index remote videos
191 tagsString += `<link rel="canonical" href="${video.url}" />` 220 tagsString += `<link rel="canonical" href="${video.url}" />`
192 221
193 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) 222 return this.addOpenGraphAndOEmbedTags(htmlStringPage, tagsString)
223 }
224
225 private static addAccountOrChannelMetaTags (htmlStringPage: string, entity: AccountModel | VideoChannelModel) {
226 // SEO, use origin account or channel URL
227 const metaTags = `<link rel="canonical" href="${entity.Actor.url}" />`
228
229 return this.addOpenGraphAndOEmbedTags(htmlStringPage, metaTags)
230 }
231
232 private static addOpenGraphAndOEmbedTags (htmlStringPage: string, metaTags: string) {
233 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.META_TAGS, metaTags)
194 } 234 }
195} 235}
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts
index 99010f473..8c06e9751 100644
--- a/server/lib/emailer.ts
+++ b/server/lib/emailer.ts
@@ -1,7 +1,7 @@
1import { createTransport, Transporter } from 'nodemailer' 1import { createTransport, Transporter } from 'nodemailer'
2import { isTestInstance } from '../helpers/core-utils' 2import { isTestInstance } from '../helpers/core-utils'
3import { bunyanLogger, logger } from '../helpers/logger' 3import { bunyanLogger, logger } from '../helpers/logger'
4import { CONFIG } from '../initializers' 4import { CONFIG } from '../initializers/config'
5import { UserModel } from '../models/account/user' 5import { UserModel } from '../models/account/user'
6import { VideoModel } from '../models/video/video' 6import { VideoModel } from '../models/video/video'
7import { JobQueue } from './job-queue' 7import { JobQueue } from './job-queue'
@@ -12,6 +12,16 @@ import { VideoAbuseModel } from '../models/video/video-abuse'
12import { VideoBlacklistModel } from '../models/video/video-blacklist' 12import { VideoBlacklistModel } from '../models/video/video-blacklist'
13import { VideoImportModel } from '../models/video/video-import' 13import { VideoImportModel } from '../models/video/video-import'
14import { ActorFollowModel } from '../models/activitypub/actor-follow' 14import { ActorFollowModel } from '../models/activitypub/actor-follow'
15import { WEBSERVER } from '../initializers/constants'
16
17type SendEmailOptions = {
18 to: string[]
19 subject: string
20 text: string
21
22 fromDisplayName?: string
23 replyTo?: string
24}
15 25
16class Emailer { 26class Emailer {
17 27
@@ -82,7 +92,7 @@ class Emailer {
82 92
83 addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) { 93 addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) {
84 const channelName = video.VideoChannel.getDisplayName() 94 const channelName = video.VideoChannel.getDisplayName()
85 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() 95 const videoUrl = WEBSERVER.URL + video.getWatchStaticPath()
86 96
87 const text = `Hi dear user,\n\n` + 97 const text = `Hi dear user,\n\n` +
88 `Your subscription ${channelName} just published a new video: ${video.name}` + 98 `Your subscription ${channelName} just published a new video: ${video.name}` +
@@ -120,8 +130,26 @@ class Emailer {
120 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 130 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
121 } 131 }
122 132
133 addNewInstanceFollowerNotification (to: string[], actorFollow: ActorFollowModel) {
134 const awaitingApproval = actorFollow.state === 'pending' ? ' awaiting manual approval.' : ''
135
136 const text = `Hi dear admin,\n\n` +
137 `Your instance has a new follower: ${actorFollow.ActorFollower.url}${awaitingApproval}` +
138 `\n\n` +
139 `Cheers,\n` +
140 `PeerTube.`
141
142 const emailPayload: EmailPayload = {
143 to,
144 subject: 'New instance follower',
145 text
146 }
147
148 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
149 }
150
123 myVideoPublishedNotification (to: string[], video: VideoModel) { 151 myVideoPublishedNotification (to: string[], video: VideoModel) {
124 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() 152 const videoUrl = WEBSERVER.URL + video.getWatchStaticPath()
125 153
126 const text = `Hi dear user,\n\n` + 154 const text = `Hi dear user,\n\n` +
127 `Your video ${video.name} has been published.` + 155 `Your video ${video.name} has been published.` +
@@ -141,7 +169,7 @@ class Emailer {
141 } 169 }
142 170
143 myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) { 171 myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) {
144 const videoUrl = CONFIG.WEBSERVER.URL + videoImport.Video.getWatchStaticPath() 172 const videoUrl = WEBSERVER.URL + videoImport.Video.getWatchStaticPath()
145 173
146 const text = `Hi dear user,\n\n` + 174 const text = `Hi dear user,\n\n` +
147 `Your video import ${videoImport.getTargetIdentifier()} is finished.` + 175 `Your video import ${videoImport.getTargetIdentifier()} is finished.` +
@@ -161,7 +189,7 @@ class Emailer {
161 } 189 }
162 190
163 myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) { 191 myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) {
164 const importUrl = CONFIG.WEBSERVER.URL + '/my-account/video-imports' 192 const importUrl = WEBSERVER.URL + '/my-account/video-imports'
165 193
166 const text = `Hi dear user,\n\n` + 194 const text = `Hi dear user,\n\n` +
167 `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` + 195 `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` +
@@ -183,7 +211,7 @@ class Emailer {
183 addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) { 211 addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) {
184 const accountName = comment.Account.getDisplayName() 212 const accountName = comment.Account.getDisplayName()
185 const video = comment.Video 213 const video = comment.Video
186 const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() 214 const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath()
187 215
188 const text = `Hi dear user,\n\n` + 216 const text = `Hi dear user,\n\n` +
189 `A new comment has been posted by ${accountName} on your video ${video.name}` + 217 `A new comment has been posted by ${accountName} on your video ${video.name}` +
@@ -205,7 +233,7 @@ class Emailer {
205 addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) { 233 addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) {
206 const accountName = comment.Account.getDisplayName() 234 const accountName = comment.Account.getDisplayName()
207 const video = comment.Video 235 const video = comment.Video
208 const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() 236 const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath()
209 237
210 const text = `Hi dear user,\n\n` + 238 const text = `Hi dear user,\n\n` +
211 `${accountName} mentioned you on video ${video.name}` + 239 `${accountName} mentioned you on video ${video.name}` +
@@ -225,10 +253,10 @@ class Emailer {
225 } 253 }
226 254
227 addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { 255 addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) {
228 const videoUrl = CONFIG.WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath() 256 const videoUrl = WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath()
229 257
230 const text = `Hi,\n\n` + 258 const text = `Hi,\n\n` +
231 `${CONFIG.WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` + 259 `${WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` +
232 `Cheers,\n` + 260 `Cheers,\n` +
233 `PeerTube.` 261 `PeerTube.`
234 262
@@ -241,15 +269,38 @@ class Emailer {
241 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 269 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
242 } 270 }
243 271
272 addVideoAutoBlacklistModeratorsNotification (to: string[], video: VideoModel) {
273 const VIDEO_AUTO_BLACKLIST_URL = WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list'
274 const videoUrl = WEBSERVER.URL + video.getWatchStaticPath()
275
276 const text = `Hi,\n\n` +
277 `A recently added video was auto-blacklisted and requires moderator review before publishing.` +
278 `\n\n` +
279 `You can view it and take appropriate action on ${videoUrl}` +
280 `\n\n` +
281 `A full list of auto-blacklisted videos can be reviewed here: ${VIDEO_AUTO_BLACKLIST_URL}` +
282 `\n\n` +
283 `Cheers,\n` +
284 `PeerTube.`
285
286 const emailPayload: EmailPayload = {
287 to,
288 subject: '[PeerTube] An auto-blacklisted video is awaiting review',
289 text
290 }
291
292 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
293 }
294
244 addNewUserRegistrationNotification (to: string[], user: UserModel) { 295 addNewUserRegistrationNotification (to: string[], user: UserModel) {
245 const text = `Hi,\n\n` + 296 const text = `Hi,\n\n` +
246 `User ${user.username} just registered on ${CONFIG.WEBSERVER.HOST} PeerTube instance.\n\n` + 297 `User ${user.username} just registered on ${WEBSERVER.HOST} PeerTube instance.\n\n` +
247 `Cheers,\n` + 298 `Cheers,\n` +
248 `PeerTube.` 299 `PeerTube.`
249 300
250 const emailPayload: EmailPayload = { 301 const emailPayload: EmailPayload = {
251 to, 302 to,
252 subject: '[PeerTube] New user registration on ' + CONFIG.WEBSERVER.HOST, 303 subject: '[PeerTube] New user registration on ' + WEBSERVER.HOST,
253 text 304 text
254 } 305 }
255 306
@@ -258,10 +309,10 @@ class Emailer {
258 309
259 addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { 310 addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) {
260 const videoName = videoBlacklist.Video.name 311 const videoName = videoBlacklist.Video.name
261 const videoUrl = CONFIG.WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath() 312 const videoUrl = WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath()
262 313
263 const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : '' 314 const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : ''
264 const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` 315 const blockedString = `Your video ${videoName} (${videoUrl} on ${WEBSERVER.HOST} has been blacklisted${reasonString}.`
265 316
266 const text = 'Hi,\n\n' + 317 const text = 'Hi,\n\n' +
267 blockedString + 318 blockedString +
@@ -279,10 +330,10 @@ class Emailer {
279 } 330 }
280 331
281 addVideoUnblacklistNotification (to: string[], video: VideoModel) { 332 addVideoUnblacklistNotification (to: string[], video: VideoModel) {
282 const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() 333 const videoUrl = WEBSERVER.URL + video.getWatchStaticPath()
283 334
284 const text = 'Hi,\n\n' + 335 const text = 'Hi,\n\n' +
285 `Your video ${video.name} (${videoUrl}) on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + 336 `Your video ${video.name} (${videoUrl}) on ${WEBSERVER.HOST} has been unblacklisted.` +
286 '\n\n' + 337 '\n\n' +
287 'Cheers,\n' + 338 'Cheers,\n' +
288 `PeerTube.` 339 `PeerTube.`
@@ -296,9 +347,9 @@ class Emailer {
296 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) 347 return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload })
297 } 348 }
298 349
299 addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { 350 addPasswordResetEmailJob (to: string, resetPasswordUrl: string) {
300 const text = `Hi dear user,\n\n` + 351 const text = `Hi dear user,\n\n` +
301 `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + 352 `A reset password procedure for your account ${to} has been requested on ${WEBSERVER.HOST} ` +
302 `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + 353 `Please follow this link to reset it: ${resetPasswordUrl}\n\n` +
303 `If you are not the person who initiated this request, please ignore this email.\n\n` + 354 `If you are not the person who initiated this request, please ignore this email.\n\n` +
304 `Cheers,\n` + 355 `Cheers,\n` +
@@ -315,7 +366,7 @@ class Emailer {
315 366
316 addVerifyEmailJob (to: string, verifyEmailUrl: string) { 367 addVerifyEmailJob (to: string, verifyEmailUrl: string) {
317 const text = `Welcome to PeerTube,\n\n` + 368 const text = `Welcome to PeerTube,\n\n` +
318 `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + 369 `To start using PeerTube on ${WEBSERVER.HOST} you must verify your email! ` +
319 `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + 370 `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` +
320 `If you are not the person who initiated this request, please ignore this email.\n\n` + 371 `If you are not the person who initiated this request, please ignore this email.\n\n` +
321 `Cheers,\n` + 372 `Cheers,\n` +
@@ -333,7 +384,7 @@ class Emailer {
333 addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { 384 addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) {
334 const reasonString = reason ? ` for the following reason: ${reason}` : '' 385 const reasonString = reason ? ` for the following reason: ${reason}` : ''
335 const blockedWord = blocked ? 'blocked' : 'unblocked' 386 const blockedWord = blocked ? 'blocked' : 'unblocked'
336 const blockedString = `Your account ${user.username} on ${CONFIG.WEBSERVER.HOST} has been ${blockedWord}${reasonString}.` 387 const blockedString = `Your account ${user.username} on ${WEBSERVER.HOST} has been ${blockedWord}${reasonString}.`
337 388
338 const text = 'Hi,\n\n' + 389 const text = 'Hi,\n\n' +
339 blockedString + 390 blockedString +
@@ -378,7 +429,7 @@ class Emailer {
378 429
379 const fromDisplayName = options.fromDisplayName 430 const fromDisplayName = options.fromDisplayName
380 ? options.fromDisplayName 431 ? options.fromDisplayName
381 : CONFIG.WEBSERVER.HOST 432 : WEBSERVER.HOST
382 433
383 return this.transporter.sendMail({ 434 return this.transporter.sendMail({
384 from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>`, 435 from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>`,
@@ -402,5 +453,6 @@ class Emailer {
402// --------------------------------------------------------------------------- 453// ---------------------------------------------------------------------------
403 454
404export { 455export {
405 Emailer 456 Emailer,
457 SendEmailOptions
406} 458}
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts
new file mode 100644
index 000000000..1908cfb06
--- /dev/null
+++ b/server/lib/files-cache/abstract-video-static-file-cache.ts
@@ -0,0 +1,30 @@
1import { remove } from 'fs-extra'
2import { logger } from '../../helpers/logger'
3import * as memoizee from 'memoizee'
4
5type GetFilePathResult = { isOwned: boolean, path: string } | undefined
6
7export abstract class AbstractVideoStaticFileCache <T> {
8
9 getFilePath: (params: T) => Promise<GetFilePathResult>
10
11 abstract getFilePathImpl (params: T): Promise<GetFilePathResult>
12
13 // Load and save the remote file, then return the local path from filesystem
14 protected abstract loadRemoteFile (key: string): Promise<GetFilePathResult>
15
16 init (max: number, maxAge: number) {
17 this.getFilePath = memoizee(this.getFilePathImpl, {
18 maxAge,
19 max,
20 promise: true,
21 dispose: (result: GetFilePathResult) => {
22 if (result.isOwned !== true) {
23 remove(result.path)
24 .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name))
25 .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err }))
26 }
27 }
28 })
29 }
30}
diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/files-cache/actor-follow-score-cache.ts
index d070bde09..5f8ee806f 100644
--- a/server/lib/cache/actor-follow-score-cache.ts
+++ b/server/lib/files-cache/actor-follow-score-cache.ts
@@ -1,4 +1,4 @@
1import { ACTOR_FOLLOW_SCORE } from '../../initializers' 1import { ACTOR_FOLLOW_SCORE } from '../../initializers/constants'
2import { logger } from '../../helpers/logger' 2import { logger } from '../../helpers/logger'
3 3
4// Cache follows scores, instead of writing them too often in database 4// Cache follows scores, instead of writing them too often in database
diff --git a/server/lib/cache/index.ts b/server/lib/files-cache/index.ts
index e921d04a7..e921d04a7 100644
--- a/server/lib/cache/index.ts
+++ b/server/lib/files-cache/index.ts
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
index f240affbc..440c3fde8 100644
--- a/server/lib/cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -1,8 +1,11 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CACHE, CONFIG } from '../../initializers' 2import { FILES_CACHE } from '../../initializers/constants'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { VideoCaptionModel } from '../../models/video/video-caption' 4import { VideoCaptionModel } from '../../models/video/video-caption'
5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 5import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
6import { CONFIG } from '../../initializers/config'
7import { logger } from '../../helpers/logger'
8import { fetchRemoteVideoStaticFile } from '../activitypub'
6 9
7type GetPathParam = { videoId: string, language: string } 10type GetPathParam = { videoId: string, language: string }
8 11
@@ -19,17 +22,19 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
19 return this.instance || (this.instance = new this()) 22 return this.instance || (this.instance = new this())
20 } 23 }
21 24
22 async getFilePath (params: GetPathParam) { 25 async getFilePathImpl (params: GetPathParam) {
23 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language) 26 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language)
24 if (!videoCaption) return undefined 27 if (!videoCaption) return undefined
25 28
26 if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) 29 if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) }
27 30
28 const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language 31 const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language
29 return this.loadFromLRU(key) 32 return this.loadRemoteFile(key)
30 } 33 }
31 34
32 protected async loadRemoteFile (key: string) { 35 protected async loadRemoteFile (key: string) {
36 logger.debug('Loading remote caption file %s.', key)
37
33 const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER) 38 const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER)
34 39
35 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language) 40 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language)
@@ -41,10 +46,13 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
41 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) 46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
42 if (!video) return undefined 47 if (!video) return undefined
43 48
49 // FIXME: use URL
44 const remoteStaticPath = videoCaption.getCaptionStaticPath() 50 const remoteStaticPath = videoCaption.getCaptionStaticPath()
45 const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) 51 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
52
53 await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath)
46 54
47 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) 55 return { isOwned: false, path: destPath }
48 } 56 }
49} 57}
50 58
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
index a5d6f5b62..14be7f24a 100644
--- a/server/lib/cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -1,7 +1,9 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers' 2import { FILES_CACHE, STATIC_PATHS } from '../../initializers/constants'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
5import { CONFIG } from '../../initializers/config'
6import { fetchRemoteVideoStaticFile } from '../activitypub'
5 7
6class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { 8class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
7 9
@@ -15,13 +17,13 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
15 return this.instance || (this.instance = new this()) 17 return this.instance || (this.instance = new this())
16 } 18 }
17 19
18 async getFilePath (videoUUID: string) { 20 async getFilePathImpl (videoUUID: string) {
19 const video = await VideoModel.loadByUUIDWithFile(videoUUID) 21 const video = await VideoModel.loadByUUIDWithFile(videoUUID)
20 if (!video) return undefined 22 if (!video) return undefined
21 23
22 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) 24 if (video.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreview().filename) }
23 25
24 return this.loadFromLRU(videoUUID) 26 return this.loadRemoteFile(videoUUID)
25 } 27 }
26 28
27 protected async loadRemoteFile (key: string) { 29 protected async loadRemoteFile (key: string) {
@@ -30,10 +32,13 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
30 32
31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 33 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
32 34
33 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) 35 // FIXME: use URL
34 const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName()) 36 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreview().filename)
37 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreview().filename)
35 38
36 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) 39 await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath)
40
41 return { isOwned: false, path: destPath }
37 } 42 }
38} 43}
39 44
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
new file mode 100644
index 000000000..98da4dcd8
--- /dev/null
+++ b/server/lib/hls.ts
@@ -0,0 +1,184 @@
1import { VideoModel } from '../models/video/video'
2import { basename, dirname, join } from 'path'
3import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import { logger } from '../helpers/logger'
9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
10import { generateRandomString } from '../helpers/utils'
11import { flatten, uniq } from 'lodash'
12import { VideoFileModel } from '../models/video/video-file'
13import { CONFIG } from '../initializers/config'
14import { sequelizeTypescript } from '../initializers/database'
15
16async function updateStreamingPlaylistsInfohashesIfNeeded () {
17 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
18
19 // Use separate SQL queries, because we could have many videos to update
20 for (const playlist of playlistsToUpdate) {
21 await sequelizeTypescript.transaction(async t => {
22 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
23
24 playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
25 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
26 await playlist.save({ transaction: t })
27 })
28 }
29}
30
31async function updateMasterHLSPlaylist (video: VideoModel) {
32 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
33 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
34 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
35
36 for (const file of video.VideoFiles) {
37 // If we did not generated a playlist for this resolution, skip
38 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
39 if (await pathExists(filePlaylistPath) === false) continue
40
41 const videoFilePath = video.getVideoFilePath(file)
42
43 const size = await getVideoFileSize(videoFilePath)
44
45 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
46 const resolution = `RESOLUTION=${size.width}x${size.height}`
47
48 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
49 if (file.fps) line += ',FRAME-RATE=' + file.fps
50
51 masterPlaylists.push(line)
52 masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
53 }
54
55 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
56}
57
58async function updateSha256Segments (video: VideoModel) {
59 const json: { [filename: string]: { [range: string]: string } } = {}
60
61 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
62
63 // For all the resolutions available for this video
64 for (const file of video.VideoFiles) {
65 const rangeHashes: { [range: string]: string } = {}
66
67 const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
68 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
69
70 // Maybe the playlist is not generated for this resolution yet
71 if (!await pathExists(playlistPath)) continue
72
73 const playlistContent = await readFile(playlistPath)
74 const ranges = getRangesFromPlaylist(playlistContent.toString())
75
76 const fd = await open(videoPath, 'r')
77 for (const range of ranges) {
78 const buf = Buffer.alloc(range.length)
79 await read(fd, buf, 0, range.length, range.offset)
80
81 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
82 }
83 await close(fd)
84
85 const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
86 json[videoFilename] = rangeHashes
87 }
88
89 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
90 await outputJSON(outputPath, json)
91}
92
93function getRangesFromPlaylist (playlistContent: string) {
94 const ranges: { offset: number, length: number }[] = []
95 const lines = playlistContent.split('\n')
96 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
97
98 for (const line of lines) {
99 const captured = regex.exec(line)
100
101 if (captured) {
102 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
103 }
104 }
105
106 return ranges
107}
108
109function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
110 let timer
111
112 logger.info('Importing HLS playlist %s', playlistUrl)
113
114 return new Promise<string>(async (res, rej) => {
115 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
116
117 await ensureDir(tmpDirectory)
118
119 timer = setTimeout(() => {
120 deleteTmpDirectory(tmpDirectory)
121
122 return rej(new Error('HLS download timeout.'))
123 }, timeout)
124
125 try {
126 // Fetch master playlist
127 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
128
129 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
130 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
131
132 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
133
134 for (const fileUrl of fileUrls) {
135 const destPath = join(tmpDirectory, basename(fileUrl))
136
137 const bodyKBLimit = 10 * 1000 * 1000 // 10GB
138 await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
139 }
140
141 clearTimeout(timer)
142
143 await move(tmpDirectory, destinationDir, { overwrite: true })
144
145 return res()
146 } catch (err) {
147 deleteTmpDirectory(tmpDirectory)
148
149 return rej(err)
150 }
151 })
152
153 function deleteTmpDirectory (directory: string) {
154 remove(directory)
155 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
156 }
157
158 async function fetchUniqUrls (playlistUrl: string) {
159 const { body } = await doRequest<string>({ uri: playlistUrl })
160
161 if (!body) return []
162
163 const urls = body.split('\n')
164 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
165 .map(url => {
166 if (url.startsWith('http://') || url.startsWith('https://')) return url
167
168 return `${dirname(playlistUrl)}/${url}`
169 })
170
171 return uniq(urls)
172 }
173}
174
175// ---------------------------------------------------------------------------
176
177export {
178 updateMasterHLSPlaylist,
179 updateSha256Segments,
180 downloadPlaylistSegments,
181 updateStreamingPlaylistsInfohashesIfNeeded
182}
183
184// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts
index b4d381062..b3defb617 100644
--- a/server/lib/job-queue/handlers/activitypub-follow.ts
+++ b/server/lib/job-queue/handlers/activitypub-follow.ts
@@ -1,6 +1,6 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { CONFIG, REMOTE_SCHEME, sequelizeTypescript } from '../../../initializers' 3import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants'
4import { sendFollow } from '../../activitypub/send' 4import { sendFollow } from '../../activitypub/send'
5import { sanitizeHost } from '../../../helpers/core-utils' 5import { sanitizeHost } from '../../../helpers/core-utils'
6import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger' 6import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger'
@@ -9,6 +9,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
9import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 9import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
10import { ActorModel } from '../../../models/activitypub/actor' 10import { ActorModel } from '../../../models/activitypub/actor'
11import { Notifier } from '../../notifier' 11import { Notifier } from '../../notifier'
12import { sequelizeTypescript } from '../../../initializers/database'
12 13
13export type ActivitypubFollowPayload = { 14export type ActivitypubFollowPayload = {
14 followerActorId: number 15 followerActorId: number
@@ -23,7 +24,7 @@ async function processActivityPubFollow (job: Bull.Job) {
23 logger.info('Processing ActivityPub follow in job %d.', job.id) 24 logger.info('Processing ActivityPub follow in job %d.', job.id)
24 25
25 let targetActor: ActorModel 26 let targetActor: ActorModel
26 if (!host || host === CONFIG.WEBSERVER.HOST) { 27 if (!host || host === WEBSERVER.HOST) {
27 targetActor = await ActorModel.loadLocalByName(payload.name) 28 targetActor = await ActorModel.loadLocalByName(payload.name)
28 } else { 29 } else {
29 const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) 30 const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP)
@@ -73,5 +74,5 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) {
73 return actorFollow 74 return actorFollow
74 }) 75 })
75 76
76 if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewFollow(actorFollow) 77 if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollow)
77} 78}
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 9493945ff..0ff7b44a0 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -2,10 +2,9 @@ import * as Bull from 'bull'
2import * as Bluebird from 'bluebird' 2import * as Bluebird from 'bluebird'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { doRequest } from '../../../helpers/requests' 4import { doRequest } from '../../../helpers/requests'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 5import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
7import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' 6import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
8import { ActorFollowScoreCache } from '../../cache' 7import { ActorFollowScoreCache } from '../../files-cache'
9 8
10export type ActivitypubHttpBroadcastPayload = { 9export type ActivitypubHttpBroadcastPayload = {
11 uris: string[] 10 uris: string[]
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 67ccfa995..23d33c26f 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,17 +1,24 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import * as Bluebird from 'bluebird'
2import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
3import { processActivities } from '../../activitypub/process' 4import { processActivities } from '../../activitypub/process'
4import { addVideoComments } from '../../activitypub/video-comments' 5import { addVideoComments } from '../../activitypub/video-comments'
5import { crawlCollectionPage } from '../../activitypub/crawl' 6import { crawlCollectionPage } from '../../activitypub/crawl'
6import { VideoModel } from '../../../models/video/video' 7import { VideoModel } from '../../../models/video/video'
7import { addVideoShares, createRates } from '../../activitypub' 8import { addVideoShares, createRates } from '../../activitypub'
9import { createAccountPlaylists } from '../../activitypub/playlist'
10import { AccountModel } from '../../../models/account/account'
11import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
12import { VideoShareModel } from '../../../models/video/video-share'
13import { VideoCommentModel } from '../../../models/video/video-comment'
8 14
9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' 15type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists'
10 16
11export type ActivitypubHttpFetcherPayload = { 17export type ActivitypubHttpFetcherPayload = {
12 uri: string 18 uri: string
13 type: FetchType 19 type: FetchType
14 videoId?: number 20 videoId?: number
21 accountId?: number
15} 22}
16 23
17async function processActivityPubHttpFetcher (job: Bull.Job) { 24async function processActivityPubHttpFetcher (job: Bull.Job) {
@@ -22,15 +29,26 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
22 let video: VideoModel 29 let video: VideoModel
23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) 30 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
24 31
32 let account: AccountModel
33 if (payload.accountId) account = await AccountModel.load(payload.accountId)
34
25 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { 35 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
26 'activity': items => processActivities(items, { outboxUrl: payload.uri }), 36 'activity': items => processActivities(items, { outboxUrl: payload.uri }),
27 'video-likes': items => createRates(items, video, 'like'), 37 'video-likes': items => createRates(items, video, 'like'),
28 'video-dislikes': items => createRates(items, video, 'dislike'), 38 'video-dislikes': items => createRates(items, video, 'dislike'),
29 'video-shares': items => addVideoShares(items, video), 39 'video-shares': items => addVideoShares(items, video),
30 'video-comments': items => addVideoComments(items, video) 40 'video-comments': items => addVideoComments(items, video),
41 'account-playlists': items => createAccountPlaylists(items, account)
42 }
43
44 const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Bluebird<any> } = {
45 'video-likes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate),
46 'video-dislikes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate),
47 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
48 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
31 } 49 }
32 50
33 return crawlCollectionPage(payload.uri, fetcherType[payload.type]) 51 return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
34} 52}
35 53
36// --------------------------------------------------------------------------- 54// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index 3973dcdc8..c70ce3be9 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -2,8 +2,8 @@ import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { doRequest } from '../../../helpers/requests' 3import { doRequest } from '../../../helpers/requests'
4import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 4import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
5import { JOB_REQUEST_TIMEOUT } from '../../../initializers' 5import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
6import { ActorFollowScoreCache } from '../../cache' 6import { ActorFollowScoreCache } from '../../files-cache'
7 7
8export type ActivitypubHttpUnicastPayload = { 8export type ActivitypubHttpUnicastPayload = {
9 uri: string 9 uri: string
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
index 454b975fe..4d6c38cfa 100644
--- a/server/lib/job-queue/handlers/activitypub-refresher.ts
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -1,11 +1,12 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { fetchVideoByUrl } from '../../../helpers/video' 3import { fetchVideoByUrl } from '../../../helpers/video'
4import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' 4import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { VideoPlaylistModel } from '../../../models/video/video-playlist'
6 7
7export type RefreshPayload = { 8export type RefreshPayload = {
8 type: 'video' | 'actor' 9 type: 'video' | 'video-playlist' | 'actor'
9 url: string 10 url: string
10} 11}
11 12
@@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) {
15 logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) 16 logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url)
16 17
17 if (payload.type === 'video') return refreshVideo(payload.url) 18 if (payload.type === 'video') return refreshVideo(payload.url)
19 if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
18 if (payload.type === 'actor') return refreshActor(payload.url) 20 if (payload.type === 'actor') return refreshActor(payload.url)
19} 21}
20 22
21// --------------------------------------------------------------------------- 23// ---------------------------------------------------------------------------
22 24
23export { 25export {
24 refreshActor,
25 refreshAPObject 26 refreshAPObject
26} 27}
27 28
@@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) {
50 if (actor) { 51 if (actor) {
51 await refreshActorIfNeeded(actor, fetchType) 52 await refreshActorIfNeeded(actor, fetchType)
52 } 53 }
54}
55
56async function refreshVideoPlaylist (playlistUrl: string) {
57 const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
53 58
59 if (playlist) {
60 await refreshVideoPlaylistIfNeeded(playlist)
61 }
54} 62}
diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts
index 2ba39a156..62701222c 100644
--- a/server/lib/job-queue/handlers/email.ts
+++ b/server/lib/job-queue/handlers/email.ts
@@ -1,15 +1,8 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { Emailer } from '../../emailer' 3import { Emailer, SendEmailOptions } from '../../emailer'
4 4
5export type EmailPayload = { 5export type EmailPayload = SendEmailOptions
6 to: string[]
7 subject: string
8 text: string
9
10 fromDisplayName?: string
11 replyTo?: string
12}
13 6
14async function processEmail (job: Bull.Job) { 7async function processEmail (job: Bull.Job) {
15 const payload = job.data as EmailPayload 8 const payload = job.data as EmailPayload
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
index 4961d4502..cdee1f6fd 100644
--- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
+++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
@@ -2,7 +2,7 @@ import { buildSignedActivity } from '../../../../helpers/activitypub'
2import { getServerActor } from '../../../../helpers/utils' 2import { getServerActor } from '../../../../helpers/utils'
3import { ActorModel } from '../../../../models/activitypub/actor' 3import { ActorModel } from '../../../../models/activitypub/actor'
4import { sha256 } from '../../../../helpers/core-utils' 4import { sha256 } from '../../../../helpers/core-utils'
5import { HTTP_SIGNATURE } from '../../../../initializers' 5import { HTTP_SIGNATURE } from '../../../../initializers/constants'
6 6
7type Payload = { body: any, signatureActorId?: number } 7type Payload = { body: any, signatureActorId?: number }
8 8
@@ -28,7 +28,7 @@ async function buildSignedRequestOptions (payload: Payload) {
28 actor = await getServerActor() 28 actor = await getServerActor()
29 } 29 }
30 30
31 const keyId = actor.getWebfingerUrl() 31 const keyId = actor.url
32 return { 32 return {
33 algorithm: HTTP_SIGNATURE.ALGORITHM, 33 algorithm: HTTP_SIGNATURE.ALGORITHM,
34 authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, 34 authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME,
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
new file mode 100644
index 000000000..921d9a083
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -0,0 +1,78 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { VideoModel } from '../../../models/video/video'
4import { publishVideoIfNeeded } from './video-transcoding'
5import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
6import { copy, stat } from 'fs-extra'
7import { VideoFileModel } from '../../../models/video/video-file'
8import { extname } from 'path'
9
10export type VideoFileImportPayload = {
11 videoUUID: string,
12 filePath: string
13}
14
15async function processVideoFileImport (job: Bull.Job) {
16 const payload = job.data as VideoFileImportPayload
17 logger.info('Processing video file import in job %d.', job.id)
18
19 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
20 // No video, maybe deleted?
21 if (!video) {
22 logger.info('Do not process job %d, video does not exist.', job.id)
23 return undefined
24 }
25
26 await updateVideoFile(video, payload.filePath)
27
28 await publishVideoIfNeeded(video)
29 return video
30}
31
32// ---------------------------------------------------------------------------
33
34export {
35 processVideoFileImport
36}
37
38// ---------------------------------------------------------------------------
39
40async function updateVideoFile (video: VideoModel, inputFilePath: string) {
41 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
42 const { size } = await stat(inputFilePath)
43 const fps = await getVideoFileFPS(inputFilePath)
44
45 let updatedVideoFile = new VideoFileModel({
46 resolution: videoFileResolution,
47 extname: extname(inputFilePath),
48 size,
49 fps,
50 videoId: video.id
51 })
52
53 const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
54
55 if (currentVideoFile) {
56 // Remove old file and old torrent
57 await video.removeFile(currentVideoFile)
58 await video.removeTorrent(currentVideoFile)
59 // Remove the old video file from the array
60 video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
61
62 // Update the database
63 currentVideoFile.set('extname', updatedVideoFile.extname)
64 currentVideoFile.set('size', updatedVideoFile.size)
65 currentVideoFile.set('fps', updatedVideoFile.fps)
66
67 updatedVideoFile = currentVideoFile
68 }
69
70 const outputPath = video.getVideoFilePath(updatedVideoFile)
71 await copy(inputFilePath, outputPath)
72
73 await video.createTorrentAndSetInfoHash(updatedVideoFile)
74
75 await updatedVideoFile.save()
76
77 video.VideoFiles.push(updatedVideoFile)
78}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 12004dcd7..1650916a6 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -6,16 +6,20 @@ import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path' 7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file' 8import { VideoFileModel } from '../../../models/video/video-file'
9import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' 9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
10import { downloadImage } from '../../../helpers/requests'
11import { VideoState } from '../../../../shared' 10import { VideoState } from '../../../../shared'
12import { JobQueue } from '../index' 11import { JobQueue } from '../index'
13import { federateVideoIfNeeded } from '../../activitypub' 12import { federateVideoIfNeeded } from '../../activitypub'
14import { VideoModel } from '../../../models/video/video' 13import { VideoModel } from '../../../models/video/video'
15import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' 14import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
16import { getSecureTorrentName } from '../../../helpers/utils' 15import { getSecureTorrentName } from '../../../helpers/utils'
17import { remove, move, stat } from 'fs-extra' 16import { move, remove, stat } from 'fs-extra'
18import { Notifier } from '../../notifier' 17import { Notifier } from '../../notifier'
18import { CONFIG } from '../../../initializers/config'
19import { sequelizeTypescript } from '../../../initializers/database'
20import { ThumbnailModel } from '../../../models/video/thumbnail'
21import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumbnail'
22import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
19 23
20type VideoImportYoutubeDLPayload = { 24type VideoImportYoutubeDLPayload = {
21 type: 'youtube-dl' 25 type: 'youtube-dl'
@@ -144,25 +148,19 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
144 tempVideoPath = null // This path is not used anymore 148 tempVideoPath = null // This path is not used anymore
145 149
146 // Process thumbnail 150 // Process thumbnail
147 if (options.downloadThumbnail) { 151 let thumbnailModel: ThumbnailModel
148 if (options.thumbnailUrl) { 152 if (options.downloadThumbnail && options.thumbnailUrl) {
149 await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE) 153 thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.MINIATURE)
150 } else { 154 } else if (options.generateThumbnail || options.downloadThumbnail) {
151 await videoImport.Video.createThumbnail(videoFile) 155 thumbnailModel = await generateVideoMiniature(videoImport.Video, videoFile, ThumbnailType.MINIATURE)
152 }
153 } else if (options.generateThumbnail) {
154 await videoImport.Video.createThumbnail(videoFile)
155 } 156 }
156 157
157 // Process preview 158 // Process preview
158 if (options.downloadPreview) { 159 let previewModel: ThumbnailModel
159 if (options.thumbnailUrl) { 160 if (options.downloadPreview && options.thumbnailUrl) {
160 await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE) 161 previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.PREVIEW)
161 } else { 162 } else if (options.generatePreview || options.downloadPreview) {
162 await videoImport.Video.createPreview(videoFile) 163 previewModel = await generateVideoMiniature(videoImport.Video, videoFile, ThumbnailType.PREVIEW)
163 }
164 } else if (options.generatePreview) {
165 await videoImport.Video.createPreview(videoFile)
166 } 164 }
167 165
168 // Create torrent 166 // Create torrent
@@ -182,6 +180,9 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
182 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 180 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
183 await video.save({ transaction: t }) 181 await video.save({ transaction: t })
184 182
183 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
184 if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
185
185 // Now we can federate the video (reload from database, we need more attributes) 186 // Now we can federate the video (reload from database, we need more attributes)
186 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 187 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
187 await federateVideoIfNeeded(videoForFederation, true, t) 188 await federateVideoIfNeeded(videoForFederation, true, t)
@@ -196,9 +197,14 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
196 return videoImportUpdated 197 return videoImportUpdated
197 }) 198 })
198 199
199 Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video)
200 Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true) 200 Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true)
201 201
202 if (videoImportUpdated.Video.VideoBlacklist) {
203 Notifier.Instance.notifyOnVideoAutoBlacklist(videoImportUpdated.Video)
204 } else {
205 Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video)
206 }
207
202 // Create transcoding jobs? 208 // Create transcoding jobs?
203 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { 209 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
204 // Put uuid because we don't have id auto incremented for now 210 // Put uuid because we don't have id auto incremented for now
@@ -207,7 +213,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
207 isNewVideo: true 213 isNewVideo: true
208 } 214 }
209 215
210 await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) 216 await JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
211 } 217 }
212 218
213 } catch (err) { 219 } catch (err) {
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 593e43cc5..48cac517e 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -8,40 +8,20 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' 11import { generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
12import { Notifier } from '../../notifier' 12import { Notifier } from '../../notifier'
13import { CONFIG } from '../../../initializers/config'
13 14
14export type VideoFilePayload = { 15export type VideoTranscodingPayload = {
15 videoUUID: string 16 videoUUID: string
16 isNewVideo?: boolean
17 resolution?: VideoResolution 17 resolution?: VideoResolution
18 isNewVideo?: boolean
18 isPortraitMode?: boolean 19 isPortraitMode?: boolean
20 generateHlsPlaylist?: boolean
19} 21}
20 22
21export type VideoFileImportPayload = { 23async function processVideoTranscoding (job: Bull.Job) {
22 videoUUID: string, 24 const payload = job.data as VideoTranscodingPayload
23 filePath: string
24}
25
26async function processVideoFileImport (job: Bull.Job) {
27 const payload = job.data as VideoFileImportPayload
28 logger.info('Processing video file import in job %d.', job.id)
29
30 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
31 // No video, maybe deleted?
32 if (!video) {
33 logger.info('Do not process job %d, video does not exist.', job.id)
34 return undefined
35 }
36
37 await importVideoFile(video, payload.filePath)
38
39 await onVideoFileTranscoderOrImportSuccess(video)
40 return video
41}
42
43async function processVideoFile (job: Bull.Job) {
44 const payload = job.data as VideoFilePayload
45 logger.info('Processing video file in job %d.', job.id) 25 logger.info('Processing video file in job %d.', job.id)
46 26
47 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) 27 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
@@ -51,23 +31,38 @@ async function processVideoFile (job: Bull.Job) {
51 return undefined 31 return undefined
52 } 32 }
53 33
54 // Transcoding in other resolution 34 if (payload.generateHlsPlaylist) {
55 if (payload.resolution) { 35 await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
36
37 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
38 } else if (payload.resolution) { // Transcoding in other resolution
56 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) 39 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
57 40
58 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 41 await retryTransactionWrapper(publishVideoIfNeeded, video, payload)
59 } else { 42 } else {
60 await optimizeVideofile(video) 43 await optimizeVideofile(video)
61 44
62 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 45 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
63 } 46 }
64 47
65 return video 48 return video
66} 49}
67 50
68async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { 51async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
69 if (video === undefined) return undefined 52 if (video === undefined) return undefined
70 53
54 await sequelizeTypescript.transaction(async t => {
55 // Maybe the video changed in database, refresh it
56 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
57 // Video does not exist anymore
58 if (!videoDatabase) return undefined
59
60 // If the video was not published, we consider it is a new one for other instances
61 await federateVideoIfNeeded(videoDatabase, false, t)
62 })
63}
64
65async function publishVideoIfNeeded (video: VideoModel, payload?: VideoTranscodingPayload) {
71 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { 66 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
72 // Maybe the video changed in database, refresh it 67 // Maybe the video changed in database, refresh it
73 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 68 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
@@ -93,11 +88,13 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
93 88
94 if (videoPublished) { 89 if (videoPublished) {
95 Notifier.Instance.notifyOnNewVideo(videoDatabase) 90 Notifier.Instance.notifyOnNewVideo(videoDatabase)
96 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 91 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
97 } 92 }
93
94 await createHlsJobIfEnabled(payload)
98} 95}
99 96
100async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { 97async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoTranscodingPayload) {
101 if (videoArg === undefined) return undefined 98 if (videoArg === undefined) return undefined
102 99
103 // Outside the transaction (IO on disk) 100 // Outside the transaction (IO on disk)
@@ -119,7 +116,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
119 let videoPublished = false 116 let videoPublished = false
120 117
121 if (resolutionsEnabled.length !== 0) { 118 if (resolutionsEnabled.length !== 0) {
122 const tasks: Bluebird<Bull.Job<any>>[] = [] 119 const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = []
123 120
124 for (const resolution of resolutionsEnabled) { 121 for (const resolution of resolutionsEnabled) {
125 const dataInput = { 122 const dataInput = {
@@ -127,7 +124,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
127 resolution 124 resolution
128 } 125 }
129 126
130 const p = JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) 127 const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
131 tasks.push(p) 128 tasks.push(p)
132 } 129 }
133 130
@@ -144,18 +141,37 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
144 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) 141 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
145 } 142 }
146 143
147 await federateVideoIfNeeded(videoDatabase, isNewVideo, t) 144 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)
148 145
149 return { videoDatabase, videoPublished } 146 return { videoDatabase, videoPublished }
150 }) 147 })
151 148
152 if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) 149 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
153 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 150 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
151
152 await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
154} 153}
155 154
156// --------------------------------------------------------------------------- 155// ---------------------------------------------------------------------------
157 156
158export { 157export {
159 processVideoFile, 158 processVideoTranscoding,
160 processVideoFileImport 159 publishVideoIfNeeded
160}
161
162// ---------------------------------------------------------------------------
163
164function createHlsJobIfEnabled (payload?: VideoTranscodingPayload) {
165 // Generate HLS playlist?
166 if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
167 const hlsTranscodingPayload = {
168 videoUUID: payload.videoUUID,
169 resolution: payload.resolution,
170 isPortraitMode: payload.isPortraitMode,
171
172 generateHlsPlaylist: true
173 }
174
175 return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload })
176 }
161} 177}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ba9cbe0d9..3c810da98 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -2,16 +2,17 @@ import * as Bull from 'bull'
2import { JobState, JobType } from '../../../shared/models' 2import { JobState, JobType } from '../../../shared/models'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { Redis } from '../redis' 4import { Redis } from '../redis'
5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS } from '../../initializers' 5import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants'
6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' 6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoTranscoding, VideoTranscodingPayload } from './handlers/video-transcoding'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import' 12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViews } from './handlers/video-views' 13import { processVideosViews } from './handlers/video-views'
14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' 14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
15 16
16type CreateJobArgument = 17type CreateJobArgument =
17 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 18 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -19,19 +20,20 @@ type CreateJobArgument =
19 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | 20 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
20 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 21 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
21 { type: 'video-file-import', payload: VideoFileImportPayload } | 22 { type: 'video-file-import', payload: VideoFileImportPayload } |
22 { type: 'video-file', payload: VideoFilePayload } | 23 { type: 'video-transcoding', payload: VideoTranscodingPayload } |
23 { type: 'email', payload: EmailPayload } | 24 { type: 'email', payload: EmailPayload } |
24 { type: 'video-import', payload: VideoImportPayload } | 25 { type: 'video-import', payload: VideoImportPayload } |
25 { type: 'activitypub-refresher', payload: RefreshPayload } | 26 { type: 'activitypub-refresher', payload: RefreshPayload } |
26 { type: 'videos-views', payload: {} } 27 { type: 'videos-views', payload: {} }
27 28
28const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { 29const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = {
29 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 30 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
30 'activitypub-http-unicast': processActivityPubHttpUnicast, 31 'activitypub-http-unicast': processActivityPubHttpUnicast,
31 'activitypub-http-fetcher': processActivityPubHttpFetcher, 32 'activitypub-http-fetcher': processActivityPubHttpFetcher,
32 'activitypub-follow': processActivityPubFollow, 33 'activitypub-follow': processActivityPubFollow,
33 'video-file-import': processVideoFileImport, 34 'video-file-import': processVideoFileImport,
34 'video-file': processVideoFile, 35 'video-transcoding': processVideoTranscoding,
36 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
35 'email': processEmail, 37 'email': processEmail,
36 'video-import': processVideoImport, 38 'video-import': processVideoImport,
37 'videos-views': processVideosViews, 39 'videos-views': processVideosViews,
@@ -44,7 +46,7 @@ const jobTypes: JobType[] = [
44 'activitypub-http-fetcher', 46 'activitypub-http-fetcher',
45 'activitypub-http-unicast', 47 'activitypub-http-unicast',
46 'email', 48 'email',
47 'video-file', 49 'video-transcoding',
48 'video-file-import', 50 'video-file-import',
49 'video-import', 51 'video-import',
50 'videos-views', 52 'videos-views',
@@ -66,10 +68,10 @@ class JobQueue {
66 if (this.initialized === true) return 68 if (this.initialized === true) return
67 this.initialized = true 69 this.initialized = true
68 70
69 this.jobRedisPrefix = 'bull-' + CONFIG.WEBSERVER.HOST 71 this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST
70 const queueOptions = { 72 const queueOptions = {
71 prefix: this.jobRedisPrefix, 73 prefix: this.jobRedisPrefix,
72 redis: Redis.getRedisClient(), 74 redis: Redis.getRedisClientOptions(),
73 settings: { 75 settings: {
74 maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts 76 maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts
75 } 77 }
diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts
index 2fa320cd7..c1e63fa8f 100644
--- a/server/lib/notifier.ts
+++ b/server/lib/notifier.ts
@@ -6,7 +6,7 @@ import { UserNotificationModel } from '../models/account/user-notification'
6import { VideoCommentModel } from '../models/video/video-comment' 6import { VideoCommentModel } from '../models/video/video-comment'
7import { UserModel } from '../models/account/user' 7import { UserModel } from '../models/account/user'
8import { PeerTubeSocket } from './peertube-socket' 8import { PeerTubeSocket } from './peertube-socket'
9import { CONFIG } from '../initializers/constants' 9import { CONFIG } from '../initializers/config'
10import { VideoPrivacy, VideoState } from '../../shared/models/videos' 10import { VideoPrivacy, VideoState } from '../../shared/models/videos'
11import { VideoAbuseModel } from '../models/video/video-abuse' 11import { VideoAbuseModel } from '../models/video/video-abuse'
12import { VideoBlacklistModel } from '../models/video/video-blacklist' 12import { VideoBlacklistModel } from '../models/video/video-blacklist'
@@ -23,19 +23,35 @@ class Notifier {
23 private constructor () {} 23 private constructor () {}
24 24
25 notifyOnNewVideo (video: VideoModel): void { 25 notifyOnNewVideo (video: VideoModel): void {
26 // Only notify on public and published videos 26 // Only notify on public and published videos which are not blacklisted
27 if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED) return 27 if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED || video.VideoBlacklist) return
28 28
29 this.notifySubscribersOfNewVideo(video) 29 this.notifySubscribersOfNewVideo(video)
30 .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) 30 .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err }))
31 } 31 }
32 32
33 notifyOnPendingVideoPublished (video: VideoModel): void { 33 notifyOnVideoPublishedAfterTranscoding (video: VideoModel): void {
34 // Only notify on public videos that has been published while the user waited transcoding/scheduled update 34 // don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update
35 if (video.waitTranscoding === false && !video.ScheduleVideoUpdate) return 35 if (!video.waitTranscoding || video.VideoBlacklist || video.ScheduleVideoUpdate) return
36 36
37 this.notifyOwnedVideoHasBeenPublished(video) 37 this.notifyOwnedVideoHasBeenPublished(video)
38 .catch(err => logger.error('Cannot notify owner that its video %s has been published.', video.url, { err })) 38 .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err }))
39 }
40
41 notifyOnVideoPublishedAfterScheduledUpdate (video: VideoModel): void {
42 // don't notify if video is still blacklisted or waiting for transcoding
43 if (video.VideoBlacklist || (video.waitTranscoding && video.state !== VideoState.PUBLISHED)) return
44
45 this.notifyOwnedVideoHasBeenPublished(video)
46 .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err }))
47 }
48
49 notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: VideoModel): void {
50 // don't notify if video is still waiting for transcoding or scheduled update
51 if (video.ScheduleVideoUpdate || (video.waitTranscoding && video.state !== VideoState.PUBLISHED)) return
52
53 this.notifyOwnedVideoHasBeenPublished(video)
54 .catch(err => logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err })) // tslint:disable-line:max-line-length
39 } 55 }
40 56
41 notifyOnNewComment (comment: VideoCommentModel): void { 57 notifyOnNewComment (comment: VideoCommentModel): void {
@@ -51,6 +67,11 @@ class Notifier {
51 .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err })) 67 .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err }))
52 } 68 }
53 69
70 notifyOnVideoAutoBlacklist (video: VideoModel): void {
71 this.notifyModeratorsOfVideoAutoBlacklist(video)
72 .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', video.url, { err }))
73 }
74
54 notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void { 75 notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void {
55 this.notifyVideoOwnerOfBlacklist(videoBlacklist) 76 this.notifyVideoOwnerOfBlacklist(videoBlacklist)
56 .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) 77 .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err }))
@@ -58,7 +79,7 @@ class Notifier {
58 79
59 notifyOnVideoUnblacklist (video: VideoModel): void { 80 notifyOnVideoUnblacklist (video: VideoModel): void {
60 this.notifyVideoOwnerOfUnblacklist(video) 81 this.notifyVideoOwnerOfUnblacklist(video)
61 .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err })) 82 .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err }))
62 } 83 }
63 84
64 notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void { 85 notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void {
@@ -71,18 +92,25 @@ class Notifier {
71 .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) 92 .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err }))
72 } 93 }
73 94
74 notifyOfNewFollow (actorFollow: ActorFollowModel): void { 95 notifyOfNewUserFollow (actorFollow: ActorFollowModel): void {
75 this.notifyUserOfNewActorFollow(actorFollow) 96 this.notifyUserOfNewActorFollow(actorFollow)
76 .catch(err => { 97 .catch(err => {
77 logger.error( 98 logger.error(
78 'Cannot notify owner of channel %s of a new follow by %s.', 99 'Cannot notify owner of channel %s of a new follow by %s.',
79 actorFollow.ActorFollowing.VideoChannel.getDisplayName(), 100 actorFollow.ActorFollowing.VideoChannel.getDisplayName(),
80 actorFollow.ActorFollower.Account.getDisplayName(), 101 actorFollow.ActorFollower.Account.getDisplayName(),
81 err 102 { err }
82 ) 103 )
83 }) 104 })
84 } 105 }
85 106
107 notifyOfNewInstanceFollow (actorFollow: ActorFollowModel): void {
108 this.notifyAdminsOfNewInstanceFollow(actorFollow)
109 .catch(err => {
110 logger.error('Cannot notify administrators of new follower %s.', actorFollow.ActorFollower.url, { err })
111 })
112 }
113
86 private async notifySubscribersOfNewVideo (video: VideoModel) { 114 private async notifySubscribersOfNewVideo (video: VideoModel) {
87 // List all followers that are users 115 // List all followers that are users
88 const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) 116 const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId)
@@ -147,10 +175,13 @@ class Notifier {
147 } 175 }
148 176
149 private async notifyOfCommentMention (comment: VideoCommentModel) { 177 private async notifyOfCommentMention (comment: VideoCommentModel) {
150 const usernames = comment.extractMentions() 178 const extractedUsernames = comment.extractMentions()
151 logger.debug('Extracted %d username from comment %s.', usernames.length, comment.url, { usernames, text: comment.text }) 179 logger.debug(
180 'Extracted %d username from comment %s.', extractedUsernames.length, comment.url,
181 { usernames: extractedUsernames, text: comment.text }
182 )
152 183
153 let users = await UserModel.listByUsernames(usernames) 184 let users = await UserModel.listByUsernames(extractedUsernames)
154 185
155 if (comment.Video.isOwned()) { 186 if (comment.Video.isOwned()) {
156 const userException = await UserModel.loadByVideoId(comment.videoId) 187 const userException = await UserModel.loadByVideoId(comment.videoId)
@@ -237,6 +268,33 @@ class Notifier {
237 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) 268 return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender })
238 } 269 }
239 270
271 private async notifyAdminsOfNewInstanceFollow (actorFollow: ActorFollowModel) {
272 const admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW)
273
274 logger.info('Notifying %d administrators of new instance follower: %s.', admins.length, actorFollow.ActorFollower.url)
275
276 function settingGetter (user: UserModel) {
277 return user.NotificationSetting.newInstanceFollower
278 }
279
280 async function notificationCreator (user: UserModel) {
281 const notification = await UserNotificationModel.create({
282 type: UserNotificationType.NEW_INSTANCE_FOLLOWER,
283 userId: user.id,
284 actorFollowId: actorFollow.id
285 })
286 notification.ActorFollow = actorFollow
287
288 return notification
289 }
290
291 function emailSender (emails: string[]) {
292 return Emailer.Instance.addNewInstanceFollowerNotification(emails, actorFollow)
293 }
294
295 return this.notify({ users: admins, settingGetter, notificationCreator, emailSender })
296 }
297
240 private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { 298 private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) {
241 const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) 299 const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES)
242 if (moderators.length === 0) return 300 if (moderators.length === 0) return
@@ -265,6 +323,34 @@ class Notifier {
265 return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) 323 return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender })
266 } 324 }
267 325
326 private async notifyModeratorsOfVideoAutoBlacklist (video: VideoModel) {
327 const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST)
328 if (moderators.length === 0) return
329
330 logger.info('Notifying %s moderators of video auto-blacklist %s.', moderators.length, video.url)
331
332 function settingGetter (user: UserModel) {
333 return user.NotificationSetting.videoAutoBlacklistAsModerator
334 }
335 async function notificationCreator (user: UserModel) {
336
337 const notification = await UserNotificationModel.create({
338 type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS,
339 userId: user.id,
340 videoId: video.id
341 })
342 notification.Video = video
343
344 return notification
345 }
346
347 function emailSender (emails: string[]) {
348 return Emailer.Instance.addVideoAutoBlacklistModeratorsNotification(emails, video)
349 }
350
351 return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender })
352 }
353
268 private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { 354 private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) {
269 const user = await UserModel.loadByVideoId(videoBlacklist.videoId) 355 const user = await UserModel.loadByVideoId(videoBlacklist.videoId)
270 if (!user) return 356 if (!user) return
@@ -436,7 +522,7 @@ class Notifier {
436 } 522 }
437 523
438 private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) { 524 private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) {
439 if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified !== true) return false 525 if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified === false) return false
440 526
441 return value & UserNotificationSettingValue.EMAIL 527 return value & UserNotificationSettingValue.EMAIL
442 } 528 }
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 2cd2ae97c..45ac3e7c4 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,12 +4,13 @@ import { logger } from '../helpers/logger'
4import { UserModel } from '../models/account/user' 4import { UserModel } from '../models/account/user'
5import { OAuthClientModel } from '../models/oauth/oauth-client' 5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { OAuthTokenModel } from '../models/oauth/oauth-token' 6import { OAuthTokenModel } from '../models/oauth/oauth-token'
7import { CONFIG } from '../initializers/constants' 7import { CACHE } from '../initializers/constants'
8import { Transaction } from 'sequelize' 8import { Transaction } from 'sequelize'
9import { CONFIG } from '../initializers/config'
9 10
10type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } 11type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
11const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} 12let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
12const userHavingToken: { [ userId: number ]: string } = {} 13let userHavingToken: { [ userId: number ]: string } = {}
13 14
14// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
15 16
@@ -38,11 +39,19 @@ function clearCacheByToken (token: string) {
38function getAccessToken (bearerToken: string) { 39function getAccessToken (bearerToken: string) {
39 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') 40 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').')
40 41
42 if (!bearerToken) return Bluebird.resolve(undefined)
43
41 if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) 44 if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken])
42 45
43 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 46 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
44 .then(tokenModel => { 47 .then(tokenModel => {
45 if (tokenModel) { 48 if (tokenModel) {
49 // Reinit our cache
50 if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
51 accessTokenCache = {}
52 userHavingToken = {}
53 }
54
46 accessTokenCache[ bearerToken ] = tokenModel 55 accessTokenCache[ bearerToken ] = tokenModel
47 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken 56 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
48 } 57 }
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 3628c0583..f77d0b62c 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -3,12 +3,13 @@ import { createClient, RedisClient } from 'redis'
3import { logger } from '../helpers/logger' 3import { logger } from '../helpers/logger'
4import { generateRandomString } from '../helpers/utils' 4import { generateRandomString } from '../helpers/utils'
5import { 5import {
6 CONFIG,
7 CONTACT_FORM_LIFETIME, 6 CONTACT_FORM_LIFETIME,
8 USER_EMAIL_VERIFY_LIFETIME, 7 USER_EMAIL_VERIFY_LIFETIME,
9 USER_PASSWORD_RESET_LIFETIME, 8 USER_PASSWORD_RESET_LIFETIME,
10 VIDEO_VIEW_LIFETIME 9 VIDEO_VIEW_LIFETIME,
11} from '../initializers' 10 WEBSERVER
11} from '../initializers/constants'
12import { CONFIG } from '../initializers/config'
12 13
13type CachedRoute = { 14type CachedRoute = {
14 body: string, 15 body: string,
@@ -30,7 +31,7 @@ class Redis {
30 if (this.initialized === true) return 31 if (this.initialized === true) return
31 this.initialized = true 32 this.initialized = true
32 33
33 this.client = createClient(Redis.getRedisClient()) 34 this.client = createClient(Redis.getRedisClientOptions())
34 35
35 this.client.on('error', err => { 36 this.client.on('error', err => {
36 logger.error('Error in Redis client.', { err }) 37 logger.error('Error in Redis client.', { err })
@@ -41,10 +42,10 @@ class Redis {
41 this.client.auth(CONFIG.REDIS.AUTH) 42 this.client.auth(CONFIG.REDIS.AUTH)
42 } 43 }
43 44
44 this.prefix = 'redis-' + CONFIG.WEBSERVER.HOST + '-' 45 this.prefix = 'redis-' + WEBSERVER.HOST + '-'
45 } 46 }
46 47
47 static getRedisClient () { 48 static getRedisClientOptions () {
48 return Object.assign({}, 49 return Object.assign({},
49 (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, 50 (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {},
50 (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, 51 (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {},
@@ -54,6 +55,14 @@ class Redis {
54 ) 55 )
55 } 56 }
56 57
58 getClient () {
59 return this.client
60 }
61
62 getPrefix () {
63 return this.prefix
64 }
65
57 /************* Forgot password *************/ 66 /************* Forgot password *************/
58 67
59 async setResetPasswordVerificationString (userId: number) { 68 async setResetPasswordVerificationString (userId: number) {
@@ -88,7 +97,7 @@ class Redis {
88 return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) 97 return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME)
89 } 98 }
90 99
91 async isContactFormIpExists (ip: string) { 100 async doesContactFormIpExist (ip: string) {
92 return this.exists(this.generateContactFormKey(ip)) 101 return this.exists(this.generateContactFormKey(ip))
93 } 102 }
94 103
@@ -98,7 +107,7 @@ class Redis {
98 return this.setValue(this.generateViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME) 107 return this.setValue(this.generateViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME)
99 } 108 }
100 109
101 async isVideoIPViewExists (ip: string, videoUUID: string) { 110 async doesVideoIPViewExist (ip: string, videoUUID: string) {
102 return this.exists(this.generateViewKey(ip, videoUUID)) 111 return this.exists(this.generateViewKey(ip, videoUUID))
103 } 112 }
104 113
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts
index 86ea7aa38..0e6088911 100644
--- a/server/lib/schedulers/abstract-scheduler.ts
+++ b/server/lib/schedulers/abstract-scheduler.ts
@@ -1,4 +1,5 @@
1import { logger } from '../../helpers/logger' 1import { logger } from '../../helpers/logger'
2import * as Bluebird from 'bluebird'
2 3
3export abstract class AbstractScheduler { 4export abstract class AbstractScheduler {
4 5
@@ -30,5 +31,5 @@ export abstract class AbstractScheduler {
30 } 31 }
31 } 32 }
32 33
33 protected abstract internalExecute (): Promise<any> 34 protected abstract internalExecute (): Promise<any> | Bluebird<any>
34} 35}
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts
index 3967be7f8..fdd3ad5fa 100644
--- a/server/lib/schedulers/actor-follow-scheduler.ts
+++ b/server/lib/schedulers/actor-follow-scheduler.ts
@@ -2,8 +2,8 @@ import { isTestInstance } from '../../helpers/core-utils'
2import { logger } from '../../helpers/logger' 2import { logger } from '../../helpers/logger'
3import { ActorFollowModel } from '../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../models/activitypub/actor-follow'
4import { AbstractScheduler } from './abstract-scheduler' 4import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
6import { ActorFollowScoreCache } from '../cache' 6import { ActorFollowScoreCache } from '../files-cache'
7 7
8export class ActorFollowScheduler extends AbstractScheduler { 8export class ActorFollowScheduler extends AbstractScheduler {
9 9
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts
new file mode 100644
index 000000000..1b5ff8394
--- /dev/null
+++ b/server/lib/schedulers/remove-old-history-scheduler.ts
@@ -0,0 +1,32 @@
1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler'
3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { UserVideoHistoryModel } from '../../models/account/user-video-history'
5import { CONFIG } from '../../initializers/config'
6import { isTestInstance } from '../../helpers/core-utils'
7
8export class RemoveOldHistoryScheduler extends AbstractScheduler {
9
10 private static instance: AbstractScheduler
11
12 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory
13
14 private constructor () {
15 super()
16 }
17
18 protected internalExecute () {
19 if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
20
21 logger.info('Removing old videos history.')
22
23 const now = new Date()
24 const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
25
26 return UserVideoHistoryModel.removeOldHistory(beforeDate)
27 }
28
29 static get Instance () {
30 return this.instance || (this.instance = new this())
31 }
32}
diff --git a/server/lib/schedulers/remove-old-jobs-scheduler.ts b/server/lib/schedulers/remove-old-jobs-scheduler.ts
index 4a4341ba9..0179a7618 100644
--- a/server/lib/schedulers/remove-old-jobs-scheduler.ts
+++ b/server/lib/schedulers/remove-old-jobs-scheduler.ts
@@ -2,7 +2,7 @@ import { isTestInstance } from '../../helpers/core-utils'
2import { logger } from '../../helpers/logger' 2import { logger } from '../../helpers/logger'
3import { JobQueue } from '../job-queue' 3import { JobQueue } from '../job-queue'
4import { AbstractScheduler } from './abstract-scheduler' 4import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
6 6
7export class RemoveOldJobsScheduler extends AbstractScheduler { 7export class RemoveOldJobsScheduler extends AbstractScheduler {
8 8
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts
new file mode 100644
index 000000000..39fbb9163
--- /dev/null
+++ b/server/lib/schedulers/remove-old-views-scheduler.ts
@@ -0,0 +1,33 @@
1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler'
3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { UserVideoHistoryModel } from '../../models/account/user-video-history'
5import { CONFIG } from '../../initializers/config'
6import { isTestInstance } from '../../helpers/core-utils'
7import { VideoViewModel } from '../../models/video/video-views'
8
9export class RemoveOldViewsScheduler extends AbstractScheduler {
10
11 private static instance: AbstractScheduler
12
13 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldViews
14
15 private constructor () {
16 super()
17 }
18
19 protected internalExecute () {
20 if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return
21
22 logger.info('Removing old videos views.')
23
24 const now = new Date()
25 const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
26
27 return VideoViewModel.removeOldRemoteViewsHistory(beforeDate)
28 }
29
30 static get Instance () {
31 return this.instance || (this.instance = new this())
32 }
33}
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 2618a5857..80080a132 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -3,10 +3,11 @@ import { AbstractScheduler } from './abstract-scheduler'
3import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' 3import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
4import { retryTransactionWrapper } from '../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../helpers/database-utils'
5import { federateVideoIfNeeded } from '../activitypub' 5import { federateVideoIfNeeded } from '../activitypub'
6import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' 6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { VideoPrivacy } from '../../../shared/models/videos' 7import { VideoPrivacy } from '../../../shared/models/videos'
8import { Notifier } from '../notifier' 8import { Notifier } from '../notifier'
9import { VideoModel } from '../../models/video/video' 9import { VideoModel } from '../../models/video/video'
10import { sequelizeTypescript } from '../../initializers/database'
10 11
11export class UpdateVideosScheduler extends AbstractScheduler { 12export class UpdateVideosScheduler extends AbstractScheduler {
12 13
@@ -57,7 +58,7 @@ export class UpdateVideosScheduler extends AbstractScheduler {
57 58
58 for (const v of publishedVideos) { 59 for (const v of publishedVideos) {
59 Notifier.Instance.notifyOnNewVideo(v) 60 Notifier.Instance.notifyOnNewVideo(v)
60 Notifier.Instance.notifyOnPendingVideoPublished(v) 61 Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v)
61 } 62 }
62 } 63 }
63 64
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index f643ee226..01af1e9d2 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,5 +1,5 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' 2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,9 +9,20 @@ import { join } from 'path'
9import { move } from 'fs-extra' 9import { move } from 'fs-extra'
10import { getServerActor } from '../../helpers/utils' 10import { getServerActor } from '../../helpers/utils'
11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 12import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
13import { removeVideoRedundancy } from '../redundancy' 13import { removeVideoRedundancy } from '../redundancy'
14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { VideoModel } from '../../models/video/video'
17import { downloadPlaylistSegments } from '../hls'
18import { CONFIG } from '../../initializers/config'
19
20type CandidateToDuplicate = {
21 redundancy: VideosRedundancy,
22 video: VideoModel,
23 files: VideoFileModel[],
24 streamingPlaylists: VideoStreamingPlaylistModel[]
25}
15 26
16export class VideosRedundancyScheduler extends AbstractScheduler { 27export class VideosRedundancyScheduler extends AbstractScheduler {
17 28
@@ -24,28 +35,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
24 } 35 }
25 36
26 protected async internalExecute () { 37 protected async internalExecute () {
27 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 38 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
28 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) 39 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
29 40
30 try { 41 try {
31 const videoToDuplicate = await this.findVideoToDuplicate(obj) 42 const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
32 if (!videoToDuplicate) continue 43 if (!videoToDuplicate) continue
33 44
34 const videoFiles = videoToDuplicate.VideoFiles 45 const candidateToDuplicate = {
35 videoFiles.forEach(f => f.Video = videoToDuplicate) 46 video: videoToDuplicate,
47 redundancy: redundancyConfig,
48 files: videoToDuplicate.VideoFiles,
49 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
50 }
36 51
37 await this.purgeCacheIfNeeded(obj, videoFiles) 52 await this.purgeCacheIfNeeded(candidateToDuplicate)
38 53
39 if (await this.isTooHeavy(obj, videoFiles)) { 54 if (await this.isTooHeavy(candidateToDuplicate)) {
40 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 55 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
41 continue 56 continue
42 } 57 }
43 58
44 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 59 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)
45 60
46 await this.createVideoRedundancy(obj, videoFiles) 61 await this.createVideoRedundancies(candidateToDuplicate)
47 } catch (err) { 62 } catch (err) {
48 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 63 logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
49 } 64 }
50 } 65 }
51 66
@@ -63,25 +78,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
63 78
64 for (const redundancyModel of expired) { 79 for (const redundancyModel of expired) {
65 try { 80 try {
66 await this.extendsOrDeleteRedundancy(redundancyModel) 81 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
82 const candidate = {
83 redundancy: redundancyConfig,
84 video: null,
85 files: [],
86 streamingPlaylists: []
87 }
88
89 // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
90 if (!redundancyConfig || await this.isTooHeavy(candidate)) {
91 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
92 await removeVideoRedundancy(redundancyModel)
93 } else {
94 await this.extendsRedundancy(redundancyModel)
95 }
67 } catch (err) { 96 } catch (err) {
68 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) 97 logger.error(
98 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
99 { err }
100 )
69 } 101 }
70 } 102 }
71 } 103 }
72 104
73 private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { 105 private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
74 // Refresh the video, maybe it was deleted
75 const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
76
77 if (!video) {
78 logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
79
80 await redundancyModel.destroy()
81 return
82 }
83
84 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 106 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
107 // Redundancy strategy disabled, remove our redundancy instead of extending expiration
108 if (!redundancy) await removeVideoRedundancy(redundancyModel)
109
85 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) 110 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
86 } 111 }
87 112
@@ -112,49 +137,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
112 } 137 }
113 } 138 }
114 139
115 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 140 private async createVideoRedundancies (data: CandidateToDuplicate) {
116 const serverActor = await getServerActor() 141 const video = await this.loadAndRefreshVideo(data.video.url)
142
143 if (!video) {
144 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)
117 145
118 for (const file of filesToDuplicate) { 146 return
119 const video = await this.loadAndRefreshVideo(file.Video.url) 147 }
120 148
149 for (const file of data.files) {
121 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) 150 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
122 if (existingRedundancy) { 151 if (existingRedundancy) {
123 await this.extendsOrDeleteRedundancy(existingRedundancy) 152 await this.extendsRedundancy(existingRedundancy)
124 153
125 continue 154 continue
126 } 155 }
127 156
128 if (!video) { 157 await this.createVideoFileRedundancy(data.redundancy, video, file)
129 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) 158 }
159
160 for (const streamingPlaylist of data.streamingPlaylists) {
161 const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
162 if (existingRedundancy) {
163 await this.extendsRedundancy(existingRedundancy)
130 164
131 continue 165 continue
132 } 166 }
133 167
134 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 168 await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
169 }
170 }
135 171
136 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 172 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
137 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 173 file.Video = video
138 174
139 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) 175 const serverActor = await getServerActor()
140 176
141 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) 177 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
142 await move(tmpPath, destPath)
143 178
144 const createdModel = await VideoRedundancyModel.create({ 179 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
145 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 180 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
146 url: getVideoCacheFileActivityPubUrl(file),
147 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
148 strategy: redundancy.strategy,
149 videoFileId: file.id,
150 actorId: serverActor.id
151 })
152 createdModel.VideoFile = file
153 181
154 await sendCreateCacheFile(serverActor, createdModel) 182 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
155 183
156 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) 184 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
157 } 185 await move(tmpPath, destPath)
186
187 const createdModel = await VideoRedundancyModel.create({
188 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
189 url: getVideoCacheFileActivityPubUrl(file),
190 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
191 strategy: redundancy.strategy,
192 videoFileId: file.id,
193 actorId: serverActor.id
194 })
195
196 createdModel.VideoFile = file
197
198 await sendCreateCacheFile(serverActor, video, createdModel)
199
200 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
201 }
202
203 private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
204 playlist.Video = video
205
206 const serverActor = await getServerActor()
207
208 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
209
210 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
211 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
212
213 const createdModel = await VideoRedundancyModel.create({
214 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
215 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
216 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
217 strategy: redundancy.strategy,
218 videoStreamingPlaylistId: playlist.id,
219 actorId: serverActor.id
220 })
221
222 createdModel.VideoStreamingPlaylist = playlist
223
224 await sendCreateCacheFile(serverActor, video, createdModel)
225
226 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
158 } 227 }
159 228
160 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { 229 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@@ -168,8 +237,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
168 await sendUpdateCacheFile(serverActor, redundancy) 237 await sendUpdateCacheFile(serverActor, redundancy)
169 } 238 }
170 239
171 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 240 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
172 while (this.isTooHeavy(redundancy, filesToDuplicate)) { 241 while (this.isTooHeavy(candidateToDuplicate)) {
242 const redundancy = candidateToDuplicate.redundancy
173 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) 243 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
174 if (!toDelete) return 244 if (!toDelete) return
175 245
@@ -177,11 +247,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
177 } 247 }
178 } 248 }
179 249
180 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 250 private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
181 const maxSize = redundancy.size 251 const maxSize = candidateToDuplicate.redundancy.size
182 252
183 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) 253 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
184 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) 254 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
185 255
186 return totalWillDuplicate > maxSize 256 return totalWillDuplicate > maxSize
187 } 257 }
@@ -191,13 +261,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
191 } 261 }
192 262
193 private buildEntryLogId (object: VideoRedundancyModel) { 263 private buildEntryLogId (object: VideoRedundancyModel) {
194 return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 264 if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
265
266 return `${object.VideoStreamingPlaylist.playlistUrl}`
195 } 267 }
196 268
197 private getTotalFileSizes (files: VideoFileModel[]) { 269 private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
198 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size 270 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
199 271
200 return files.reduce(fileReducer, 0) 272 return files.reduce(fileReducer, 0) * playlists.length
201 } 273 }
202 274
203 private async loadAndRefreshVideo (videoUrl: string) { 275 private async loadAndRefreshVideo (videoUrl: string) {
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
index aa027116d..aefe6aba4 100644
--- a/server/lib/schedulers/youtube-dl-update-scheduler.ts
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -1,5 +1,5 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
3import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' 3import { updateYoutubeDLBinary } from '../../helpers/youtube-dl'
4 4
5export class YoutubeDlUpdateScheduler extends AbstractScheduler { 5export class YoutubeDlUpdateScheduler extends AbstractScheduler {
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
new file mode 100644
index 000000000..950b14c3b
--- /dev/null
+++ b/server/lib/thumbnail.ts
@@ -0,0 +1,151 @@
1import { VideoFileModel } from '../models/video/video-file'
2import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
3import { CONFIG } from '../initializers/config'
4import { PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants'
5import { VideoModel } from '../models/video/video'
6import { ThumbnailModel } from '../models/video/thumbnail'
7import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
8import { processImage } from '../helpers/image-utils'
9import { join } from 'path'
10import { downloadImage } from '../helpers/requests'
11import { VideoPlaylistModel } from '../models/video/video-playlist'
12
13type ImageSize = { height: number, width: number }
14
15function createPlaylistMiniatureFromExisting (inputPath: string, playlist: VideoPlaylistModel, keepOriginal = false, size?: ImageSize) {
16 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
17 const type = ThumbnailType.MINIATURE
18
19 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal)
20 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail })
21}
22
23function createPlaylistMiniatureFromUrl (fileUrl: string, playlist: VideoPlaylistModel, size?: ImageSize) {
24 const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
25 const type = ThumbnailType.MINIATURE
26
27 const thumbnailCreator = () => downloadImage(fileUrl, basePath, filename, { width, height })
28 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
29}
30
31function createVideoMiniatureFromUrl (fileUrl: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) {
32 const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
33 const thumbnailCreator = () => downloadImage(fileUrl, basePath, filename, { width, height })
34
35 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
36}
37
38function createVideoMiniatureFromExisting (inputPath: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) {
39 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
40 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height })
41
42 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail })
43}
44
45function generateVideoMiniature (video: VideoModel, videoFile: VideoFileModel, type: ThumbnailType) {
46 const input = video.getVideoFilePath(videoFile)
47
48 const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type)
49 const thumbnailCreator = () => generateImageFromVideoFile(input, basePath, filename, { height, width })
50
51 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail })
52}
53
54function createPlaceholderThumbnail (fileUrl: string, video: VideoModel, type: ThumbnailType, size: ImageSize) {
55 const { filename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
56
57 const thumbnail = existingThumbnail ? existingThumbnail : new ThumbnailModel()
58
59 thumbnail.filename = filename
60 thumbnail.height = height
61 thumbnail.width = width
62 thumbnail.type = type
63 thumbnail.fileUrl = fileUrl
64
65 return thumbnail
66}
67
68// ---------------------------------------------------------------------------
69
70export {
71 generateVideoMiniature,
72 createVideoMiniatureFromUrl,
73 createVideoMiniatureFromExisting,
74 createPlaceholderThumbnail,
75 createPlaylistMiniatureFromUrl,
76 createPlaylistMiniatureFromExisting
77}
78
79function buildMetadataFromPlaylist (playlist: VideoPlaylistModel, size: ImageSize) {
80 const filename = playlist.generateThumbnailName()
81 const basePath = CONFIG.STORAGE.THUMBNAILS_DIR
82
83 return {
84 filename,
85 basePath,
86 existingThumbnail: playlist.Thumbnail,
87 outputPath: join(basePath, filename),
88 height: size ? size.height : THUMBNAILS_SIZE.height,
89 width: size ? size.width : THUMBNAILS_SIZE.width
90 }
91}
92
93function buildMetadataFromVideo (video: VideoModel, type: ThumbnailType, size?: ImageSize) {
94 const existingThumbnail = Array.isArray(video.Thumbnails)
95 ? video.Thumbnails.find(t => t.type === type)
96 : undefined
97
98 if (type === ThumbnailType.MINIATURE) {
99 const filename = video.generateThumbnailName()
100 const basePath = CONFIG.STORAGE.THUMBNAILS_DIR
101
102 return {
103 filename,
104 basePath,
105 existingThumbnail,
106 outputPath: join(basePath, filename),
107 height: size ? size.height : THUMBNAILS_SIZE.height,
108 width: size ? size.width : THUMBNAILS_SIZE.width
109 }
110 }
111
112 if (type === ThumbnailType.PREVIEW) {
113 const filename = video.generatePreviewName()
114 const basePath = CONFIG.STORAGE.PREVIEWS_DIR
115
116 return {
117 filename,
118 basePath,
119 existingThumbnail,
120 outputPath: join(basePath, filename),
121 height: size ? size.height : PREVIEWS_SIZE.height,
122 width: size ? size.width : PREVIEWS_SIZE.width
123 }
124 }
125
126 return undefined
127}
128
129async function createThumbnailFromFunction (parameters: {
130 thumbnailCreator: () => Promise<any>,
131 filename: string,
132 height: number,
133 width: number,
134 type: ThumbnailType,
135 fileUrl?: string,
136 existingThumbnail?: ThumbnailModel
137}) {
138 const { thumbnailCreator, filename, width, height, type, existingThumbnail, fileUrl = null } = parameters
139
140 const thumbnail = existingThumbnail ? existingThumbnail : new ThumbnailModel()
141
142 thumbnail.filename = filename
143 thumbnail.height = height
144 thumbnail.width = width
145 thumbnail.type = type
146 thumbnail.fileUrl = fileUrl
147
148 await thumbnailCreator()
149
150 return thumbnail
151}
diff --git a/server/lib/user.ts b/server/lib/user.ts
index a39ef6c3d..7badb3e72 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -1,18 +1,19 @@
1import * as Sequelize from 'sequelize' 1import * as Sequelize from 'sequelize'
2import * as uuidv4 from 'uuid/v4' 2import * as uuidv4 from 'uuid/v4'
3import { ActivityPubActorType } from '../../shared/models/activitypub' 3import { ActivityPubActorType } from '../../shared/models/activitypub'
4import { sequelizeTypescript, SERVER_ACTOR_NAME } from '../initializers' 4import { SERVER_ACTOR_NAME } from '../initializers/constants'
5import { AccountModel } from '../models/account/account' 5import { AccountModel } from '../models/account/account'
6import { UserModel } from '../models/account/user' 6import { UserModel } from '../models/account/user'
7import { buildActorInstance, getAccountActivityPubUrl, setAsyncActorKeys } from './activitypub' 7import { buildActorInstance, getAccountActivityPubUrl, setAsyncActorKeys } from './activitypub'
8import { createVideoChannel } from './video-channel' 8import { createVideoChannel } from './video-channel'
9import { VideoChannelModel } from '../models/video/video-channel' 9import { VideoChannelModel } from '../models/video/video-channel'
10import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
11import { ActorModel } from '../models/activitypub/actor' 10import { ActorModel } from '../models/activitypub/actor'
12import { UserNotificationSettingModel } from '../models/account/user-notification-setting' 11import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
13import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' 12import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users'
13import { createWatchLaterPlaylist } from './video-playlist'
14import { sequelizeTypescript } from '../initializers/database'
14 15
15async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { 16async function createUserAccountAndChannelAndPlaylist (userToCreate: UserModel, validateUser = true) {
16 const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { 17 const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => {
17 const userOptions = { 18 const userOptions = {
18 transaction: t, 19 transaction: t,
@@ -38,7 +39,9 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse
38 } 39 }
39 const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) 40 const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t)
40 41
41 return { user: userCreated, account: accountCreated, videoChannel } 42 const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t)
43
44 return { user: userCreated, account: accountCreated, videoChannel, videoPlaylist }
42 }) 45 })
43 46
44 const [ accountKeys, channelKeys ] = await Promise.all([ 47 const [ accountKeys, channelKeys ] = await Promise.all([
@@ -69,7 +72,7 @@ async function createLocalAccountWithoutKeys (
69 userId, 72 userId,
70 applicationId, 73 applicationId,
71 actorId: actorInstanceCreated.id 74 actorId: actorInstanceCreated.id
72 } as FilteredModelAttributes<AccountModel>) 75 })
73 76
74 const accountInstanceCreated = await accountInstance.save({ transaction: t }) 77 const accountInstanceCreated = await accountInstance.save({ transaction: t })
75 accountInstanceCreated.Actor = actorInstanceCreated 78 accountInstanceCreated.Actor = actorInstanceCreated
@@ -89,7 +92,7 @@ async function createApplicationActor (applicationId: number) {
89 92
90export { 93export {
91 createApplicationActor, 94 createApplicationActor,
92 createUserAccountAndChannel, 95 createUserAccountAndChannelAndPlaylist,
93 createLocalAccountWithoutKeys 96 createLocalAccountWithoutKeys
94} 97}
95 98
@@ -103,10 +106,12 @@ function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Tr
103 myVideoImportFinished: UserNotificationSettingValue.WEB, 106 myVideoImportFinished: UserNotificationSettingValue.WEB,
104 myVideoPublished: UserNotificationSettingValue.WEB, 107 myVideoPublished: UserNotificationSettingValue.WEB,
105 videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, 108 videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL,
109 videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL,
106 blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, 110 blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL,
107 newUserRegistration: UserNotificationSettingValue.WEB, 111 newUserRegistration: UserNotificationSettingValue.WEB,
108 commentMention: UserNotificationSettingValue.WEB, 112 commentMention: UserNotificationSettingValue.WEB,
109 newFollow: UserNotificationSettingValue.WEB 113 newFollow: UserNotificationSettingValue.WEB,
114 newInstanceFollower: UserNotificationSettingValue.WEB
110 } 115 }
111 116
112 return UserNotificationSettingModel.create(values, { transaction: t }) 117 return UserNotificationSettingModel.create(values, { transaction: t })
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts
new file mode 100644
index 000000000..985b89e31
--- /dev/null
+++ b/server/lib/video-blacklist.ts
@@ -0,0 +1,33 @@
1import * as sequelize from 'sequelize'
2import { CONFIG } from '../initializers/config'
3import { UserRight, VideoBlacklistType } from '../../shared/models'
4import { VideoBlacklistModel } from '../models/video/video-blacklist'
5import { UserModel } from '../models/account/user'
6import { VideoModel } from '../models/video/video'
7import { logger } from '../helpers/logger'
8import { UserAdminFlag } from '../../shared/models/users/user-flag.model'
9
10async function autoBlacklistVideoIfNeeded (video: VideoModel, user: UserModel, transaction: sequelize.Transaction) {
11 if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED) return false
12
13 if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BY_PASS_VIDEO_AUTO_BLACKLIST)) return false
14
15 const sequelizeOptions = { transaction }
16 const videoBlacklistToCreate = {
17 videoId: video.id,
18 unfederated: true,
19 reason: 'Auto-blacklisted. Moderator review required.',
20 type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED
21 }
22 await VideoBlacklistModel.create(videoBlacklistToCreate, sequelizeOptions)
23
24 logger.info('Video %s auto-blacklisted.', video.uuid)
25
26 return true
27}
28
29// ---------------------------------------------------------------------------
30
31export {
32 autoBlacklistVideoIfNeeded
33}
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts
index 59bce7520..bfe22d225 100644
--- a/server/lib/video-comment.ts
+++ b/server/lib/video-comment.ts
@@ -28,7 +28,7 @@ async function createVideoComment (obj: {
28 videoId: obj.video.id, 28 videoId: obj.video.id,
29 accountId: obj.account.id, 29 accountId: obj.account.id,
30 url: 'fake url' 30 url: 'fake url'
31 }, { transaction: t, validate: false }) 31 }, { transaction: t, validate: false } as any) // FIXME: sequelize typings
32 32
33 comment.set('url', getVideoCommentActivityPubUrl(obj.video, comment)) 33 comment.set('url', getVideoCommentActivityPubUrl(obj.video, comment))
34 34
diff --git a/server/lib/video-playlist.ts b/server/lib/video-playlist.ts
new file mode 100644
index 000000000..6e214e60f
--- /dev/null
+++ b/server/lib/video-playlist.ts
@@ -0,0 +1,29 @@
1import * as Sequelize from 'sequelize'
2import { AccountModel } from '../models/account/account'
3import { VideoPlaylistModel } from '../models/video/video-playlist'
4import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model'
5import { getVideoPlaylistActivityPubUrl } from './activitypub'
6import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model'
7
8async function createWatchLaterPlaylist (account: AccountModel, t: Sequelize.Transaction) {
9 const videoPlaylist = new VideoPlaylistModel({
10 name: 'Watch later',
11 privacy: VideoPlaylistPrivacy.PRIVATE,
12 type: VideoPlaylistType.WATCH_LATER,
13 ownerAccountId: account.id
14 })
15
16 videoPlaylist.url = getVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object
17
18 await videoPlaylist.save({ transaction: t })
19
20 videoPlaylist.OwnerAccount = account
21
22 return videoPlaylist
23}
24
25// ---------------------------------------------------------------------------
26
27export {
28 createWatchLaterPlaylist
29}
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 4460f46e4..0fe0ff12a 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,11 +1,15 @@
1import { CONFIG } from '../initializers' 1import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants'
2import { extname, join } from 'path' 2import { join } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' 3import { getVideoFileFPS, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, move, stat } from 'fs-extra' 4import { ensureDir, move, remove, stat } from 'fs-extra'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos' 6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file' 7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video' 8import { VideoModel } from '../models/video/video'
9import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
11import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
12import { CONFIG } from '../initializers/config'
9 13
10async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { 14async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 15 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
@@ -17,7 +21,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
17 21
18 const transcodeOptions = { 22 const transcodeOptions = {
19 inputPath: videoInputPath, 23 inputPath: videoInputPath,
20 outputPath: videoTranscodedPath 24 outputPath: videoTranscodedPath,
25 resolution: inputVideoFile.resolution
21 } 26 }
22 27
23 // Could be very long! 28 // Could be very long!
@@ -47,7 +52,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
47 } 52 }
48} 53}
49 54
50async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { 55async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
51 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 56 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
52 const extname = '.mp4' 57 const extname = '.mp4'
53 58
@@ -60,13 +65,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
60 size: 0, 65 size: 0,
61 videoId: video.id 66 videoId: video.id
62 }) 67 })
63 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) 68 const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
64 69
65 const transcodeOptions = { 70 const transcodeOptions = {
66 inputPath: videoInputPath, 71 inputPath: videoInputPath,
67 outputPath: videoOutputPath, 72 outputPath: videoOutputPath,
68 resolution, 73 resolution,
69 isPortraitMode 74 isPortraitMode: isPortrait
70 } 75 }
71 76
72 await transcode(transcodeOptions) 77 await transcode(transcodeOptions)
@@ -84,48 +89,44 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
84 video.VideoFiles.push(newVideoFile) 89 video.VideoFiles.push(newVideoFile)
85} 90}
86 91
87async function importVideoFile (video: VideoModel, inputFilePath: string) { 92async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
88 const { videoFileResolution } = await getVideoFileResolution(inputFilePath) 93 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
89 const { size } = await stat(inputFilePath) 94 await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid))
90 const fps = await getVideoFileFPS(inputFilePath)
91 95
92 let updatedVideoFile = new VideoFileModel({ 96 const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
93 resolution: videoFileResolution, 97 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
94 extname: extname(inputFilePath),
95 size,
96 fps,
97 videoId: video.id
98 })
99
100 const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
101 98
102 if (currentVideoFile) { 99 const transcodeOptions = {
103 // Remove old file and old torrent 100 inputPath: videoInputPath,
104 await video.removeFile(currentVideoFile) 101 outputPath,
105 await video.removeTorrent(currentVideoFile) 102 resolution,
106 // Remove the old video file from the array 103 isPortraitMode,
107 video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
108
109 // Update the database
110 currentVideoFile.set('extname', updatedVideoFile.extname)
111 currentVideoFile.set('size', updatedVideoFile.size)
112 currentVideoFile.set('fps', updatedVideoFile.fps)
113 104
114 updatedVideoFile = currentVideoFile 105 hlsPlaylist: {
106 videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution)
107 }
115 } 108 }
116 109
117 const outputPath = video.getVideoFilePath(updatedVideoFile) 110 await transcode(transcodeOptions)
118 await copy(inputFilePath, outputPath)
119 111
120 await video.createTorrentAndSetInfoHash(updatedVideoFile) 112 await updateMasterHLSPlaylist(video)
113 await updateSha256Segments(video)
121 114
122 await updatedVideoFile.save() 115 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
123 116
124 video.VideoFiles.push(updatedVideoFile) 117 await VideoStreamingPlaylistModel.upsert({
118 videoId: video.id,
119 playlistUrl,
120 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
121 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
122 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
123
124 type: VideoStreamingPlaylistType.HLS
125 })
125} 126}
126 127
127export { 128export {
129 generateHlsPlaylist,
128 optimizeVideofile, 130 optimizeVideofile,
129 transcodeOriginalVideofile, 131 transcodeOriginalVideofile
130 importVideoFile
131} 132}