diff options
Diffstat (limited to 'server/lib')
65 files changed, 2039 insertions, 546 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 8215840da..25cd40905 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -12,7 +12,7 @@ import { logger } from '../../helpers/logger' | |||
12 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' | 12 | import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' |
13 | import { doRequest, downloadImage } from '../../helpers/requests' | 13 | import { doRequest, downloadImage } from '../../helpers/requests' |
14 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | 14 | import { getUrlFromWebfinger } from '../../helpers/webfinger' |
15 | import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers' | 15 | import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants' |
16 | import { AccountModel } from '../../models/account/account' | 16 | import { AccountModel } from '../../models/account/account' |
17 | import { ActorModel } from '../../models/activitypub/actor' | 17 | import { ActorModel } from '../../models/activitypub/actor' |
18 | import { AvatarModel } from '../../models/avatar/avatar' | 18 | import { AvatarModel } from '../../models/avatar/avatar' |
@@ -21,6 +21,8 @@ import { VideoChannelModel } from '../../models/video/video-channel' | |||
21 | import { JobQueue } from '../job-queue' | 21 | import { JobQueue } from '../job-queue' |
22 | import { getServerActor } from '../../helpers/utils' | 22 | import { getServerActor } from '../../helpers/utils' |
23 | import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' | 23 | import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' |
24 | import { CONFIG } from '../../initializers/config' | ||
25 | import { sequelizeTypescript } from '../../initializers/database' | ||
24 | 26 | ||
25 | // Set account keys, this could be long so process after the account creation and do not block the client | 27 | // Set account keys, this could be long so process after the account creation and do not block the client |
26 | function setAsyncActorKeys (actor: ActorModel) { | 28 | function setAsyncActorKeys (actor: ActorModel) { |
@@ -44,6 +46,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
44 | ) { | 46 | ) { |
45 | const actorUrl = getAPId(activityActor) | 47 | const actorUrl = getAPId(activityActor) |
46 | let created = false | 48 | let created = false |
49 | let accountPlaylistsUrl: string | ||
47 | 50 | ||
48 | let actor = await fetchActorByUrl(actorUrl, fetchType) | 51 | let actor = await fetchActorByUrl(actorUrl, fetchType) |
49 | // Orphan actor (not associated to an account of channel) so recreate it | 52 | // Orphan actor (not associated to an account of channel) so recreate it |
@@ -70,7 +73,8 @@ async function getOrCreateActorAndServerAndModel ( | |||
70 | 73 | ||
71 | try { | 74 | try { |
72 | // Don't recurse another time | 75 | // Don't recurse another time |
73 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) | 76 | const recurseIfNeeded = false |
77 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded) | ||
74 | } catch (err) { | 78 | } catch (err) { |
75 | logger.error('Cannot get or create account attributed to video channel ' + actor.url) | 79 | logger.error('Cannot get or create account attributed to video channel ' + actor.url) |
76 | throw new Error(err) | 80 | throw new Error(err) |
@@ -79,6 +83,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
79 | 83 | ||
80 | actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor) | 84 | actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor) |
81 | created = true | 85 | created = true |
86 | accountPlaylistsUrl = result.playlists | ||
82 | } | 87 | } |
83 | 88 | ||
84 | if (actor.Account) actor.Account.Actor = actor | 89 | if (actor.Account) actor.Account.Actor = actor |
@@ -92,6 +97,12 @@ async function getOrCreateActorAndServerAndModel ( | |||
92 | await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | 97 | await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) |
93 | } | 98 | } |
94 | 99 | ||
100 | // We created a new account: fetch the playlists | ||
101 | if (created === true && actor.Account && accountPlaylistsUrl) { | ||
102 | const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' } | ||
103 | await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | ||
104 | } | ||
105 | |||
95 | return actorRefreshed | 106 | return actorRefreshed |
96 | } | 107 | } |
97 | 108 | ||
@@ -107,7 +118,7 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU | |||
107 | followingCount: 0, | 118 | followingCount: 0, |
108 | inboxUrl: url + '/inbox', | 119 | inboxUrl: url + '/inbox', |
109 | outboxUrl: url + '/outbox', | 120 | outboxUrl: url + '/outbox', |
110 | sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox', | 121 | sharedInboxUrl: WEBSERVER.URL + '/inbox', |
111 | followersUrl: url + '/followers', | 122 | followersUrl: url + '/followers', |
112 | followingUrl: url + '/following' | 123 | followingUrl: url + '/following' |
113 | }) | 124 | }) |
@@ -259,7 +270,7 @@ async function refreshActorIfNeeded ( | |||
259 | return { refreshed: true, actor } | 270 | return { refreshed: true, actor } |
260 | }) | 271 | }) |
261 | } catch (err) { | 272 | } catch (err) { |
262 | logger.warn('Cannot refresh actor.', { err }) | 273 | logger.warn('Cannot refresh actor %s.', actor.url, { err }) |
263 | return { actor, refreshed: false } | 274 | return { actor, refreshed: false } |
264 | } | 275 | } |
265 | } | 276 | } |
@@ -333,6 +344,8 @@ function saveActorAndServerAndModelIfNotExist ( | |||
333 | actorCreated.VideoChannel.Account = ownerActor.Account | 344 | actorCreated.VideoChannel.Account = ownerActor.Account |
334 | } | 345 | } |
335 | 346 | ||
347 | actorCreated.Server = server | ||
348 | |||
336 | return actorCreated | 349 | return actorCreated |
337 | } | 350 | } |
338 | } | 351 | } |
@@ -342,6 +355,7 @@ type FetchRemoteActorResult = { | |||
342 | name: string | 355 | name: string |
343 | summary: string | 356 | summary: string |
344 | support?: string | 357 | support?: string |
358 | playlists?: string | ||
345 | avatarName?: string | 359 | avatarName?: string |
346 | attributedTo: ActivityPubAttributedTo[] | 360 | attributedTo: ActivityPubAttributedTo[] |
347 | } | 361 | } |
@@ -355,17 +369,18 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
355 | 369 | ||
356 | logger.info('Fetching remote actor %s.', actorUrl) | 370 | logger.info('Fetching remote actor %s.', actorUrl) |
357 | 371 | ||
358 | const requestResult = await doRequest(options) | 372 | const requestResult = await doRequest<ActivityPubActor>(options) |
359 | normalizeActor(requestResult.body) | 373 | normalizeActor(requestResult.body) |
360 | 374 | ||
361 | const actorJSON: ActivityPubActor = requestResult.body | 375 | const actorJSON = requestResult.body |
362 | if (isActorObjectValid(actorJSON) === false) { | 376 | if (isActorObjectValid(actorJSON) === false) { |
363 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) | 377 | logger.debug('Remote actor JSON is not valid.', { actorJSON }) |
364 | return { result: undefined, statusCode: requestResult.response.statusCode } | 378 | return { result: undefined, statusCode: requestResult.response.statusCode } |
365 | } | 379 | } |
366 | 380 | ||
367 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { | 381 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { |
368 | throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) | 382 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) |
383 | return { result: undefined, statusCode: requestResult.response.statusCode } | ||
369 | } | 384 | } |
370 | 385 | ||
371 | const followersCount = await fetchActorTotalItems(actorJSON.followers) | 386 | const followersCount = await fetchActorTotalItems(actorJSON.followers) |
@@ -398,6 +413,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
398 | avatarName, | 413 | avatarName, |
399 | summary: actorJSON.summary, | 414 | summary: actorJSON.summary, |
400 | support: actorJSON.support, | 415 | support: actorJSON.support, |
416 | playlists: actorJSON.playlists, | ||
401 | attributedTo: actorJSON.attributedTo | 417 | attributedTo: actorJSON.attributedTo |
402 | } | 418 | } |
403 | } | 419 | } |
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts index 10277eca7..771a01366 100644 --- a/server/lib/activitypub/audience.ts +++ b/server/lib/activitypub/audience.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { ActivityAudience } from '../../../shared/models/activitypub' | 2 | import { ActivityAudience } from '../../../shared/models/activitypub' |
3 | import { ACTIVITY_PUB } from '../../initializers' | 3 | import { ACTIVITY_PUB } from '../../initializers/constants' |
4 | import { ActorModel } from '../../models/activitypub/actor' | 4 | import { ActorModel } from '../../models/activitypub/actor' |
5 | import { VideoModel } from '../../models/video/video' | 5 | import { VideoModel } from '../../models/video/video' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index f6f068b45..de5cc54ac 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -2,10 +2,27 @@ import { CacheFileObject } from '../../../shared/index' | |||
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
4 | import { Transaction } from 'sequelize' | 4 | import { Transaction } from 'sequelize' |
5 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
5 | 6 | ||
6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
7 | const url = cacheFileObject.url | ||
8 | 8 | ||
9 | if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { | ||
10 | const url = cacheFileObject.url | ||
11 | |||
12 | const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) | ||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | ||
14 | |||
15 | return { | ||
16 | expiresOn: new Date(cacheFileObject.expires), | ||
17 | url: cacheFileObject.id, | ||
18 | fileUrl: url.href, | ||
19 | strategy: null, | ||
20 | videoStreamingPlaylistId: playlist.id, | ||
21 | actorId: byActor.id | ||
22 | } | ||
23 | } | ||
24 | |||
25 | const url = cacheFileObject.url | ||
9 | const videoFile = video.VideoFiles.find(f => { | 26 | const videoFile = video.VideoFiles.find(f => { |
10 | return f.resolution === url.height && f.fps === url.fps | 27 | return f.resolution === url.height && f.fps === url.fps |
11 | }) | 28 | }) |
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
15 | return { | 32 | return { |
16 | expiresOn: new Date(cacheFileObject.expires), | 33 | expiresOn: new Date(cacheFileObject.expires), |
17 | url: cacheFileObject.id, | 34 | url: cacheFileObject.id, |
18 | fileUrl: cacheFileObject.url.href, | 35 | fileUrl: url.href, |
19 | strategy: null, | 36 | strategy: null, |
20 | videoFileId: videoFile.id, | 37 | videoFileId: videoFile.id, |
21 | actorId: byActor.id | 38 | actorId: byActor.id |
@@ -51,8 +68,8 @@ function updateCacheFile ( | |||
51 | 68 | ||
52 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | 69 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) |
53 | 70 | ||
54 | redundancyModel.set('expires', attributes.expiresOn) | 71 | redundancyModel.expiresOn = attributes.expiresOn |
55 | redundancyModel.set('fileUrl', attributes.fileUrl) | 72 | redundancyModel.fileUrl = attributes.fileUrl |
56 | 73 | ||
57 | return redundancyModel.save({ transaction: t }) | 74 | return redundancyModel.save({ transaction: t }) |
58 | } | 75 | } |
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 1b9b14c2e..686eef04d 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts | |||
@@ -1,10 +1,14 @@ | |||
1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' | 1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' |
2 | import { doRequest } from '../../helpers/requests' | 2 | import { doRequest } from '../../helpers/requests' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import * as Bluebird from 'bluebird' | 4 | import * as Bluebird from 'bluebird' |
5 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | 5 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' |
6 | import { parse } from 'url' | ||
6 | 7 | ||
7 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { | 8 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) |
9 | type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) | ||
10 | |||
11 | async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) { | ||
8 | logger.info('Crawling ActivityPub data on %s.', uri) | 12 | logger.info('Crawling ActivityPub data on %s.', uri) |
9 | 13 | ||
10 | const options = { | 14 | const options = { |
@@ -15,6 +19,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr | |||
15 | timeout: JOB_REQUEST_TIMEOUT | 19 | timeout: JOB_REQUEST_TIMEOUT |
16 | } | 20 | } |
17 | 21 | ||
22 | const startDate = new Date() | ||
23 | |||
18 | const response = await doRequest<ActivityPubOrderedCollection<T>>(options) | 24 | const response = await doRequest<ActivityPubOrderedCollection<T>>(options) |
19 | const firstBody = response.body | 25 | const firstBody = response.body |
20 | 26 | ||
@@ -22,6 +28,10 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr | |||
22 | let i = 0 | 28 | let i = 0 |
23 | let nextLink = firstBody.first | 29 | let nextLink = firstBody.first |
24 | while (nextLink && i < limit) { | 30 | while (nextLink && i < limit) { |
31 | // Don't crawl ourselves | ||
32 | const remoteHost = parse(nextLink).host | ||
33 | if (remoteHost === WEBSERVER.HOST) continue | ||
34 | |||
25 | options.uri = nextLink | 35 | options.uri = nextLink |
26 | 36 | ||
27 | const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options) | 37 | const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options) |
@@ -35,6 +45,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Pr | |||
35 | await handler(items) | 45 | await handler(items) |
36 | } | 46 | } |
37 | } | 47 | } |
48 | |||
49 | if (cleaner) await cleaner(startDate) | ||
38 | } | 50 | } |
39 | 51 | ||
40 | export { | 52 | export { |
diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts index 6906bf9d3..d8c7d83b7 100644 --- a/server/lib/activitypub/index.ts +++ b/server/lib/activitypub/index.ts | |||
@@ -2,6 +2,7 @@ export * from './process' | |||
2 | export * from './send' | 2 | export * from './send' |
3 | export * from './actor' | 3 | export * from './actor' |
4 | export * from './share' | 4 | export * from './share' |
5 | export * from './playlist' | ||
5 | export * from './videos' | 6 | export * from './videos' |
6 | export * from './video-comments' | 7 | export * from './video-comments' |
7 | export * from './video-rates' | 8 | export * from './video-rates' |
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts new file mode 100644 index 000000000..36a91faec --- /dev/null +++ b/server/lib/activitypub/playlist.ts | |||
@@ -0,0 +1,213 @@ | |||
1 | import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' | ||
2 | import { crawlCollectionPage } from './crawl' | ||
3 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | ||
4 | import { AccountModel } from '../../models/account/account' | ||
5 | import { isArray } from '../../helpers/custom-validators/misc' | ||
6 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
7 | import { logger } from '../../helpers/logger' | ||
8 | import { VideoPlaylistModel } from '../../models/video/video-playlist' | ||
9 | import { doRequest } from '../../helpers/requests' | ||
10 | import { checkUrlsSameHost } from '../../helpers/activitypub' | ||
11 | import * as Bluebird from 'bluebird' | ||
12 | import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' | ||
13 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | ||
14 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' | ||
15 | import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
18 | import { sequelizeTypescript } from '../../initializers/database' | ||
19 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' | ||
20 | import { FilteredModelAttributes } from '../../typings/sequelize' | ||
21 | |||
22 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) { | ||
23 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED | ||
24 | |||
25 | return { | ||
26 | name: playlistObject.name, | ||
27 | description: playlistObject.content, | ||
28 | privacy, | ||
29 | url: playlistObject.id, | ||
30 | uuid: playlistObject.uuid, | ||
31 | ownerAccountId: byAccount.id, | ||
32 | videoChannelId: null, | ||
33 | createdAt: new Date(playlistObject.published), | ||
34 | updatedAt: new Date(playlistObject.updated) | ||
35 | } | ||
36 | } | ||
37 | |||
38 | function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) { | ||
39 | return { | ||
40 | position: elementObject.position, | ||
41 | url: elementObject.id, | ||
42 | startTimestamp: elementObject.startTimestamp || null, | ||
43 | stopTimestamp: elementObject.stopTimestamp || null, | ||
44 | videoPlaylistId: videoPlaylist.id, | ||
45 | videoId: video.id | ||
46 | } | ||
47 | } | ||
48 | |||
49 | async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) { | ||
50 | await Bluebird.map(playlistUrls, async playlistUrl => { | ||
51 | try { | ||
52 | const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) | ||
53 | if (exists === true) return | ||
54 | |||
55 | // Fetch url | ||
56 | const { body } = await doRequest<PlaylistObject>({ | ||
57 | uri: playlistUrl, | ||
58 | json: true, | ||
59 | activityPub: true | ||
60 | }) | ||
61 | |||
62 | if (!isPlaylistObjectValid(body)) { | ||
63 | throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) | ||
64 | } | ||
65 | |||
66 | if (!isArray(body.to)) { | ||
67 | throw new Error('Playlist does not have an audience.') | ||
68 | } | ||
69 | |||
70 | return createOrUpdateVideoPlaylist(body, account, body.to) | ||
71 | } catch (err) { | ||
72 | logger.warn('Cannot add playlist element %s.', playlistUrl, { err }) | ||
73 | } | ||
74 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
75 | } | ||
76 | |||
77 | async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) { | ||
78 | const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to) | ||
79 | |||
80 | if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) { | ||
81 | const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0]) | ||
82 | |||
83 | if (actor.VideoChannel) { | ||
84 | playlistAttributes.videoChannelId = actor.VideoChannel.id | ||
85 | } else { | ||
86 | logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject }) | ||
87 | } | ||
88 | } | ||
89 | |||
90 | const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true }) | ||
91 | |||
92 | let accItems: string[] = [] | ||
93 | await crawlCollectionPage<string>(playlistObject.id, items => { | ||
94 | accItems = accItems.concat(items) | ||
95 | |||
96 | return Promise.resolve() | ||
97 | }) | ||
98 | |||
99 | const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null) | ||
100 | |||
101 | if (playlistObject.icon) { | ||
102 | try { | ||
103 | const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist) | ||
104 | await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined) | ||
105 | } catch (err) { | ||
106 | logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err }) | ||
107 | } | ||
108 | } | ||
109 | |||
110 | return resetVideoPlaylistElements(accItems, refreshedPlaylist) | ||
111 | } | ||
112 | |||
113 | async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> { | ||
114 | if (!videoPlaylist.isOutdated()) return videoPlaylist | ||
115 | |||
116 | try { | ||
117 | const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) | ||
118 | if (statusCode === 404) { | ||
119 | logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url) | ||
120 | |||
121 | await videoPlaylist.destroy() | ||
122 | return undefined | ||
123 | } | ||
124 | |||
125 | if (playlistObject === undefined) { | ||
126 | logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) | ||
127 | |||
128 | await videoPlaylist.setAsRefreshed() | ||
129 | return videoPlaylist | ||
130 | } | ||
131 | |||
132 | const byAccount = videoPlaylist.OwnerAccount | ||
133 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) | ||
134 | |||
135 | return videoPlaylist | ||
136 | } catch (err) { | ||
137 | logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) | ||
138 | |||
139 | await videoPlaylist.setAsRefreshed() | ||
140 | return videoPlaylist | ||
141 | } | ||
142 | } | ||
143 | |||
144 | // --------------------------------------------------------------------------- | ||
145 | |||
146 | export { | ||
147 | createAccountPlaylists, | ||
148 | playlistObjectToDBAttributes, | ||
149 | playlistElementObjectToDBAttributes, | ||
150 | createOrUpdateVideoPlaylist, | ||
151 | refreshVideoPlaylistIfNeeded | ||
152 | } | ||
153 | |||
154 | // --------------------------------------------------------------------------- | ||
155 | |||
156 | async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) { | ||
157 | const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = [] | ||
158 | |||
159 | await Bluebird.map(elementUrls, async elementUrl => { | ||
160 | try { | ||
161 | // Fetch url | ||
162 | const { body } = await doRequest<PlaylistElementObject>({ | ||
163 | uri: elementUrl, | ||
164 | json: true, | ||
165 | activityPub: true | ||
166 | }) | ||
167 | |||
168 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) | ||
169 | |||
170 | if (checkUrlsSameHost(body.id, elementUrl) !== true) { | ||
171 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) | ||
172 | } | ||
173 | |||
174 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' }) | ||
175 | |||
176 | elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) | ||
177 | } catch (err) { | ||
178 | logger.warn('Cannot add playlist element %s.', elementUrl, { err }) | ||
179 | } | ||
180 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
181 | |||
182 | await sequelizeTypescript.transaction(async t => { | ||
183 | await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) | ||
184 | |||
185 | for (const element of elementsToCreate) { | ||
186 | await VideoPlaylistElementModel.create(element, { transaction: t }) | ||
187 | } | ||
188 | }) | ||
189 | |||
190 | logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length) | ||
191 | |||
192 | return undefined | ||
193 | } | ||
194 | |||
195 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { | ||
196 | const options = { | ||
197 | uri: playlistUrl, | ||
198 | method: 'GET', | ||
199 | json: true, | ||
200 | activityPub: true | ||
201 | } | ||
202 | |||
203 | logger.info('Fetching remote playlist %s.', playlistUrl) | ||
204 | |||
205 | const { response, body } = await doRequest(options) | ||
206 | |||
207 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | ||
208 | logger.debug('Remote video playlist JSON is not valid.', { body }) | ||
209 | return { statusCode: response.statusCode, playlistObject: undefined } | ||
210 | } | ||
211 | |||
212 | return { statusCode: response.statusCode, playlistObject: body } | ||
213 | } | ||
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 5f4d793a5..e882669ce 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -12,6 +12,8 @@ import { Notifier } from '../../notifier' | |||
12 | import { processViewActivity } from './process-view' | 12 | import { processViewActivity } from './process-view' |
13 | import { processDislikeActivity } from './process-dislike' | 13 | import { processDislikeActivity } from './process-dislike' |
14 | import { processFlagActivity } from './process-flag' | 14 | import { processFlagActivity } from './process-flag' |
15 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' | ||
16 | import { createOrUpdateVideoPlaylist } from '../playlist' | ||
15 | 17 | ||
16 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { | 18 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { |
17 | const activityObject = activity.object | 19 | const activityObject = activity.object |
@@ -38,7 +40,11 @@ async function processCreateActivity (activity: ActivityCreate, byActor: ActorMo | |||
38 | } | 40 | } |
39 | 41 | ||
40 | if (activityType === 'CacheFile') { | 42 | if (activityType === 'CacheFile') { |
41 | return retryTransactionWrapper(processCacheFile, activity, byActor) | 43 | return retryTransactionWrapper(processCreateCacheFile, activity, byActor) |
44 | } | ||
45 | |||
46 | if (activityType === 'Playlist') { | ||
47 | return retryTransactionWrapper(processCreatePlaylist, activity, byActor) | ||
42 | } | 48 | } |
43 | 49 | ||
44 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | 50 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) |
@@ -63,7 +69,7 @@ async function processCreateVideo (activity: ActivityCreate) { | |||
63 | return video | 69 | return video |
64 | } | 70 | } |
65 | 71 | ||
66 | async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) { | 72 | async function processCreateCacheFile (activity: ActivityCreate, byActor: ActorModel) { |
67 | const cacheFile = activity.object as CacheFileObject | 73 | const cacheFile = activity.object as CacheFileObject |
68 | 74 | ||
69 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) | 75 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) |
@@ -98,3 +104,12 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act | |||
98 | 104 | ||
99 | if (created === true) Notifier.Instance.notifyOnNewComment(comment) | 105 | if (created === true) Notifier.Instance.notifyOnNewComment(comment) |
100 | } | 106 | } |
107 | |||
108 | async function processCreatePlaylist (activity: ActivityCreate, byActor: ActorModel) { | ||
109 | const playlistObject = activity.object as PlaylistObject | ||
110 | const byAccount = byActor.Account | ||
111 | |||
112 | if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) | ||
113 | |||
114 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) | ||
115 | } | ||
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index 155d2ffcc..76f07fd8a 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts | |||
@@ -8,6 +8,7 @@ import { VideoModel } from '../../../models/video/video' | |||
8 | import { VideoChannelModel } from '../../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../../models/video/video-channel' |
9 | import { VideoCommentModel } from '../../../models/video/video-comment' | 9 | import { VideoCommentModel } from '../../../models/video/video-comment' |
10 | import { forwardVideoRelatedActivity } from '../send/utils' | 10 | import { forwardVideoRelatedActivity } from '../send/utils' |
11 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
11 | 12 | ||
12 | async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) { | 13 | async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) { |
13 | const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id | 14 | const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id |
@@ -45,6 +46,15 @@ async function processDeleteActivity (activity: ActivityDelete, byActor: ActorMo | |||
45 | } | 46 | } |
46 | } | 47 | } |
47 | 48 | ||
49 | { | ||
50 | const videoPlaylist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(objectUrl) | ||
51 | if (videoPlaylist) { | ||
52 | if (videoPlaylist.isOwned()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`) | ||
53 | |||
54 | return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist) | ||
55 | } | ||
56 | } | ||
57 | |||
48 | return undefined | 58 | return undefined |
49 | } | 59 | } |
50 | 60 | ||
@@ -70,6 +80,20 @@ async function processDeleteVideo (actor: ActorModel, videoToDelete: VideoModel) | |||
70 | logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) | 80 | logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) |
71 | } | 81 | } |
72 | 82 | ||
83 | async function processDeleteVideoPlaylist (actor: ActorModel, playlistToDelete: VideoPlaylistModel) { | ||
84 | logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid) | ||
85 | |||
86 | await sequelizeTypescript.transaction(async t => { | ||
87 | if (playlistToDelete.OwnerAccount.Actor.id !== actor.id) { | ||
88 | throw new Error('Account ' + actor.url + ' does not own video playlist ' + playlistToDelete.url) | ||
89 | } | ||
90 | |||
91 | await playlistToDelete.destroy({ transaction: t }) | ||
92 | }) | ||
93 | |||
94 | logger.info('Remote video playlist with uuid %s removed.', playlistToDelete.uuid) | ||
95 | } | ||
96 | |||
73 | async function processDeleteAccount (accountToRemove: AccountModel) { | 97 | async function processDeleteAccount (accountToRemove: AccountModel) { |
74 | logger.debug('Removing remote account "%s".', accountToRemove.Actor.uuid) | 98 | logger.debug('Removing remote account "%s".', accountToRemove.Actor.uuid) |
75 | 99 | ||
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index 0cd537187..ed16ba172 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts | |||
@@ -4,9 +4,11 @@ import { logger } from '../../../helpers/logger' | |||
4 | import { sequelizeTypescript } from '../../../initializers' | 4 | import { sequelizeTypescript } from '../../../initializers' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
7 | import { sendAccept } from '../send' | 7 | import { sendAccept, sendReject } from '../send' |
8 | import { Notifier } from '../../notifier' | 8 | import { Notifier } from '../../notifier' |
9 | import { getAPId } from '../../../helpers/activitypub' | 9 | import { getAPId } from '../../../helpers/activitypub' |
10 | import { getServerActor } from '../../../helpers/utils' | ||
11 | import { CONFIG } from '../../../initializers/config' | ||
10 | 12 | ||
11 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { | 13 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { |
12 | const activityObject = getAPId(activity.object) | 14 | const activityObject = getAPId(activity.object) |
@@ -23,12 +25,23 @@ export { | |||
23 | // --------------------------------------------------------------------------- | 25 | // --------------------------------------------------------------------------- |
24 | 26 | ||
25 | async function processFollow (actor: ActorModel, targetActorURL: string) { | 27 | async function processFollow (actor: ActorModel, targetActorURL: string) { |
26 | const { actorFollow, created } = await sequelizeTypescript.transaction(async t => { | 28 | const { actorFollow, created, isFollowingInstance } = await sequelizeTypescript.transaction(async t => { |
27 | const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) | 29 | const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) |
28 | 30 | ||
29 | if (!targetActor) throw new Error('Unknown actor') | 31 | if (!targetActor) throw new Error('Unknown actor') |
30 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') | 32 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') |
31 | 33 | ||
34 | const serverActor = await getServerActor() | ||
35 | const isFollowingInstance = targetActor.id === serverActor.id | ||
36 | |||
37 | if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { | ||
38 | logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) | ||
39 | |||
40 | await sendReject(actor, targetActor) | ||
41 | |||
42 | return { actorFollow: undefined } | ||
43 | } | ||
44 | |||
32 | const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({ | 45 | const [ actorFollow, created ] = await ActorFollowModel.findOrCreate({ |
33 | where: { | 46 | where: { |
34 | actorId: actor.id, | 47 | actorId: actor.id, |
@@ -37,15 +50,12 @@ async function processFollow (actor: ActorModel, targetActorURL: string) { | |||
37 | defaults: { | 50 | defaults: { |
38 | actorId: actor.id, | 51 | actorId: actor.id, |
39 | targetActorId: targetActor.id, | 52 | targetActorId: targetActor.id, |
40 | state: 'accepted' | 53 | state: CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL ? 'pending' : 'accepted' |
41 | }, | 54 | }, |
42 | transaction: t | 55 | transaction: t |
43 | }) | 56 | }) |
44 | 57 | ||
45 | actorFollow.ActorFollower = actor | 58 | if (actorFollow.state !== 'accepted' && CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL === false) { |
46 | actorFollow.ActorFollowing = targetActor | ||
47 | |||
48 | if (actorFollow.state !== 'accepted') { | ||
49 | actorFollow.state = 'accepted' | 59 | actorFollow.state = 'accepted' |
50 | await actorFollow.save({ transaction: t }) | 60 | await actorFollow.save({ transaction: t }) |
51 | } | 61 | } |
@@ -54,12 +64,18 @@ async function processFollow (actor: ActorModel, targetActorURL: string) { | |||
54 | actorFollow.ActorFollowing = targetActor | 64 | actorFollow.ActorFollowing = targetActor |
55 | 65 | ||
56 | // Target sends to actor he accepted the follow request | 66 | // Target sends to actor he accepted the follow request |
57 | await sendAccept(actorFollow) | 67 | if (actorFollow.state === 'accepted') await sendAccept(actorFollow) |
58 | 68 | ||
59 | return { actorFollow, created } | 69 | return { actorFollow, created, isFollowingInstance } |
60 | }) | 70 | }) |
61 | 71 | ||
62 | if (created) Notifier.Instance.notifyOfNewFollow(actorFollow) | 72 | // Rejected |
73 | if (!actorFollow) return | ||
74 | |||
75 | if (created) { | ||
76 | if (isFollowingInstance) Notifier.Instance.notifyOfNewInstanceFollow(actorFollow) | ||
77 | else Notifier.Instance.notifyOfNewUserFollow(actorFollow) | ||
78 | } | ||
63 | 79 | ||
64 | logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) | 80 | logger.info('Actor %s is followed by actor %s.', targetActorURL, actor.url) |
65 | } | 81 | } |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index ed0177a67..2d48848fe 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -108,7 +108,10 @@ async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo | |||
108 | 108 | ||
109 | return sequelizeTypescript.transaction(async t => { | 109 | return sequelizeTypescript.transaction(async t => { |
110 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 110 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) |
111 | if (!cacheFile) throw new Error('Unknown video cache ' + cacheFileObject.id) | 111 | if (!cacheFile) { |
112 | logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) | ||
113 | return | ||
114 | } | ||
112 | 115 | ||
113 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') | 116 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') |
114 | 117 | ||
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index c6b42d846..54a9234bb 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -12,6 +12,8 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-vali | |||
12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | 12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' |
13 | import { createOrUpdateCacheFile } from '../cache-file' | 13 | import { createOrUpdateCacheFile } from '../cache-file' |
14 | import { forwardVideoRelatedActivity } from '../send/utils' | 14 | import { forwardVideoRelatedActivity } from '../send/utils' |
15 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' | ||
16 | import { createOrUpdateVideoPlaylist } from '../playlist' | ||
15 | 17 | ||
16 | async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { | 18 | async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { |
17 | const objectType = activity.object.type | 19 | const objectType = activity.object.type |
@@ -32,6 +34,10 @@ async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorMo | |||
32 | return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity) | 34 | return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity) |
33 | } | 35 | } |
34 | 36 | ||
37 | if (objectType === 'Playlist') { | ||
38 | return retryTransactionWrapper(processUpdatePlaylist, byActor, activity) | ||
39 | } | ||
40 | |||
35 | return undefined | 41 | return undefined |
36 | } | 42 | } |
37 | 43 | ||
@@ -114,9 +120,11 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) | |||
114 | 120 | ||
115 | await actor.save({ transaction: t }) | 121 | await actor.save({ transaction: t }) |
116 | 122 | ||
117 | accountOrChannelInstance.set('name', actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername) | 123 | accountOrChannelInstance.name = actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername |
118 | accountOrChannelInstance.set('description', actorAttributesToUpdate.summary) | 124 | accountOrChannelInstance.description = actorAttributesToUpdate.summary |
119 | accountOrChannelInstance.set('support', actorAttributesToUpdate.support) | 125 | |
126 | if (accountOrChannelInstance instanceof VideoChannelModel) accountOrChannelInstance.support = actorAttributesToUpdate.support | ||
127 | |||
120 | await accountOrChannelInstance.save({ transaction: t }) | 128 | await accountOrChannelInstance.save({ transaction: t }) |
121 | }) | 129 | }) |
122 | 130 | ||
@@ -135,3 +143,12 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) | |||
135 | throw err | 143 | throw err |
136 | } | 144 | } |
137 | } | 145 | } |
146 | |||
147 | async function processUpdatePlaylist (byActor: ActorModel, activity: ActivityUpdate) { | ||
148 | const playlistObject = activity.object as PlaylistObject | ||
149 | const byAccount = byActor.Account | ||
150 | |||
151 | if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) | ||
152 | |||
153 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) | ||
154 | } | ||
diff --git a/server/lib/activitypub/send/index.ts b/server/lib/activitypub/send/index.ts index 79ba6c7fe..028936810 100644 --- a/server/lib/activitypub/send/index.ts +++ b/server/lib/activitypub/send/index.ts | |||
@@ -1,8 +1,10 @@ | |||
1 | export * from './send-accept' | 1 | export * from './send-accept' |
2 | export * from './send-accept' | ||
2 | export * from './send-announce' | 3 | export * from './send-announce' |
3 | export * from './send-create' | 4 | export * from './send-create' |
4 | export * from './send-delete' | 5 | export * from './send-delete' |
5 | export * from './send-follow' | 6 | export * from './send-follow' |
6 | export * from './send-like' | 7 | export * from './send-like' |
8 | export * from './send-reject' | ||
7 | export * from './send-undo' | 9 | export * from './send-undo' |
8 | export * from './send-update' | 10 | export * from './send-update' |
diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts index b6abde13d..388a9ed23 100644 --- a/server/lib/activitypub/send/send-accept.ts +++ b/server/lib/activitypub/send/send-accept.ts | |||
@@ -17,7 +17,7 @@ async function sendAccept (actorFollow: ActorFollowModel) { | |||
17 | 17 | ||
18 | logger.info('Creating job to accept follower %s.', follower.url) | 18 | logger.info('Creating job to accept follower %s.', follower.url) |
19 | 19 | ||
20 | const followUrl = getActorFollowActivityPubUrl(actorFollow) | 20 | const followUrl = getActorFollowActivityPubUrl(follower, me) |
21 | const followData = buildFollowActivity(followUrl, follower, me) | 21 | const followData = buildFollowActivity(followUrl, follower, me) |
22 | 22 | ||
23 | const url = getActorFollowAcceptActivityPubUrl(actorFollow) | 23 | const url = getActorFollowAcceptActivityPubUrl(actorFollow) |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index e3fca0a17..28f18595b 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -3,13 +3,14 @@ import { ActivityAudience, ActivityCreate } from '../../../../shared/models/acti | |||
3 | import { VideoPrivacy } from '../../../../shared/models/videos' | 3 | import { VideoPrivacy } from '../../../../shared/models/videos' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { VideoModel } from '../../../models/video/video' | 5 | import { VideoModel } from '../../../models/video/video' |
6 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
7 | import { VideoCommentModel } from '../../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../../models/video/video-comment' |
8 | import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' | ||
9 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 7 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
10 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' | 8 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' |
11 | import { logger } from '../../../helpers/logger' | 9 | import { logger } from '../../../helpers/logger' |
12 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 10 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
11 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
12 | import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
13 | import { getServerActor } from '../../../helpers/utils' | ||
13 | 14 | ||
14 | async function sendCreateVideo (video: VideoModel, t: Transaction) { | 15 | async function sendCreateVideo (video: VideoModel, t: Transaction) { |
15 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | 16 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined |
@@ -25,34 +26,36 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { | |||
25 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) | 26 | return broadcastToFollowers(createActivity, byActor, [ byActor ], t) |
26 | } | 27 | } |
27 | 28 | ||
28 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { | 29 | async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) { |
29 | if (!video.VideoChannel.Account.Actor.serverId) return // Local | ||
30 | |||
31 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | ||
32 | |||
33 | logger.info('Creating job to send video abuse %s.', url) | ||
34 | |||
35 | // Custom audience, we only send the abuse to the origin instance | ||
36 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | ||
37 | const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) | ||
38 | |||
39 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
40 | } | ||
41 | |||
42 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | ||
43 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | 30 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) |
44 | 31 | ||
45 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | ||
46 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
47 | |||
48 | return sendVideoRelatedCreateActivity({ | 32 | return sendVideoRelatedCreateActivity({ |
49 | byActor, | 33 | byActor, |
50 | video, | 34 | video, |
51 | url: fileRedundancy.url, | 35 | url: fileRedundancy.url, |
52 | object: redundancyObject | 36 | object: fileRedundancy.toActivityPubObject() |
53 | }) | 37 | }) |
54 | } | 38 | } |
55 | 39 | ||
40 | async function sendCreateVideoPlaylist (playlist: VideoPlaylistModel, t: Transaction) { | ||
41 | if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined | ||
42 | |||
43 | logger.info('Creating job to send create video playlist of %s.', playlist.url) | ||
44 | |||
45 | const byActor = playlist.OwnerAccount.Actor | ||
46 | const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) | ||
47 | |||
48 | const object = await playlist.toActivityPubObject(null, t) | ||
49 | const createActivity = buildCreateActivity(playlist.url, byActor, object, audience) | ||
50 | |||
51 | const serverActor = await getServerActor() | ||
52 | const toFollowersOf = [ byActor, serverActor ] | ||
53 | |||
54 | if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor) | ||
55 | |||
56 | return broadcastToFollowers(createActivity, byActor, toFollowersOf, t) | ||
57 | } | ||
58 | |||
56 | async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { | 59 | async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { |
57 | logger.info('Creating job to send comment %s.', comment.url) | 60 | logger.info('Creating job to send comment %s.', comment.url) |
58 | 61 | ||
@@ -91,37 +94,6 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio | |||
91 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) | 94 | return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) |
92 | } | 95 | } |
93 | 96 | ||
94 | async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
95 | logger.info('Creating job to send view of %s.', video.url) | ||
96 | |||
97 | const url = getVideoViewActivityPubUrl(byActor, video) | ||
98 | const viewActivity = buildViewActivity(url, byActor, video) | ||
99 | |||
100 | return sendVideoRelatedCreateActivity({ | ||
101 | // Use the server actor to send the view | ||
102 | byActor, | ||
103 | video, | ||
104 | url, | ||
105 | object: viewActivity, | ||
106 | transaction: t | ||
107 | }) | ||
108 | } | ||
109 | |||
110 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
111 | logger.info('Creating job to dislike %s.', video.url) | ||
112 | |||
113 | const url = getVideoDislikeActivityPubUrl(byActor, video) | ||
114 | const dislikeActivity = buildDislikeActivity(url, byActor, video) | ||
115 | |||
116 | return sendVideoRelatedCreateActivity({ | ||
117 | byActor, | ||
118 | video, | ||
119 | url, | ||
120 | object: dislikeActivity, | ||
121 | transaction: t | ||
122 | }) | ||
123 | } | ||
124 | |||
125 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { | 97 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { |
126 | if (!audience) audience = getAudience(byActor) | 98 | if (!audience) audience = getAudience(byActor) |
127 | 99 | ||
@@ -136,34 +108,13 @@ function buildCreateActivity (url: string, byActor: ActorModel, object: any, aud | |||
136 | ) | 108 | ) |
137 | } | 109 | } |
138 | 110 | ||
139 | function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel) { | ||
140 | return { | ||
141 | id: url, | ||
142 | type: 'Dislike', | ||
143 | actor: byActor.url, | ||
144 | object: video.url | ||
145 | } | ||
146 | } | ||
147 | |||
148 | function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel) { | ||
149 | return { | ||
150 | id: url, | ||
151 | type: 'View', | ||
152 | actor: byActor.url, | ||
153 | object: video.url | ||
154 | } | ||
155 | } | ||
156 | |||
157 | // --------------------------------------------------------------------------- | 111 | // --------------------------------------------------------------------------- |
158 | 112 | ||
159 | export { | 113 | export { |
160 | sendCreateVideo, | 114 | sendCreateVideo, |
161 | sendVideoAbuse, | ||
162 | buildCreateActivity, | 115 | buildCreateActivity, |
163 | sendCreateView, | ||
164 | sendCreateDislike, | ||
165 | buildDislikeActivity, | ||
166 | sendCreateVideoComment, | 116 | sendCreateVideoComment, |
117 | sendCreateVideoPlaylist, | ||
167 | sendCreateCacheFile | 118 | sendCreateCacheFile |
168 | } | 119 | } |
169 | 120 | ||
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index 18969433a..7bf5ca520 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts | |||
@@ -8,6 +8,8 @@ import { getDeleteActivityPubUrl } from '../url' | |||
8 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 8 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
9 | import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' | 9 | import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' |
10 | import { logger } from '../../../helpers/logger' | 10 | import { logger } from '../../../helpers/logger' |
11 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
12 | import { getServerActor } from '../../../helpers/utils' | ||
11 | 13 | ||
12 | async function sendDeleteVideo (video: VideoModel, transaction: Transaction) { | 14 | async function sendDeleteVideo (video: VideoModel, transaction: Transaction) { |
13 | logger.info('Creating job to broadcast delete of video %s.', video.url) | 15 | logger.info('Creating job to broadcast delete of video %s.', video.url) |
@@ -29,7 +31,12 @@ async function sendDeleteActor (byActor: ActorModel, t: Transaction) { | |||
29 | const url = getDeleteActivityPubUrl(byActor.url) | 31 | const url = getDeleteActivityPubUrl(byActor.url) |
30 | const activity = buildDeleteActivity(url, byActor.url, byActor) | 32 | const activity = buildDeleteActivity(url, byActor.url, byActor) |
31 | 33 | ||
32 | const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) | 34 | const actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, t) |
35 | |||
36 | // In case the actor did not have any videos | ||
37 | const serverActor = await getServerActor() | ||
38 | actorsInvolved.push(serverActor) | ||
39 | |||
33 | actorsInvolved.push(byActor) | 40 | actorsInvolved.push(byActor) |
34 | 41 | ||
35 | return broadcastToFollowers(activity, byActor, actorsInvolved, t) | 42 | return broadcastToFollowers(activity, byActor, actorsInvolved, t) |
@@ -64,12 +71,29 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans | |||
64 | return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) | 71 | return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) |
65 | } | 72 | } |
66 | 73 | ||
74 | async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) { | ||
75 | logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url) | ||
76 | |||
77 | const byActor = videoPlaylist.OwnerAccount.Actor | ||
78 | |||
79 | const url = getDeleteActivityPubUrl(videoPlaylist.url) | ||
80 | const activity = buildDeleteActivity(url, videoPlaylist.url, byActor) | ||
81 | |||
82 | const serverActor = await getServerActor() | ||
83 | const toFollowersOf = [ byActor, serverActor ] | ||
84 | |||
85 | if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) | ||
86 | |||
87 | return broadcastToFollowers(activity, byActor, toFollowersOf, t) | ||
88 | } | ||
89 | |||
67 | // --------------------------------------------------------------------------- | 90 | // --------------------------------------------------------------------------- |
68 | 91 | ||
69 | export { | 92 | export { |
70 | sendDeleteVideo, | 93 | sendDeleteVideo, |
71 | sendDeleteActor, | 94 | sendDeleteActor, |
72 | sendDeleteVideoComment | 95 | sendDeleteVideoComment, |
96 | sendDeleteVideoPlaylist | ||
73 | } | 97 | } |
74 | 98 | ||
75 | // --------------------------------------------------------------------------- | 99 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts new file mode 100644 index 000000000..a88436f2c --- /dev/null +++ b/server/lib/activitypub/send/send-dislike.ts | |||
@@ -0,0 +1,41 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { ActorModel } from '../../../models/activitypub/actor' | ||
3 | import { VideoModel } from '../../../models/video/video' | ||
4 | import { getVideoDislikeActivityPubUrl } from '../url' | ||
5 | import { logger } from '../../../helpers/logger' | ||
6 | import { ActivityAudience, ActivityDislike } from '../../../../shared/models/activitypub' | ||
7 | import { sendVideoRelatedActivity } from './utils' | ||
8 | import { audiencify, getAudience } from '../audience' | ||
9 | |||
10 | async function sendDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
11 | logger.info('Creating job to dislike %s.', video.url) | ||
12 | |||
13 | const activityBuilder = (audience: ActivityAudience) => { | ||
14 | const url = getVideoDislikeActivityPubUrl(byActor, video) | ||
15 | |||
16 | return buildDislikeActivity(url, byActor, video, audience) | ||
17 | } | ||
18 | |||
19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) | ||
20 | } | ||
21 | |||
22 | function buildDislikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityDislike { | ||
23 | if (!audience) audience = getAudience(byActor) | ||
24 | |||
25 | return audiencify( | ||
26 | { | ||
27 | id: url, | ||
28 | type: 'Dislike' as 'Dislike', | ||
29 | actor: byActor.url, | ||
30 | object: video.url | ||
31 | }, | ||
32 | audience | ||
33 | ) | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | sendDislike, | ||
40 | buildDislikeActivity | ||
41 | } | ||
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts new file mode 100644 index 000000000..96a7311b9 --- /dev/null +++ b/server/lib/activitypub/send/send-flag.ts | |||
@@ -0,0 +1,39 @@ | |||
1 | import { ActorModel } from '../../../models/activitypub/actor' | ||
2 | import { VideoModel } from '../../../models/video/video' | ||
3 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | ||
4 | import { getVideoAbuseActivityPubUrl } from '../url' | ||
5 | import { unicastTo } from './utils' | ||
6 | import { logger } from '../../../helpers/logger' | ||
7 | import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub' | ||
8 | import { audiencify, getAudience } from '../audience' | ||
9 | |||
10 | async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { | ||
11 | if (!video.VideoChannel.Account.Actor.serverId) return // Local user | ||
12 | |||
13 | const url = getVideoAbuseActivityPubUrl(videoAbuse) | ||
14 | |||
15 | logger.info('Creating job to send video abuse %s.', url) | ||
16 | |||
17 | // Custom audience, we only send the abuse to the origin instance | ||
18 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | ||
19 | const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience) | ||
20 | |||
21 | return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
22 | } | ||
23 | |||
24 | function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag { | ||
25 | if (!audience) audience = getAudience(byActor) | ||
26 | |||
27 | const activity = Object.assign( | ||
28 | { id: url, actor: byActor.url }, | ||
29 | videoAbuse.toActivityPubObject() | ||
30 | ) | ||
31 | |||
32 | return audiencify(activity, audience) | ||
33 | } | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | export { | ||
38 | sendVideoAbuse | ||
39 | } | ||
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts index 170b46b48..2c3d02014 100644 --- a/server/lib/activitypub/send/send-follow.ts +++ b/server/lib/activitypub/send/send-follow.ts | |||
@@ -14,7 +14,7 @@ function sendFollow (actorFollow: ActorFollowModel) { | |||
14 | 14 | ||
15 | logger.info('Creating job to send follow request to %s.', following.url) | 15 | logger.info('Creating job to send follow request to %s.', following.url) |
16 | 16 | ||
17 | const url = getActorFollowActivityPubUrl(actorFollow) | 17 | const url = getActorFollowActivityPubUrl(me, following) |
18 | const data = buildFollowActivity(url, me, following) | 18 | const data = buildFollowActivity(url, me, following) |
19 | 19 | ||
20 | return unicastTo(data, me, following.inboxUrl) | 20 | return unicastTo(data, me, following.inboxUrl) |
diff --git a/server/lib/activitypub/send/send-reject.ts b/server/lib/activitypub/send/send-reject.ts new file mode 100644 index 000000000..bac7ff556 --- /dev/null +++ b/server/lib/activitypub/send/send-reject.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { ActivityFollow, ActivityReject } from '../../../../shared/models/activitypub' | ||
2 | import { ActorModel } from '../../../models/activitypub/actor' | ||
3 | import { getActorFollowActivityPubUrl, getActorFollowRejectActivityPubUrl } from '../url' | ||
4 | import { unicastTo } from './utils' | ||
5 | import { buildFollowActivity } from './send-follow' | ||
6 | import { logger } from '../../../helpers/logger' | ||
7 | |||
8 | async function sendReject (follower: ActorModel, following: ActorModel) { | ||
9 | if (!follower.serverId) { // This should never happen | ||
10 | logger.warn('Do not sending reject to local follower.') | ||
11 | return | ||
12 | } | ||
13 | |||
14 | logger.info('Creating job to reject follower %s.', follower.url) | ||
15 | |||
16 | const followUrl = getActorFollowActivityPubUrl(follower, following) | ||
17 | const followData = buildFollowActivity(followUrl, follower, following) | ||
18 | |||
19 | const url = getActorFollowRejectActivityPubUrl(follower, following) | ||
20 | const data = buildRejectActivity(url, following, followData) | ||
21 | |||
22 | return unicastTo(data, following, follower.inboxUrl) | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
26 | |||
27 | export { | ||
28 | sendReject | ||
29 | } | ||
30 | |||
31 | // --------------------------------------------------------------------------- | ||
32 | |||
33 | function buildRejectActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityReject { | ||
34 | return { | ||
35 | type: 'Reject', | ||
36 | id: url, | ||
37 | actor: byActor.url, | ||
38 | object: followActivityData | ||
39 | } | ||
40 | } | ||
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index bf1b6e117..8727a121e 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize' | |||
2 | import { | 2 | import { |
3 | ActivityAnnounce, | 3 | ActivityAnnounce, |
4 | ActivityAudience, | 4 | ActivityAudience, |
5 | ActivityCreate, | 5 | ActivityCreate, ActivityDislike, |
6 | ActivityFollow, | 6 | ActivityFollow, |
7 | ActivityLike, | 7 | ActivityLike, |
8 | ActivityUndo | 8 | ActivityUndo |
@@ -13,13 +13,14 @@ import { VideoModel } from '../../../models/video/video' | |||
13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' | 13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' |
14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
15 | import { audiencify, getAudience } from '../audience' | 15 | import { audiencify, getAudience } from '../audience' |
16 | import { buildCreateActivity, buildDislikeActivity } from './send-create' | 16 | import { buildCreateActivity } from './send-create' |
17 | import { buildFollowActivity } from './send-follow' | 17 | import { buildFollowActivity } from './send-follow' |
18 | import { buildLikeActivity } from './send-like' | 18 | import { buildLikeActivity } from './send-like' |
19 | import { VideoShareModel } from '../../../models/video/video-share' | 19 | import { VideoShareModel } from '../../../models/video/video-share' |
20 | import { buildAnnounceWithVideoAudience } from './send-announce' | 20 | import { buildAnnounceWithVideoAudience } from './send-announce' |
21 | import { logger } from '../../../helpers/logger' | 21 | import { logger } from '../../../helpers/logger' |
22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 22 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
23 | import { buildDislikeActivity } from './send-dislike' | ||
23 | 24 | ||
24 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | 25 | async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { |
25 | const me = actorFollow.ActorFollower | 26 | const me = actorFollow.ActorFollower |
@@ -30,7 +31,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | |||
30 | 31 | ||
31 | logger.info('Creating job to send an unfollow request to %s.', following.url) | 32 | logger.info('Creating job to send an unfollow request to %s.', following.url) |
32 | 33 | ||
33 | const followUrl = getActorFollowActivityPubUrl(actorFollow) | 34 | const followUrl = getActorFollowActivityPubUrl(me, following) |
34 | const undoUrl = getUndoActivityPubUrl(followUrl) | 35 | const undoUrl = getUndoActivityPubUrl(followUrl) |
35 | 36 | ||
36 | const followActivity = buildFollowActivity(followUrl, me, following) | 37 | const followActivity = buildFollowActivity(followUrl, me, following) |
@@ -65,15 +66,15 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans | |||
65 | 66 | ||
66 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) | 67 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) |
67 | const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) | 68 | const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) |
68 | const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) | ||
69 | 69 | ||
70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) | 70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) |
71 | } | 71 | } |
72 | 72 | ||
73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | 73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { |
74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | 74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) |
75 | 75 | ||
76 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 76 | const videoId = redundancyModel.getVideo().id |
77 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
77 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | 78 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) |
78 | 79 | ||
79 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) | 80 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) |
@@ -94,7 +95,7 @@ export { | |||
94 | function undoActivityData ( | 95 | function undoActivityData ( |
95 | url: string, | 96 | url: string, |
96 | byActor: ActorModel, | 97 | byActor: ActorModel, |
97 | object: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | 98 | object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, |
98 | audience?: ActivityAudience | 99 | audience?: ActivityAudience |
99 | ): ActivityUndo { | 100 | ): ActivityUndo { |
100 | if (!audience) audience = getAudience(byActor) | 101 | if (!audience) audience = getAudience(byActor) |
@@ -114,7 +115,7 @@ async function sendUndoVideoRelatedActivity (options: { | |||
114 | byActor: ActorModel, | 115 | byActor: ActorModel, |
115 | video: VideoModel, | 116 | video: VideoModel, |
116 | url: string, | 117 | url: string, |
117 | activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | 118 | activity: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, |
118 | transaction: Transaction | 119 | transaction: Transaction |
119 | }) { | 120 | }) { |
120 | const activityBuilder = (audience: ActivityAudience) => { | 121 | const activityBuilder = (audience: ActivityAudience) => { |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index a68f03edf..7411c08d5 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -12,8 +12,13 @@ import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience' | |||
12 | import { logger } from '../../../helpers/logger' | 12 | import { logger } from '../../../helpers/logger' |
13 | import { VideoCaptionModel } from '../../../models/video/video-caption' | 13 | import { VideoCaptionModel } from '../../../models/video/video-caption' |
14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
15 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
16 | import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
17 | import { getServerActor } from '../../../helpers/utils' | ||
15 | 18 | ||
16 | async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { | 19 | async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { |
20 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | ||
21 | |||
17 | logger.info('Creating job to update video %s.', video.url) | 22 | logger.info('Creating job to update video %s.', video.url) |
18 | 23 | ||
19 | const byActor = overrodeByActor ? overrodeByActor : video.VideoChannel.Account.Actor | 24 | const byActor = overrodeByActor ? overrodeByActor : video.VideoChannel.Account.Actor |
@@ -47,7 +52,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
47 | let actorsInvolved: ActorModel[] | 52 | let actorsInvolved: ActorModel[] |
48 | if (accountOrChannel instanceof AccountModel) { | 53 | if (accountOrChannel instanceof AccountModel) { |
49 | // Actors that shared my videos are involved too | 54 | // Actors that shared my videos are involved too |
50 | actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) | 55 | actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, t) |
51 | } else { | 56 | } else { |
52 | // Actors that shared videos of my channel are involved too | 57 | // Actors that shared videos of my channel are involved too |
53 | actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, t) | 58 | actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, t) |
@@ -61,7 +66,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | 66 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { |
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | 67 | logger.info('Creating job to update cache file %s.', redundancyModel.url) |
63 | 68 | ||
64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 69 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id) |
65 | 70 | ||
66 | const activityBuilder = (audience: ActivityAudience) => { | 71 | const activityBuilder = (audience: ActivityAudience) => { |
67 | const redundancyObject = redundancyModel.toActivityPubObject() | 72 | const redundancyObject = redundancyModel.toActivityPubObject() |
@@ -73,12 +78,35 @@ async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoR | |||
73 | return sendVideoRelatedActivity(activityBuilder, { byActor, video }) | 78 | return sendVideoRelatedActivity(activityBuilder, { byActor, video }) |
74 | } | 79 | } |
75 | 80 | ||
81 | async function sendUpdateVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) { | ||
82 | if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined | ||
83 | |||
84 | const byActor = videoPlaylist.OwnerAccount.Actor | ||
85 | |||
86 | logger.info('Creating job to update video playlist %s.', videoPlaylist.url) | ||
87 | |||
88 | const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString()) | ||
89 | |||
90 | const object = await videoPlaylist.toActivityPubObject(null, t) | ||
91 | const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC) | ||
92 | |||
93 | const updateActivity = buildUpdateActivity(url, byActor, object, audience) | ||
94 | |||
95 | const serverActor = await getServerActor() | ||
96 | const toFollowersOf = [ byActor, serverActor ] | ||
97 | |||
98 | if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) | ||
99 | |||
100 | return broadcastToFollowers(updateActivity, byActor, toFollowersOf, t) | ||
101 | } | ||
102 | |||
76 | // --------------------------------------------------------------------------- | 103 | // --------------------------------------------------------------------------- |
77 | 104 | ||
78 | export { | 105 | export { |
79 | sendUpdateActor, | 106 | sendUpdateActor, |
80 | sendUpdateVideo, | 107 | sendUpdateVideo, |
81 | sendUpdateCacheFile | 108 | sendUpdateCacheFile, |
109 | sendUpdateVideoPlaylist | ||
82 | } | 110 | } |
83 | 111 | ||
84 | // --------------------------------------------------------------------------- | 112 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts new file mode 100644 index 000000000..8ad126be0 --- /dev/null +++ b/server/lib/activitypub/send/send-view.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' | ||
3 | import { ActorModel } from '../../../models/activitypub/actor' | ||
4 | import { VideoModel } from '../../../models/video/video' | ||
5 | import { getVideoLikeActivityPubUrl } from '../url' | ||
6 | import { sendVideoRelatedActivity } from './utils' | ||
7 | import { audiencify, getAudience } from '../audience' | ||
8 | import { logger } from '../../../helpers/logger' | ||
9 | |||
10 | async function sendView (byActor: ActorModel, video: VideoModel, t: Transaction) { | ||
11 | logger.info('Creating job to send view of %s.', video.url) | ||
12 | |||
13 | const activityBuilder = (audience: ActivityAudience) => { | ||
14 | const url = getVideoLikeActivityPubUrl(byActor, video) | ||
15 | |||
16 | return buildViewActivity(url, byActor, video, audience) | ||
17 | } | ||
18 | |||
19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) | ||
20 | } | ||
21 | |||
22 | function buildViewActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityView { | ||
23 | if (!audience) audience = getAudience(byActor) | ||
24 | |||
25 | return audiencify( | ||
26 | { | ||
27 | id: url, | ||
28 | type: 'View' as 'View', | ||
29 | actor: byActor.url, | ||
30 | object: video.url | ||
31 | }, | ||
32 | audience | ||
33 | ) | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | sendView | ||
40 | } | ||
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 1767df0ae..7f38402b6 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts | |||
@@ -10,7 +10,7 @@ import * as Bluebird from 'bluebird' | |||
10 | import { doRequest } from '../../helpers/requests' | 10 | import { doRequest } from '../../helpers/requests' |
11 | import { getOrCreateActorAndServerAndModel } from './actor' | 11 | import { getOrCreateActorAndServerAndModel } from './actor' |
12 | import { logger } from '../../helpers/logger' | 12 | import { logger } from '../../helpers/logger' |
13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
14 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 14 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
15 | 15 | ||
16 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { | 16 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { |
@@ -54,12 +54,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { | |||
54 | url: shareUrl | 54 | url: shareUrl |
55 | } | 55 | } |
56 | 56 | ||
57 | await VideoShareModel.findOrCreate({ | 57 | await VideoShareModel.upsert(entry) |
58 | where: { | ||
59 | url: shareUrl | ||
60 | }, | ||
61 | defaults: entry | ||
62 | }) | ||
63 | } catch (err) { | 58 | } catch (err) { |
64 | logger.warn('Cannot add share %s.', shareUrl, { err }) | 59 | logger.warn('Cannot add share %s.', shareUrl, { err }) |
65 | } | 60 | } |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 38f15448c..bcb7a4ee2 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -1,35 +1,49 @@ | |||
1 | import { CONFIG } from '../../initializers' | 1 | import { WEBSERVER } from '../../initializers/constants' |
2 | import { ActorModel } from '../../models/activitypub/actor' | 2 | import { ActorModel } from '../../models/activitypub/actor' |
3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' |
4 | import { VideoModel } from '../../models/video/video' | 4 | import { VideoModel } from '../../models/video/video' |
5 | import { VideoAbuseModel } from '../../models/video/video-abuse' | 5 | import { VideoAbuseModel } from '../../models/video/video-abuse' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoFileModel } from '../../models/video/video-file' | 7 | import { VideoFileModel } from '../../models/video/video-file' |
8 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
9 | import { VideoPlaylistModel } from '../../models/video/video-playlist' | ||
8 | 10 | ||
9 | function getVideoActivityPubUrl (video: VideoModel) { | 11 | function getVideoActivityPubUrl (video: VideoModel) { |
10 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | 12 | return WEBSERVER.URL + '/videos/watch/' + video.uuid |
13 | } | ||
14 | |||
15 | function getVideoPlaylistActivityPubUrl (videoPlaylist: VideoPlaylistModel) { | ||
16 | return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid | ||
17 | } | ||
18 | |||
19 | function getVideoPlaylistElementActivityPubUrl (videoPlaylist: VideoPlaylistModel, video: VideoModel) { | ||
20 | return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/' + video.uuid | ||
11 | } | 21 | } |
12 | 22 | ||
13 | function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { | 23 | function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { |
14 | const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' | 24 | const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' |
15 | 25 | ||
16 | return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` | 26 | return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` |
27 | } | ||
28 | |||
29 | function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
30 | return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}` | ||
17 | } | 31 | } |
18 | 32 | ||
19 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { | 33 | function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { |
20 | return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id | 34 | return WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id |
21 | } | 35 | } |
22 | 36 | ||
23 | function getVideoChannelActivityPubUrl (videoChannelName: string) { | 37 | function getVideoChannelActivityPubUrl (videoChannelName: string) { |
24 | return CONFIG.WEBSERVER.URL + '/video-channels/' + videoChannelName | 38 | return WEBSERVER.URL + '/video-channels/' + videoChannelName |
25 | } | 39 | } |
26 | 40 | ||
27 | function getAccountActivityPubUrl (accountName: string) { | 41 | function getAccountActivityPubUrl (accountName: string) { |
28 | return CONFIG.WEBSERVER.URL + '/accounts/' + accountName | 42 | return WEBSERVER.URL + '/accounts/' + accountName |
29 | } | 43 | } |
30 | 44 | ||
31 | function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) { | 45 | function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseModel) { |
32 | return CONFIG.WEBSERVER.URL + '/admin/video-abuses/' + videoAbuse.id | 46 | return WEBSERVER.URL + '/admin/video-abuses/' + videoAbuse.id |
33 | } | 47 | } |
34 | 48 | ||
35 | function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { | 49 | function getVideoViewActivityPubUrl (byActor: ActorModel, video: VideoModel) { |
@@ -60,11 +74,8 @@ function getVideoDislikesActivityPubUrl (video: VideoModel) { | |||
60 | return video.url + '/dislikes' | 74 | return video.url + '/dislikes' |
61 | } | 75 | } |
62 | 76 | ||
63 | function getActorFollowActivityPubUrl (actorFollow: ActorFollowModel) { | 77 | function getActorFollowActivityPubUrl (follower: ActorModel, following: ActorModel) { |
64 | const me = actorFollow.ActorFollower | 78 | return follower.url + '/follows/' + following.id |
65 | const following = actorFollow.ActorFollowing | ||
66 | |||
67 | return me.url + '/follows/' + following.id | ||
68 | } | 79 | } |
69 | 80 | ||
70 | function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) { | 81 | function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) { |
@@ -74,6 +85,10 @@ function getActorFollowAcceptActivityPubUrl (actorFollow: ActorFollowModel) { | |||
74 | return follower.url + '/accepts/follows/' + me.id | 85 | return follower.url + '/accepts/follows/' + me.id |
75 | } | 86 | } |
76 | 87 | ||
88 | function getActorFollowRejectActivityPubUrl (follower: ActorModel, following: ActorModel) { | ||
89 | return follower.url + '/rejects/follows/' + following.id | ||
90 | } | ||
91 | |||
77 | function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) { | 92 | function getVideoAnnounceActivityPubUrl (byActor: ActorModel, video: VideoModel) { |
78 | return video.url + '/announces/' + byActor.id | 93 | return video.url + '/announces/' + byActor.id |
79 | } | 94 | } |
@@ -92,6 +107,9 @@ function getUndoActivityPubUrl (originalUrl: string) { | |||
92 | 107 | ||
93 | export { | 108 | export { |
94 | getVideoActivityPubUrl, | 109 | getVideoActivityPubUrl, |
110 | getVideoPlaylistElementActivityPubUrl, | ||
111 | getVideoPlaylistActivityPubUrl, | ||
112 | getVideoCacheStreamingPlaylistActivityPubUrl, | ||
95 | getVideoChannelActivityPubUrl, | 113 | getVideoChannelActivityPubUrl, |
96 | getAccountActivityPubUrl, | 114 | getAccountActivityPubUrl, |
97 | getVideoAbuseActivityPubUrl, | 115 | getVideoAbuseActivityPubUrl, |
@@ -103,6 +121,7 @@ export { | |||
103 | getVideoViewActivityPubUrl, | 121 | getVideoViewActivityPubUrl, |
104 | getVideoLikeActivityPubUrl, | 122 | getVideoLikeActivityPubUrl, |
105 | getVideoDislikeActivityPubUrl, | 123 | getVideoDislikeActivityPubUrl, |
124 | getActorFollowRejectActivityPubUrl, | ||
106 | getVideoCommentActivityPubUrl, | 125 | getVideoCommentActivityPubUrl, |
107 | getDeleteActivityPubUrl, | 126 | getDeleteActivityPubUrl, |
108 | getVideoSharesActivityPubUrl, | 127 | getVideoSharesActivityPubUrl, |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index e87301fe7..18f44d50e 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -2,7 +2,7 @@ import { VideoCommentObject } from '../../../shared/models/activitypub/objects/v | |||
2 | import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' | 2 | import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { doRequest } from '../../helpers/requests' | 4 | import { doRequest } from '../../helpers/requests' |
5 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 5 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
6 | import { ActorModel } from '../../models/activitypub/actor' | 6 | import { ActorModel } from '../../models/activitypub/actor' |
7 | import { VideoModel } from '../../models/video/video' | 7 | import { VideoModel } from '../../models/video/video' |
8 | import { VideoCommentModel } from '../../models/video/video-comment' | 8 | import { VideoCommentModel } from '../../models/video/video-comment' |
@@ -34,8 +34,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto | |||
34 | accountId: actor.Account.id, | 34 | accountId: actor.Account.id, |
35 | inReplyToCommentId, | 35 | inReplyToCommentId, |
36 | originCommentId, | 36 | originCommentId, |
37 | createdAt: new Date(comment.published), | 37 | createdAt: new Date(comment.published) |
38 | updatedAt: new Date(comment.updated) | ||
39 | } | 38 | } |
40 | } | 39 | } |
41 | 40 | ||
@@ -74,12 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { | |||
74 | const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) | 73 | const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) |
75 | if (!entry) return { created: false } | 74 | if (!entry) return { created: false } |
76 | 75 | ||
77 | const [ comment, created ] = await VideoCommentModel.findOrCreate({ | 76 | const [ comment, created ] = await VideoCommentModel.upsert<VideoCommentModel>(entry, { returning: true }) |
78 | where: { | ||
79 | url: body.id | ||
80 | }, | ||
81 | defaults: entry | ||
82 | }) | ||
83 | comment.Account = actor.Account | 77 | comment.Account = actor.Account |
84 | comment.Video = videoInstance | 78 | comment.Video = videoInstance |
85 | 79 | ||
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 45a2b22ea..cda5b2981 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts | |||
@@ -1,17 +1,18 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { AccountModel } from '../../models/account/account' | 2 | import { AccountModel } from '../../models/account/account' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' | 4 | import { sendLike, sendUndoDislike, sendUndoLike } from './send' |
5 | import { VideoRateType } from '../../../shared/models/videos' | 5 | import { VideoRateType } from '../../../shared/models/videos' |
6 | import * as Bluebird from 'bluebird' | 6 | import * as Bluebird from 'bluebird' |
7 | import { getOrCreateActorAndServerAndModel } from './actor' | 7 | import { getOrCreateActorAndServerAndModel } from './actor' |
8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | 8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' |
9 | import { logger } from '../../helpers/logger' | 9 | import { logger } from '../../helpers/logger' |
10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | 10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
11 | import { doRequest } from '../../helpers/requests' | 11 | import { doRequest } from '../../helpers/requests' |
12 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 12 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
13 | import { ActorModel } from '../../models/activitypub/actor' | 13 | import { ActorModel } from '../../models/activitypub/actor' |
14 | import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' | 14 | import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url' |
15 | import { sendDislike } from './send/send-dislike' | ||
15 | 16 | ||
16 | async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { | 17 | async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRateType) { |
17 | let rateCounts = 0 | 18 | let rateCounts = 0 |
@@ -37,19 +38,14 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa | |||
37 | 38 | ||
38 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 39 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) |
39 | 40 | ||
40 | const [ , created ] = await AccountVideoRateModel | 41 | const entry = { |
41 | .findOrCreate({ | 42 | videoId: video.id, |
42 | where: { | 43 | accountId: actor.Account.id, |
43 | videoId: video.id, | 44 | type: rate, |
44 | accountId: actor.Account.id | 45 | url: body.id |
45 | }, | 46 | } |
46 | defaults: { | 47 | |
47 | videoId: video.id, | 48 | const created = await AccountVideoRateModel.upsert(entry) |
48 | accountId: actor.Account.id, | ||
49 | type: rate, | ||
50 | url: body.id | ||
51 | } | ||
52 | }) | ||
53 | 49 | ||
54 | if (created) rateCounts += 1 | 50 | if (created) rateCounts += 1 |
55 | } catch (err) { | 51 | } catch (err) { |
@@ -60,7 +56,10 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa | |||
60 | logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) | 56 | logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) |
61 | 57 | ||
62 | // This is "likes" and "dislikes" | 58 | // This is "likes" and "dislikes" |
63 | if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) | 59 | if (rateCounts !== 0) { |
60 | const field = rate === 'like' ? 'likes' : 'dislikes' | ||
61 | await video.increment(field, { by: rateCounts }) | ||
62 | } | ||
64 | 63 | ||
65 | return | 64 | return |
66 | } | 65 | } |
@@ -82,7 +81,7 @@ async function sendVideoRateChange (account: AccountModel, | |||
82 | // Like | 81 | // Like |
83 | if (likes > 0) await sendLike(actor, video, t) | 82 | if (likes > 0) await sendLike(actor, video, t) |
84 | // Dislike | 83 | // Dislike |
85 | if (dislikes > 0) await sendCreateDislike(actor, video, t) | 84 | if (dislikes > 0) await sendDislike(actor, video, t) |
86 | } | 85 | } |
87 | 86 | ||
88 | function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { | 87 | function getRateUrl (rateType: VideoRateType, actor: ActorModel, video: VideoModel) { |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index e1e523499..4f26cb6be 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -2,15 +2,28 @@ import * as Bluebird from 'bluebird' | |||
2 | import * as sequelize from 'sequelize' | 2 | import * as sequelize from 'sequelize' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as request from 'request' | 4 | import * as request from 'request' |
5 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' | 5 | import { |
6 | ActivityPlaylistSegmentHashesObject, | ||
7 | ActivityPlaylistUrlObject, | ||
8 | ActivityUrlObject, | ||
9 | ActivityVideoUrlObject, | ||
10 | VideoState | ||
11 | } from '../../../shared/index' | ||
6 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 12 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | 13 | import { VideoPrivacy } from '../../../shared/models/videos' |
8 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 14 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
9 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | 15 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' |
10 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' | 16 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' |
11 | import { logger } from '../../helpers/logger' | 17 | import { logger } from '../../helpers/logger' |
12 | import { doRequest, downloadImage } from '../../helpers/requests' | 18 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' |
13 | import { ACTIVITY_PUB, CONFIG, MIMETYPES, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers' | 19 | import { |
20 | ACTIVITY_PUB, | ||
21 | MIMETYPES, | ||
22 | P2P_MEDIA_LOADER_PEER_VERSION, | ||
23 | PREVIEWS_SIZE, | ||
24 | REMOTE_SCHEME, | ||
25 | STATIC_PATHS | ||
26 | } from '../../initializers/constants' | ||
14 | import { ActorModel } from '../../models/activitypub/actor' | 27 | import { ActorModel } from '../../models/activitypub/actor' |
15 | import { TagModel } from '../../models/video/tag' | 28 | import { TagModel } from '../../models/video/tag' |
16 | import { VideoModel } from '../../models/video/video' | 29 | import { VideoModel } from '../../models/video/video' |
@@ -30,9 +43,20 @@ import { AccountModel } from '../../models/account/account' | |||
30 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | 43 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' |
31 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' | 44 | import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' |
32 | import { Notifier } from '../notifier' | 45 | import { Notifier } from '../notifier' |
46 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
47 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
48 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
49 | import { VideoShareModel } from '../../models/video/video-share' | ||
50 | import { VideoCommentModel } from '../../models/video/video-comment' | ||
51 | import { sequelizeTypescript } from '../../initializers/database' | ||
52 | import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail' | ||
53 | import { ThumbnailModel } from '../../models/video/thumbnail' | ||
54 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' | ||
55 | import { join } from 'path' | ||
56 | import { FilteredModelAttributes } from '../../typings/sequelize' | ||
33 | 57 | ||
34 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 58 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
35 | // If the video is not private and published, we federate it | 59 | // If the video is not private and is published, we federate it |
36 | if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) { | 60 | if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) { |
37 | // Fetch more attributes that we will need to serialize in AP object | 61 | // Fetch more attributes that we will need to serialize in AP object |
38 | if (isArray(video.VideoCaptions) === false) { | 62 | if (isArray(video.VideoCaptions) === false) { |
@@ -84,19 +108,17 @@ async function fetchRemoteVideoDescription (video: VideoModel) { | |||
84 | return body.description ? body.description : '' | 108 | return body.description ? body.description : '' |
85 | } | 109 | } |
86 | 110 | ||
87 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | 111 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, destPath: string) { |
88 | const host = video.VideoChannel.Account.Actor.Server.host | 112 | const url = buildRemoteBaseUrl(video, path) |
89 | 113 | ||
90 | // We need to provide a callback, if no we could have an uncaught exception | 114 | // We need to provide a callback, if no we could have an uncaught exception |
91 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | 115 | return doRequestAndSaveToFile({ uri: url }, destPath) |
92 | if (err) reject(err) | ||
93 | }) | ||
94 | } | 116 | } |
95 | 117 | ||
96 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { | 118 | function buildRemoteBaseUrl (video: VideoModel, path: string) { |
97 | const thumbnailName = video.getThumbnailName() | 119 | const host = video.VideoChannel.Account.Actor.Server.host |
98 | 120 | ||
99 | return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) | 121 | return REMOTE_SCHEME.HTTP + '://' + host + path |
100 | } | 122 | } |
101 | 123 | ||
102 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { | 124 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { |
@@ -124,31 +146,43 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid | |||
124 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] | 146 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] |
125 | 147 | ||
126 | if (syncParam.likes === true) { | 148 | if (syncParam.likes === true) { |
127 | await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like')) | 149 | const handler = items => createRates(items, video, 'like') |
150 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) | ||
151 | |||
152 | await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner) | ||
128 | .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) | 153 | .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) |
129 | } else { | 154 | } else { |
130 | jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) | 155 | jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) |
131 | } | 156 | } |
132 | 157 | ||
133 | if (syncParam.dislikes === true) { | 158 | if (syncParam.dislikes === true) { |
134 | await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike')) | 159 | const handler = items => createRates(items, video, 'dislike') |
160 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) | ||
161 | |||
162 | await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner) | ||
135 | .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) | 163 | .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) |
136 | } else { | 164 | } else { |
137 | jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) | 165 | jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) |
138 | } | 166 | } |
139 | 167 | ||
140 | if (syncParam.shares === true) { | 168 | if (syncParam.shares === true) { |
141 | await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video)) | 169 | const handler = items => addVideoShares(items, video) |
170 | const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) | ||
171 | |||
172 | await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner) | ||
142 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) | 173 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) |
143 | } else { | 174 | } else { |
144 | jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) | 175 | jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) |
145 | } | 176 | } |
146 | 177 | ||
147 | if (syncParam.comments === true) { | 178 | if (syncParam.comments === true) { |
148 | await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video)) | 179 | const handler = items => addVideoComments(items, video) |
180 | const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | ||
181 | |||
182 | await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner) | ||
149 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) | 183 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) |
150 | } else { | 184 | } else { |
151 | jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) | 185 | jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) |
152 | } | 186 | } |
153 | 187 | ||
154 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) | 188 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) |
@@ -170,8 +204,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { | |||
170 | 204 | ||
171 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | 205 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) |
172 | if (videoFromDatabase) { | 206 | if (videoFromDatabase) { |
173 | 207 | if (videoFromDatabase.isOutdated() && allowRefresh === true) { | |
174 | if (allowRefresh === true) { | ||
175 | const refreshOptions = { | 208 | const refreshOptions = { |
176 | video: videoFromDatabase, | 209 | video: videoFromDatabase, |
177 | fetchedType: fetchType, | 210 | fetchedType: fetchType, |
@@ -210,6 +243,14 @@ async function updateVideoFromAP (options: { | |||
210 | const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED | 243 | const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED |
211 | 244 | ||
212 | try { | 245 | try { |
246 | let thumbnailModel: ThumbnailModel | ||
247 | |||
248 | try { | ||
249 | thumbnailModel = await createVideoMiniatureFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.MINIATURE) | ||
250 | } catch (err) { | ||
251 | logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }) | ||
252 | } | ||
253 | |||
213 | await sequelizeTypescript.transaction(async t => { | 254 | await sequelizeTypescript.transaction(async t => { |
214 | const sequelizeOptions = { transaction: t } | 255 | const sequelizeOptions = { transaction: t } |
215 | 256 | ||
@@ -233,17 +274,26 @@ async function updateVideoFromAP (options: { | |||
233 | options.video.set('support', videoData.support) | 274 | options.video.set('support', videoData.support) |
234 | options.video.set('nsfw', videoData.nsfw) | 275 | options.video.set('nsfw', videoData.nsfw) |
235 | options.video.set('commentsEnabled', videoData.commentsEnabled) | 276 | options.video.set('commentsEnabled', videoData.commentsEnabled) |
277 | options.video.set('downloadEnabled', videoData.downloadEnabled) | ||
236 | options.video.set('waitTranscoding', videoData.waitTranscoding) | 278 | options.video.set('waitTranscoding', videoData.waitTranscoding) |
237 | options.video.set('state', videoData.state) | 279 | options.video.set('state', videoData.state) |
238 | options.video.set('duration', videoData.duration) | 280 | options.video.set('duration', videoData.duration) |
239 | options.video.set('createdAt', videoData.createdAt) | 281 | options.video.set('createdAt', videoData.createdAt) |
240 | options.video.set('publishedAt', videoData.publishedAt) | 282 | options.video.set('publishedAt', videoData.publishedAt) |
283 | options.video.set('originallyPublishedAt', videoData.originallyPublishedAt) | ||
241 | options.video.set('privacy', videoData.privacy) | 284 | options.video.set('privacy', videoData.privacy) |
242 | options.video.set('channelId', videoData.channelId) | 285 | options.video.set('channelId', videoData.channelId) |
243 | options.video.set('views', videoData.views) | 286 | options.video.set('views', videoData.views) |
244 | 287 | ||
245 | await options.video.save(sequelizeOptions) | 288 | await options.video.save(sequelizeOptions) |
246 | 289 | ||
290 | if (thumbnailModel) if (thumbnailModel) await options.video.addAndSaveThumbnail(thumbnailModel, t) | ||
291 | |||
292 | // FIXME: use icon URL instead | ||
293 | const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename)) | ||
294 | const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) | ||
295 | await options.video.addAndSaveThumbnail(previewModel, t) | ||
296 | |||
247 | { | 297 | { |
248 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) | 298 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) |
249 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) | 299 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) |
@@ -264,6 +314,29 @@ async function updateVideoFromAP (options: { | |||
264 | } | 314 | } |
265 | 315 | ||
266 | { | 316 | { |
317 | const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes( | ||
318 | options.video, | ||
319 | options.videoObject, | ||
320 | options.video.VideoFiles | ||
321 | ) | ||
322 | const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) | ||
323 | |||
324 | // Remove video files that do not exist anymore | ||
325 | const destroyTasks = options.video.VideoStreamingPlaylists | ||
326 | .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f))) | ||
327 | .map(f => f.destroy(sequelizeOptions)) | ||
328 | await Promise.all(destroyTasks) | ||
329 | |||
330 | // Update or add other one | ||
331 | const upsertTasks = streamingPlaylistAttributes.map(a => { | ||
332 | return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) | ||
333 | .then(([ streamingPlaylist ]) => streamingPlaylist) | ||
334 | }) | ||
335 | |||
336 | options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks) | ||
337 | } | ||
338 | |||
339 | { | ||
267 | // Update Tags | 340 | // Update Tags |
268 | const tags = options.videoObject.tag.map(tag => tag.name) | 341 | const tags = options.videoObject.tag.map(tag => tag.name) |
269 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 342 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
@@ -296,12 +369,6 @@ async function updateVideoFromAP (options: { | |||
296 | logger.debug('Cannot update the remote video.', { err }) | 369 | logger.debug('Cannot update the remote video.', { err }) |
297 | throw err | 370 | throw err |
298 | } | 371 | } |
299 | |||
300 | try { | ||
301 | await generateThumbnailFromUrl(options.video, options.videoObject.icon) | ||
302 | } catch (err) { | ||
303 | logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }) | ||
304 | } | ||
305 | } | 372 | } |
306 | 373 | ||
307 | async function refreshVideoIfNeeded (options: { | 374 | async function refreshVideoIfNeeded (options: { |
@@ -361,29 +428,55 @@ export { | |||
361 | getOrCreateVideoAndAccountAndChannel, | 428 | getOrCreateVideoAndAccountAndChannel, |
362 | fetchRemoteVideoStaticFile, | 429 | fetchRemoteVideoStaticFile, |
363 | fetchRemoteVideoDescription, | 430 | fetchRemoteVideoDescription, |
364 | generateThumbnailFromUrl, | ||
365 | getOrCreateVideoChannelFromVideoObject | 431 | getOrCreateVideoChannelFromVideoObject |
366 | } | 432 | } |
367 | 433 | ||
368 | // --------------------------------------------------------------------------- | 434 | // --------------------------------------------------------------------------- |
369 | 435 | ||
370 | function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { | 436 | function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { |
371 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) | 437 | const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) |
372 | 438 | ||
373 | const urlMediaType = url.mediaType || url.mimeType | 439 | const urlMediaType = url.mediaType || url.mimeType |
374 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') | 440 | return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') |
375 | } | 441 | } |
376 | 442 | ||
443 | function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { | ||
444 | const urlMediaType = url.mediaType || url.mimeType | ||
445 | |||
446 | return urlMediaType === 'application/x-mpegURL' | ||
447 | } | ||
448 | |||
449 | function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { | ||
450 | const urlMediaType = tag.mediaType || tag.mimeType | ||
451 | |||
452 | return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' | ||
453 | } | ||
454 | |||
377 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 455 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
378 | logger.debug('Adding remote video %s.', videoObject.id) | 456 | logger.debug('Adding remote video %s.', videoObject.id) |
379 | 457 | ||
458 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
459 | const video = VideoModel.build(videoData) | ||
460 | |||
461 | const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) | ||
462 | |||
463 | let thumbnailModel: ThumbnailModel | ||
464 | if (waitThumbnail === true) { | ||
465 | thumbnailModel = await promiseThumbnail | ||
466 | } | ||
467 | |||
380 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | 468 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { |
381 | const sequelizeOptions = { transaction: t } | 469 | const sequelizeOptions = { transaction: t } |
382 | 470 | ||
383 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
384 | const video = VideoModel.build(videoData) | ||
385 | |||
386 | const videoCreated = await video.save(sequelizeOptions) | 471 | const videoCreated = await video.save(sequelizeOptions) |
472 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
473 | |||
474 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | ||
475 | |||
476 | // FIXME: use icon URL instead | ||
477 | const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) | ||
478 | const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) | ||
479 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | ||
387 | 480 | ||
388 | // Process files | 481 | // Process files |
389 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | 482 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) |
@@ -392,10 +485,16 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
392 | } | 485 | } |
393 | 486 | ||
394 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | 487 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) |
395 | await Promise.all(videoFilePromises) | 488 | const videoFiles = await Promise.all(videoFilePromises) |
489 | |||
490 | const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) | ||
491 | const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) | ||
492 | await Promise.all(playlistPromises) | ||
396 | 493 | ||
397 | // Process tags | 494 | // Process tags |
398 | const tags = videoObject.tag.map(t => t.name) | 495 | const tags = videoObject.tag |
496 | .filter(t => t.type === 'Hashtag') | ||
497 | .map(t => t.name) | ||
399 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 498 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
400 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | 499 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) |
401 | 500 | ||
@@ -407,14 +506,16 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor | |||
407 | 506 | ||
408 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | 507 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) |
409 | 508 | ||
410 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
411 | return videoCreated | 509 | return videoCreated |
412 | }) | 510 | }) |
413 | 511 | ||
414 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | 512 | if (waitThumbnail === false) { |
415 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | 513 | promiseThumbnail.then(thumbnailModel => { |
514 | thumbnailModel = videoCreated.id | ||
416 | 515 | ||
417 | if (waitThumbnail === true) await p | 516 | return thumbnailModel.save() |
517 | }) | ||
518 | } | ||
418 | 519 | ||
419 | return videoCreated | 520 | return videoCreated |
420 | } | 521 | } |
@@ -456,12 +557,14 @@ async function videoActivityObjectToDBAttributes ( | |||
456 | support, | 557 | support, |
457 | nsfw: videoObject.sensitive, | 558 | nsfw: videoObject.sensitive, |
458 | commentsEnabled: videoObject.commentsEnabled, | 559 | commentsEnabled: videoObject.commentsEnabled, |
560 | downloadEnabled: videoObject.downloadEnabled, | ||
459 | waitTranscoding: videoObject.waitTranscoding, | 561 | waitTranscoding: videoObject.waitTranscoding, |
460 | state: videoObject.state, | 562 | state: videoObject.state, |
461 | channelId: videoChannel.id, | 563 | channelId: videoChannel.id, |
462 | duration: parseInt(duration, 10), | 564 | duration: parseInt(duration, 10), |
463 | createdAt: new Date(videoObject.published), | 565 | createdAt: new Date(videoObject.published), |
464 | publishedAt: new Date(videoObject.published), | 566 | publishedAt: new Date(videoObject.published), |
567 | originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null, | ||
465 | // FIXME: updatedAt does not seems to be considered by Sequelize | 568 | // FIXME: updatedAt does not seems to be considered by Sequelize |
466 | updatedAt: new Date(videoObject.updated), | 569 | updatedAt: new Date(videoObject.updated), |
467 | views: videoObject.views, | 570 | views: videoObject.views, |
@@ -473,13 +576,13 @@ async function videoActivityObjectToDBAttributes ( | |||
473 | } | 576 | } |
474 | 577 | ||
475 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { | 578 | function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { |
476 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | 579 | const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] |
477 | 580 | ||
478 | if (fileUrls.length === 0) { | 581 | if (fileUrls.length === 0) { |
479 | throw new Error('Cannot find video files for ' + video.url) | 582 | throw new Error('Cannot find video files for ' + video.url) |
480 | } | 583 | } |
481 | 584 | ||
482 | const attributes: VideoFileModel[] = [] | 585 | const attributes: FilteredModelAttributes<VideoFileModel>[] = [] |
483 | for (const fileUrl of fileUrls) { | 586 | for (const fileUrl of fileUrls) { |
484 | // Fetch associated magnet uri | 587 | // Fetch associated magnet uri |
485 | const magnet = videoObject.url.find(u => { | 588 | const magnet = videoObject.url.find(u => { |
@@ -502,7 +605,38 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid | |||
502 | size: fileUrl.size, | 605 | size: fileUrl.size, |
503 | videoId: video.id, | 606 | videoId: video.id, |
504 | fps: fileUrl.fps || -1 | 607 | fps: fileUrl.fps || -1 |
505 | } as VideoFileModel | 608 | } |
609 | |||
610 | attributes.push(attribute) | ||
611 | } | ||
612 | |||
613 | return attributes | ||
614 | } | ||
615 | |||
616 | function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) { | ||
617 | const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] | ||
618 | if (playlistUrls.length === 0) return [] | ||
619 | |||
620 | const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] | ||
621 | for (const playlistUrlObject of playlistUrls) { | ||
622 | const segmentsSha256UrlObject = playlistUrlObject.tag | ||
623 | .find(t => { | ||
624 | return isAPPlaylistSegmentHashesUrlObject(t) | ||
625 | }) as ActivityPlaylistSegmentHashesObject | ||
626 | if (!segmentsSha256UrlObject) { | ||
627 | logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) | ||
628 | continue | ||
629 | } | ||
630 | |||
631 | const attribute = { | ||
632 | type: VideoStreamingPlaylistType.HLS, | ||
633 | playlistUrl: playlistUrlObject.href, | ||
634 | segmentsSha256Url: segmentsSha256UrlObject.href, | ||
635 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles), | ||
636 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
637 | videoId: video.id | ||
638 | } | ||
639 | |||
506 | attributes.push(attribute) | 640 | attributes.push(attribute) |
507 | } | 641 | } |
508 | 642 | ||
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 021426a1a..09b4e38ca 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import 'multer' | 1 | import 'multer' |
2 | import { sendUpdateActor } from './activitypub/send' | 2 | import { sendUpdateActor } from './activitypub/send' |
3 | import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' | 3 | import { AVATARS_SIZE } from '../initializers/constants' |
4 | import { updateActorAvatarInstance } from './activitypub' | 4 | import { updateActorAvatarInstance } from './activitypub' |
5 | import { processImage } from '../helpers/image-utils' | 5 | import { processImage } from '../helpers/image-utils' |
6 | import { AccountModel } from '../models/account/account' | 6 | import { AccountModel } from '../models/account/account' |
@@ -8,12 +8,14 @@ import { VideoChannelModel } from '../models/video/video-channel' | |||
8 | import { extname, join } from 'path' | 8 | import { extname, join } from 'path' |
9 | import { retryTransactionWrapper } from '../helpers/database-utils' | 9 | import { retryTransactionWrapper } from '../helpers/database-utils' |
10 | import * as uuidv4 from 'uuid/v4' | 10 | import * as uuidv4 from 'uuid/v4' |
11 | import { CONFIG } from '../initializers/config' | ||
12 | import { sequelizeTypescript } from '../initializers/database' | ||
11 | 13 | ||
12 | async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { | 14 | async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { |
13 | const extension = extname(avatarPhysicalFile.filename) | 15 | const extension = extname(avatarPhysicalFile.filename) |
14 | const avatarName = uuidv4() + extension | 16 | const avatarName = uuidv4() + extension |
15 | const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) | 17 | const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) |
16 | await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) | 18 | await processImage(avatarPhysicalFile.path, destination, AVATARS_SIZE) |
17 | 19 | ||
18 | return retryTransactionWrapper(() => { | 20 | return retryTransactionWrapper(() => { |
19 | return sequelizeTypescript.transaction(async t => { | 21 | return sequelizeTypescript.transaction(async t => { |
diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/cache/abstract-video-static-file-cache.ts deleted file mode 100644 index 7512f2b9d..000000000 --- a/server/lib/cache/abstract-video-static-file-cache.ts +++ /dev/null | |||
@@ -1,52 +0,0 @@ | |||
1 | import * as AsyncLRU from 'async-lru' | ||
2 | import { createWriteStream, remove } from 'fs-extra' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | import { VideoModel } from '../../models/video/video' | ||
5 | import { fetchRemoteVideoStaticFile } from '../activitypub' | ||
6 | |||
7 | export abstract class AbstractVideoStaticFileCache <T> { | ||
8 | |||
9 | protected lru | ||
10 | |||
11 | abstract getFilePath (params: T): Promise<string> | ||
12 | |||
13 | // Load and save the remote file, then return the local path from filesystem | ||
14 | protected abstract loadRemoteFile (key: string): Promise<string> | ||
15 | |||
16 | init (max: number, maxAge: number) { | ||
17 | this.lru = new AsyncLRU({ | ||
18 | max, | ||
19 | maxAge, | ||
20 | load: (key, cb) => { | ||
21 | this.loadRemoteFile(key) | ||
22 | .then(res => cb(null, res)) | ||
23 | .catch(err => cb(err)) | ||
24 | } | ||
25 | }) | ||
26 | |||
27 | this.lru.on('evict', (obj: { key: string, value: string }) => { | ||
28 | remove(obj.value) | ||
29 | .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name)) | ||
30 | }) | ||
31 | } | ||
32 | |||
33 | protected loadFromLRU (key: string) { | ||
34 | return new Promise<string>((res, rej) => { | ||
35 | this.lru.get(key, (err, value) => { | ||
36 | err ? rej(err) : res(value) | ||
37 | }) | ||
38 | }) | ||
39 | } | ||
40 | |||
41 | protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) { | ||
42 | return new Promise<string>((res, rej) => { | ||
43 | const req = fetchRemoteVideoStaticFile(video, remoteStaticPath, rej) | ||
44 | |||
45 | const stream = createWriteStream(destPath) | ||
46 | |||
47 | req.pipe(stream) | ||
48 | .on('error', (err) => rej(err)) | ||
49 | .on('finish', () => res(destPath)) | ||
50 | }) | ||
51 | } | ||
52 | } | ||
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index b2c376e20..516827a05 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import * as Bluebird from 'bluebird' | ||
3 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' | 2 | import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/models/i18n/i18n' |
4 | import { CONFIG, CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE } from '../initializers' | 3 | import { CUSTOM_HTML_TAG_COMMENTS, EMBED_SIZE, WEBSERVER } from '../initializers/constants' |
5 | import { join } from 'path' | 4 | import { join } from 'path' |
6 | import { escapeHTML } from '../helpers/core-utils' | 5 | import { escapeHTML } from '../helpers/core-utils' |
7 | import { VideoModel } from '../models/video/video' | 6 | import { VideoModel } from '../models/video/video' |
@@ -9,10 +8,14 @@ import * as validator from 'validator' | |||
9 | import { VideoPrivacy } from '../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../shared/models/videos' |
10 | import { readFile } from 'fs-extra' | 9 | import { readFile } from 'fs-extra' |
11 | import { getActivityStreamDuration } from '../models/video/video-format-utils' | 10 | import { getActivityStreamDuration } from '../models/video/video-format-utils' |
11 | import { AccountModel } from '../models/account/account' | ||
12 | import { VideoChannelModel } from '../models/video/video-channel' | ||
13 | import * as Bluebird from 'bluebird' | ||
14 | import { CONFIG } from '../initializers/config' | ||
12 | 15 | ||
13 | export class ClientHtml { | 16 | export class ClientHtml { |
14 | 17 | ||
15 | private static htmlCache: { [path: string]: string } = {} | 18 | private static htmlCache: { [ path: string ]: string } = {} |
16 | 19 | ||
17 | static invalidCache () { | 20 | static invalidCache () { |
18 | ClientHtml.htmlCache = {} | 21 | ClientHtml.htmlCache = {} |
@@ -28,18 +31,14 @@ export class ClientHtml { | |||
28 | } | 31 | } |
29 | 32 | ||
30 | static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { | 33 | static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { |
31 | let videoPromise: Bluebird<VideoModel> | ||
32 | |||
33 | // Let Angular application handle errors | 34 | // Let Angular application handle errors |
34 | if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) { | 35 | if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { |
35 | videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | ||
36 | } else { | ||
37 | return ClientHtml.getIndexHTML(req, res) | 36 | return ClientHtml.getIndexHTML(req, res) |
38 | } | 37 | } |
39 | 38 | ||
40 | const [ html, video ] = await Promise.all([ | 39 | const [ html, video ] = await Promise.all([ |
41 | ClientHtml.getIndexHTML(req, res), | 40 | ClientHtml.getIndexHTML(req, res), |
42 | videoPromise | 41 | VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) |
43 | ]) | 42 | ]) |
44 | 43 | ||
45 | // Let Angular application handle errors | 44 | // Let Angular application handle errors |
@@ -49,14 +48,44 @@ export class ClientHtml { | |||
49 | 48 | ||
50 | let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name)) | 49 | let customHtml = ClientHtml.addTitleTag(html, escapeHTML(video.name)) |
51 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description)) | 50 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(video.description)) |
52 | customHtml = ClientHtml.addOpenGraphAndOEmbedTags(customHtml, video) | 51 | customHtml = ClientHtml.addVideoOpenGraphAndOEmbedTags(customHtml, video) |
52 | |||
53 | return customHtml | ||
54 | } | ||
55 | |||
56 | static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | ||
57 | return this.getAccountOrChannelHTMLPage(() => AccountModel.loadByNameWithHost(nameWithHost), req, res) | ||
58 | } | ||
59 | |||
60 | static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { | ||
61 | return this.getAccountOrChannelHTMLPage(() => VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost), req, res) | ||
62 | } | ||
63 | |||
64 | private static async getAccountOrChannelHTMLPage ( | ||
65 | loader: () => Bluebird<AccountModel | VideoChannelModel>, | ||
66 | req: express.Request, | ||
67 | res: express.Response | ||
68 | ) { | ||
69 | const [ html, entity ] = await Promise.all([ | ||
70 | ClientHtml.getIndexHTML(req, res), | ||
71 | loader() | ||
72 | ]) | ||
73 | |||
74 | // Let Angular application handle errors | ||
75 | if (!entity) { | ||
76 | return ClientHtml.getIndexHTML(req, res) | ||
77 | } | ||
78 | |||
79 | let customHtml = ClientHtml.addTitleTag(html, escapeHTML(entity.getDisplayName())) | ||
80 | customHtml = ClientHtml.addDescriptionTag(customHtml, escapeHTML(entity.description)) | ||
81 | customHtml = ClientHtml.addAccountOrChannelMetaTags(customHtml, entity) | ||
53 | 82 | ||
54 | return customHtml | 83 | return customHtml |
55 | } | 84 | } |
56 | 85 | ||
57 | private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { | 86 | private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { |
58 | const path = ClientHtml.getIndexPath(req, res, paramLang) | 87 | const path = ClientHtml.getIndexPath(req, res, paramLang) |
59 | if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | 88 | if (ClientHtml.htmlCache[ path ]) return ClientHtml.htmlCache[ path ] |
60 | 89 | ||
61 | const buffer = await readFile(path) | 90 | const buffer = await readFile(path) |
62 | 91 | ||
@@ -64,7 +93,7 @@ export class ClientHtml { | |||
64 | 93 | ||
65 | html = ClientHtml.addCustomCSS(html) | 94 | html = ClientHtml.addCustomCSS(html) |
66 | 95 | ||
67 | ClientHtml.htmlCache[path] = html | 96 | ClientHtml.htmlCache[ path ] = html |
68 | 97 | ||
69 | return html | 98 | return html |
70 | } | 99 | } |
@@ -78,7 +107,7 @@ export class ClientHtml { | |||
78 | 107 | ||
79 | // Save locale in cookies | 108 | // Save locale in cookies |
80 | res.cookie('clientLanguage', lang, { | 109 | res.cookie('clientLanguage', lang, { |
81 | secure: CONFIG.WEBSERVER.SCHEME === 'https', | 110 | secure: WEBSERVER.SCHEME === 'https', |
82 | sameSite: true, | 111 | sameSite: true, |
83 | maxAge: 1000 * 3600 * 24 * 90 // 3 months | 112 | maxAge: 1000 * 3600 * 24 * 90 // 3 months |
84 | }) | 113 | }) |
@@ -114,13 +143,13 @@ export class ClientHtml { | |||
114 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) | 143 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) |
115 | } | 144 | } |
116 | 145 | ||
117 | private static addOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { | 146 | private static addVideoOpenGraphAndOEmbedTags (htmlStringPage: string, video: VideoModel) { |
118 | const previewUrl = CONFIG.WEBSERVER.URL + video.getPreviewStaticPath() | 147 | const previewUrl = WEBSERVER.URL + video.getPreviewStaticPath() |
119 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() | 148 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() |
120 | 149 | ||
121 | const videoNameEscaped = escapeHTML(video.name) | 150 | const videoNameEscaped = escapeHTML(video.name) |
122 | const videoDescriptionEscaped = escapeHTML(video.description) | 151 | const videoDescriptionEscaped = escapeHTML(video.description) |
123 | const embedUrl = CONFIG.WEBSERVER.URL + video.getEmbedStaticPath() | 152 | const embedUrl = WEBSERVER.URL + video.getEmbedStaticPath() |
124 | 153 | ||
125 | const openGraphMetaTags = { | 154 | const openGraphMetaTags = { |
126 | 'og:type': 'video', | 155 | 'og:type': 'video', |
@@ -152,7 +181,7 @@ export class ClientHtml { | |||
152 | const oembedLinkTags = [ | 181 | const oembedLinkTags = [ |
153 | { | 182 | { |
154 | type: 'application/json+oembed', | 183 | type: 'application/json+oembed', |
155 | href: CONFIG.WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(videoUrl), | 184 | href: WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(videoUrl), |
156 | title: videoNameEscaped | 185 | title: videoNameEscaped |
157 | } | 186 | } |
158 | ] | 187 | ] |
@@ -174,7 +203,7 @@ export class ClientHtml { | |||
174 | 203 | ||
175 | // Opengraph | 204 | // Opengraph |
176 | Object.keys(openGraphMetaTags).forEach(tagName => { | 205 | Object.keys(openGraphMetaTags).forEach(tagName => { |
177 | const tagValue = openGraphMetaTags[tagName] | 206 | const tagValue = openGraphMetaTags[ tagName ] |
178 | 207 | ||
179 | tagsString += `<meta property="${tagName}" content="${tagValue}" />` | 208 | tagsString += `<meta property="${tagName}" content="${tagValue}" />` |
180 | }) | 209 | }) |
@@ -190,6 +219,17 @@ export class ClientHtml { | |||
190 | // SEO, use origin video url so Google does not index remote videos | 219 | // SEO, use origin video url so Google does not index remote videos |
191 | tagsString += `<link rel="canonical" href="${video.url}" />` | 220 | tagsString += `<link rel="canonical" href="${video.url}" />` |
192 | 221 | ||
193 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.OPENGRAPH_AND_OEMBED, tagsString) | 222 | return this.addOpenGraphAndOEmbedTags(htmlStringPage, tagsString) |
223 | } | ||
224 | |||
225 | private static addAccountOrChannelMetaTags (htmlStringPage: string, entity: AccountModel | VideoChannelModel) { | ||
226 | // SEO, use origin account or channel URL | ||
227 | const metaTags = `<link rel="canonical" href="${entity.Actor.url}" />` | ||
228 | |||
229 | return this.addOpenGraphAndOEmbedTags(htmlStringPage, metaTags) | ||
230 | } | ||
231 | |||
232 | private static addOpenGraphAndOEmbedTags (htmlStringPage: string, metaTags: string) { | ||
233 | return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.META_TAGS, metaTags) | ||
194 | } | 234 | } |
195 | } | 235 | } |
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 99010f473..8c06e9751 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { createTransport, Transporter } from 'nodemailer' | 1 | import { createTransport, Transporter } from 'nodemailer' |
2 | import { isTestInstance } from '../helpers/core-utils' | 2 | import { isTestInstance } from '../helpers/core-utils' |
3 | import { bunyanLogger, logger } from '../helpers/logger' | 3 | import { bunyanLogger, logger } from '../helpers/logger' |
4 | import { CONFIG } from '../initializers' | 4 | import { CONFIG } from '../initializers/config' |
5 | import { UserModel } from '../models/account/user' | 5 | import { UserModel } from '../models/account/user' |
6 | import { VideoModel } from '../models/video/video' | 6 | import { VideoModel } from '../models/video/video' |
7 | import { JobQueue } from './job-queue' | 7 | import { JobQueue } from './job-queue' |
@@ -12,6 +12,16 @@ import { VideoAbuseModel } from '../models/video/video-abuse' | |||
12 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | 12 | import { VideoBlacklistModel } from '../models/video/video-blacklist' |
13 | import { VideoImportModel } from '../models/video/video-import' | 13 | import { VideoImportModel } from '../models/video/video-import' |
14 | import { ActorFollowModel } from '../models/activitypub/actor-follow' | 14 | import { ActorFollowModel } from '../models/activitypub/actor-follow' |
15 | import { WEBSERVER } from '../initializers/constants' | ||
16 | |||
17 | type SendEmailOptions = { | ||
18 | to: string[] | ||
19 | subject: string | ||
20 | text: string | ||
21 | |||
22 | fromDisplayName?: string | ||
23 | replyTo?: string | ||
24 | } | ||
15 | 25 | ||
16 | class Emailer { | 26 | class Emailer { |
17 | 27 | ||
@@ -82,7 +92,7 @@ class Emailer { | |||
82 | 92 | ||
83 | addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) { | 93 | addNewVideoFromSubscriberNotification (to: string[], video: VideoModel) { |
84 | const channelName = video.VideoChannel.getDisplayName() | 94 | const channelName = video.VideoChannel.getDisplayName() |
85 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() | 95 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() |
86 | 96 | ||
87 | const text = `Hi dear user,\n\n` + | 97 | const text = `Hi dear user,\n\n` + |
88 | `Your subscription ${channelName} just published a new video: ${video.name}` + | 98 | `Your subscription ${channelName} just published a new video: ${video.name}` + |
@@ -120,8 +130,26 @@ class Emailer { | |||
120 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 130 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
121 | } | 131 | } |
122 | 132 | ||
133 | addNewInstanceFollowerNotification (to: string[], actorFollow: ActorFollowModel) { | ||
134 | const awaitingApproval = actorFollow.state === 'pending' ? ' awaiting manual approval.' : '' | ||
135 | |||
136 | const text = `Hi dear admin,\n\n` + | ||
137 | `Your instance has a new follower: ${actorFollow.ActorFollower.url}${awaitingApproval}` + | ||
138 | `\n\n` + | ||
139 | `Cheers,\n` + | ||
140 | `PeerTube.` | ||
141 | |||
142 | const emailPayload: EmailPayload = { | ||
143 | to, | ||
144 | subject: 'New instance follower', | ||
145 | text | ||
146 | } | ||
147 | |||
148 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
149 | } | ||
150 | |||
123 | myVideoPublishedNotification (to: string[], video: VideoModel) { | 151 | myVideoPublishedNotification (to: string[], video: VideoModel) { |
124 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() | 152 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() |
125 | 153 | ||
126 | const text = `Hi dear user,\n\n` + | 154 | const text = `Hi dear user,\n\n` + |
127 | `Your video ${video.name} has been published.` + | 155 | `Your video ${video.name} has been published.` + |
@@ -141,7 +169,7 @@ class Emailer { | |||
141 | } | 169 | } |
142 | 170 | ||
143 | myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) { | 171 | myVideoImportSuccessNotification (to: string[], videoImport: VideoImportModel) { |
144 | const videoUrl = CONFIG.WEBSERVER.URL + videoImport.Video.getWatchStaticPath() | 172 | const videoUrl = WEBSERVER.URL + videoImport.Video.getWatchStaticPath() |
145 | 173 | ||
146 | const text = `Hi dear user,\n\n` + | 174 | const text = `Hi dear user,\n\n` + |
147 | `Your video import ${videoImport.getTargetIdentifier()} is finished.` + | 175 | `Your video import ${videoImport.getTargetIdentifier()} is finished.` + |
@@ -161,7 +189,7 @@ class Emailer { | |||
161 | } | 189 | } |
162 | 190 | ||
163 | myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) { | 191 | myVideoImportErrorNotification (to: string[], videoImport: VideoImportModel) { |
164 | const importUrl = CONFIG.WEBSERVER.URL + '/my-account/video-imports' | 192 | const importUrl = WEBSERVER.URL + '/my-account/video-imports' |
165 | 193 | ||
166 | const text = `Hi dear user,\n\n` + | 194 | const text = `Hi dear user,\n\n` + |
167 | `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` + | 195 | `Your video import ${videoImport.getTargetIdentifier()} encountered an error.` + |
@@ -183,7 +211,7 @@ class Emailer { | |||
183 | addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) { | 211 | addNewCommentOnMyVideoNotification (to: string[], comment: VideoCommentModel) { |
184 | const accountName = comment.Account.getDisplayName() | 212 | const accountName = comment.Account.getDisplayName() |
185 | const video = comment.Video | 213 | const video = comment.Video |
186 | const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() | 214 | const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath() |
187 | 215 | ||
188 | const text = `Hi dear user,\n\n` + | 216 | const text = `Hi dear user,\n\n` + |
189 | `A new comment has been posted by ${accountName} on your video ${video.name}` + | 217 | `A new comment has been posted by ${accountName} on your video ${video.name}` + |
@@ -205,7 +233,7 @@ class Emailer { | |||
205 | addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) { | 233 | addNewCommentMentionNotification (to: string[], comment: VideoCommentModel) { |
206 | const accountName = comment.Account.getDisplayName() | 234 | const accountName = comment.Account.getDisplayName() |
207 | const video = comment.Video | 235 | const video = comment.Video |
208 | const commentUrl = CONFIG.WEBSERVER.URL + comment.getCommentStaticPath() | 236 | const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath() |
209 | 237 | ||
210 | const text = `Hi dear user,\n\n` + | 238 | const text = `Hi dear user,\n\n` + |
211 | `${accountName} mentioned you on video ${video.name}` + | 239 | `${accountName} mentioned you on video ${video.name}` + |
@@ -225,10 +253,10 @@ class Emailer { | |||
225 | } | 253 | } |
226 | 254 | ||
227 | addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { | 255 | addVideoAbuseModeratorsNotification (to: string[], videoAbuse: VideoAbuseModel) { |
228 | const videoUrl = CONFIG.WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath() | 256 | const videoUrl = WEBSERVER.URL + videoAbuse.Video.getWatchStaticPath() |
229 | 257 | ||
230 | const text = `Hi,\n\n` + | 258 | const text = `Hi,\n\n` + |
231 | `${CONFIG.WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` + | 259 | `${WEBSERVER.HOST} received an abuse for the following video ${videoUrl}\n\n` + |
232 | `Cheers,\n` + | 260 | `Cheers,\n` + |
233 | `PeerTube.` | 261 | `PeerTube.` |
234 | 262 | ||
@@ -241,15 +269,38 @@ class Emailer { | |||
241 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 269 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
242 | } | 270 | } |
243 | 271 | ||
272 | addVideoAutoBlacklistModeratorsNotification (to: string[], video: VideoModel) { | ||
273 | const VIDEO_AUTO_BLACKLIST_URL = WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list' | ||
274 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() | ||
275 | |||
276 | const text = `Hi,\n\n` + | ||
277 | `A recently added video was auto-blacklisted and requires moderator review before publishing.` + | ||
278 | `\n\n` + | ||
279 | `You can view it and take appropriate action on ${videoUrl}` + | ||
280 | `\n\n` + | ||
281 | `A full list of auto-blacklisted videos can be reviewed here: ${VIDEO_AUTO_BLACKLIST_URL}` + | ||
282 | `\n\n` + | ||
283 | `Cheers,\n` + | ||
284 | `PeerTube.` | ||
285 | |||
286 | const emailPayload: EmailPayload = { | ||
287 | to, | ||
288 | subject: '[PeerTube] An auto-blacklisted video is awaiting review', | ||
289 | text | ||
290 | } | ||
291 | |||
292 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | ||
293 | } | ||
294 | |||
244 | addNewUserRegistrationNotification (to: string[], user: UserModel) { | 295 | addNewUserRegistrationNotification (to: string[], user: UserModel) { |
245 | const text = `Hi,\n\n` + | 296 | const text = `Hi,\n\n` + |
246 | `User ${user.username} just registered on ${CONFIG.WEBSERVER.HOST} PeerTube instance.\n\n` + | 297 | `User ${user.username} just registered on ${WEBSERVER.HOST} PeerTube instance.\n\n` + |
247 | `Cheers,\n` + | 298 | `Cheers,\n` + |
248 | `PeerTube.` | 299 | `PeerTube.` |
249 | 300 | ||
250 | const emailPayload: EmailPayload = { | 301 | const emailPayload: EmailPayload = { |
251 | to, | 302 | to, |
252 | subject: '[PeerTube] New user registration on ' + CONFIG.WEBSERVER.HOST, | 303 | subject: '[PeerTube] New user registration on ' + WEBSERVER.HOST, |
253 | text | 304 | text |
254 | } | 305 | } |
255 | 306 | ||
@@ -258,10 +309,10 @@ class Emailer { | |||
258 | 309 | ||
259 | addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { | 310 | addVideoBlacklistNotification (to: string[], videoBlacklist: VideoBlacklistModel) { |
260 | const videoName = videoBlacklist.Video.name | 311 | const videoName = videoBlacklist.Video.name |
261 | const videoUrl = CONFIG.WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath() | 312 | const videoUrl = WEBSERVER.URL + videoBlacklist.Video.getWatchStaticPath() |
262 | 313 | ||
263 | const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : '' | 314 | const reasonString = videoBlacklist.reason ? ` for the following reason: ${videoBlacklist.reason}` : '' |
264 | const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.WEBSERVER.HOST} has been blacklisted${reasonString}.` | 315 | const blockedString = `Your video ${videoName} (${videoUrl} on ${WEBSERVER.HOST} has been blacklisted${reasonString}.` |
265 | 316 | ||
266 | const text = 'Hi,\n\n' + | 317 | const text = 'Hi,\n\n' + |
267 | blockedString + | 318 | blockedString + |
@@ -279,10 +330,10 @@ class Emailer { | |||
279 | } | 330 | } |
280 | 331 | ||
281 | addVideoUnblacklistNotification (to: string[], video: VideoModel) { | 332 | addVideoUnblacklistNotification (to: string[], video: VideoModel) { |
282 | const videoUrl = CONFIG.WEBSERVER.URL + video.getWatchStaticPath() | 333 | const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() |
283 | 334 | ||
284 | const text = 'Hi,\n\n' + | 335 | const text = 'Hi,\n\n' + |
285 | `Your video ${video.name} (${videoUrl}) on ${CONFIG.WEBSERVER.HOST} has been unblacklisted.` + | 336 | `Your video ${video.name} (${videoUrl}) on ${WEBSERVER.HOST} has been unblacklisted.` + |
286 | '\n\n' + | 337 | '\n\n' + |
287 | 'Cheers,\n' + | 338 | 'Cheers,\n' + |
288 | `PeerTube.` | 339 | `PeerTube.` |
@@ -296,9 +347,9 @@ class Emailer { | |||
296 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) | 347 | return JobQueue.Instance.createJob({ type: 'email', payload: emailPayload }) |
297 | } | 348 | } |
298 | 349 | ||
299 | addForgetPasswordEmailJob (to: string, resetPasswordUrl: string) { | 350 | addPasswordResetEmailJob (to: string, resetPasswordUrl: string) { |
300 | const text = `Hi dear user,\n\n` + | 351 | const text = `Hi dear user,\n\n` + |
301 | `It seems you forgot your password on ${CONFIG.WEBSERVER.HOST}! ` + | 352 | `A reset password procedure for your account ${to} has been requested on ${WEBSERVER.HOST} ` + |
302 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + | 353 | `Please follow this link to reset it: ${resetPasswordUrl}\n\n` + |
303 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | 354 | `If you are not the person who initiated this request, please ignore this email.\n\n` + |
304 | `Cheers,\n` + | 355 | `Cheers,\n` + |
@@ -315,7 +366,7 @@ class Emailer { | |||
315 | 366 | ||
316 | addVerifyEmailJob (to: string, verifyEmailUrl: string) { | 367 | addVerifyEmailJob (to: string, verifyEmailUrl: string) { |
317 | const text = `Welcome to PeerTube,\n\n` + | 368 | const text = `Welcome to PeerTube,\n\n` + |
318 | `To start using PeerTube on ${CONFIG.WEBSERVER.HOST} you must verify your email! ` + | 369 | `To start using PeerTube on ${WEBSERVER.HOST} you must verify your email! ` + |
319 | `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + | 370 | `Please follow this link to verify this email belongs to you: ${verifyEmailUrl}\n\n` + |
320 | `If you are not the person who initiated this request, please ignore this email.\n\n` + | 371 | `If you are not the person who initiated this request, please ignore this email.\n\n` + |
321 | `Cheers,\n` + | 372 | `Cheers,\n` + |
@@ -333,7 +384,7 @@ class Emailer { | |||
333 | addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { | 384 | addUserBlockJob (user: UserModel, blocked: boolean, reason?: string) { |
334 | const reasonString = reason ? ` for the following reason: ${reason}` : '' | 385 | const reasonString = reason ? ` for the following reason: ${reason}` : '' |
335 | const blockedWord = blocked ? 'blocked' : 'unblocked' | 386 | const blockedWord = blocked ? 'blocked' : 'unblocked' |
336 | const blockedString = `Your account ${user.username} on ${CONFIG.WEBSERVER.HOST} has been ${blockedWord}${reasonString}.` | 387 | const blockedString = `Your account ${user.username} on ${WEBSERVER.HOST} has been ${blockedWord}${reasonString}.` |
337 | 388 | ||
338 | const text = 'Hi,\n\n' + | 389 | const text = 'Hi,\n\n' + |
339 | blockedString + | 390 | blockedString + |
@@ -378,7 +429,7 @@ class Emailer { | |||
378 | 429 | ||
379 | const fromDisplayName = options.fromDisplayName | 430 | const fromDisplayName = options.fromDisplayName |
380 | ? options.fromDisplayName | 431 | ? options.fromDisplayName |
381 | : CONFIG.WEBSERVER.HOST | 432 | : WEBSERVER.HOST |
382 | 433 | ||
383 | return this.transporter.sendMail({ | 434 | return this.transporter.sendMail({ |
384 | from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>`, | 435 | from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>`, |
@@ -402,5 +453,6 @@ class Emailer { | |||
402 | // --------------------------------------------------------------------------- | 453 | // --------------------------------------------------------------------------- |
403 | 454 | ||
404 | export { | 455 | export { |
405 | Emailer | 456 | Emailer, |
457 | SendEmailOptions | ||
406 | } | 458 | } |
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts new file mode 100644 index 000000000..1908cfb06 --- /dev/null +++ b/server/lib/files-cache/abstract-video-static-file-cache.ts | |||
@@ -0,0 +1,30 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { logger } from '../../helpers/logger' | ||
3 | import * as memoizee from 'memoizee' | ||
4 | |||
5 | type GetFilePathResult = { isOwned: boolean, path: string } | undefined | ||
6 | |||
7 | export abstract class AbstractVideoStaticFileCache <T> { | ||
8 | |||
9 | getFilePath: (params: T) => Promise<GetFilePathResult> | ||
10 | |||
11 | abstract getFilePathImpl (params: T): Promise<GetFilePathResult> | ||
12 | |||
13 | // Load and save the remote file, then return the local path from filesystem | ||
14 | protected abstract loadRemoteFile (key: string): Promise<GetFilePathResult> | ||
15 | |||
16 | init (max: number, maxAge: number) { | ||
17 | this.getFilePath = memoizee(this.getFilePathImpl, { | ||
18 | maxAge, | ||
19 | max, | ||
20 | promise: true, | ||
21 | dispose: (result: GetFilePathResult) => { | ||
22 | if (result.isOwned !== true) { | ||
23 | remove(result.path) | ||
24 | .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) | ||
25 | .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) | ||
26 | } | ||
27 | } | ||
28 | }) | ||
29 | } | ||
30 | } | ||
diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/files-cache/actor-follow-score-cache.ts index d070bde09..5f8ee806f 100644 --- a/server/lib/cache/actor-follow-score-cache.ts +++ b/server/lib/files-cache/actor-follow-score-cache.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { ACTOR_FOLLOW_SCORE } from '../../initializers' | 1 | import { ACTOR_FOLLOW_SCORE } from '../../initializers/constants' |
2 | import { logger } from '../../helpers/logger' | 2 | import { logger } from '../../helpers/logger' |
3 | 3 | ||
4 | // Cache follows scores, instead of writing them too often in database | 4 | // Cache follows scores, instead of writing them too often in database |
diff --git a/server/lib/cache/index.ts b/server/lib/files-cache/index.ts index e921d04a7..e921d04a7 100644 --- a/server/lib/cache/index.ts +++ b/server/lib/files-cache/index.ts | |||
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts index f240affbc..440c3fde8 100644 --- a/server/lib/cache/videos-caption-cache.ts +++ b/server/lib/files-cache/videos-caption-cache.ts | |||
@@ -1,8 +1,11 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { CACHE, CONFIG } from '../../initializers' | 2 | import { FILES_CACHE } from '../../initializers/constants' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { VideoCaptionModel } from '../../models/video/video-caption' | 4 | import { VideoCaptionModel } from '../../models/video/video-caption' |
5 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' | 5 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' |
6 | import { CONFIG } from '../../initializers/config' | ||
7 | import { logger } from '../../helpers/logger' | ||
8 | import { fetchRemoteVideoStaticFile } from '../activitypub' | ||
6 | 9 | ||
7 | type GetPathParam = { videoId: string, language: string } | 10 | type GetPathParam = { videoId: string, language: string } |
8 | 11 | ||
@@ -19,17 +22,19 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> { | |||
19 | return this.instance || (this.instance = new this()) | 22 | return this.instance || (this.instance = new this()) |
20 | } | 23 | } |
21 | 24 | ||
22 | async getFilePath (params: GetPathParam) { | 25 | async getFilePathImpl (params: GetPathParam) { |
23 | const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language) | 26 | const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language) |
24 | if (!videoCaption) return undefined | 27 | if (!videoCaption) return undefined |
25 | 28 | ||
26 | if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) | 29 | if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) } |
27 | 30 | ||
28 | const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language | 31 | const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language |
29 | return this.loadFromLRU(key) | 32 | return this.loadRemoteFile(key) |
30 | } | 33 | } |
31 | 34 | ||
32 | protected async loadRemoteFile (key: string) { | 35 | protected async loadRemoteFile (key: string) { |
36 | logger.debug('Loading remote caption file %s.', key) | ||
37 | |||
33 | const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER) | 38 | const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER) |
34 | 39 | ||
35 | const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language) | 40 | const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language) |
@@ -41,10 +46,13 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> { | |||
41 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) | 46 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) |
42 | if (!video) return undefined | 47 | if (!video) return undefined |
43 | 48 | ||
49 | // FIXME: use URL | ||
44 | const remoteStaticPath = videoCaption.getCaptionStaticPath() | 50 | const remoteStaticPath = videoCaption.getCaptionStaticPath() |
45 | const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) | 51 | const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) |
52 | |||
53 | await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath) | ||
46 | 54 | ||
47 | return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) | 55 | return { isOwned: false, path: destPath } |
48 | } | 56 | } |
49 | } | 57 | } |
50 | 58 | ||
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts index a5d6f5b62..14be7f24a 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts | |||
@@ -1,7 +1,9 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers' | 2 | import { FILES_CACHE, STATIC_PATHS } from '../../initializers/constants' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' | 4 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' |
5 | import { CONFIG } from '../../initializers/config' | ||
6 | import { fetchRemoteVideoStaticFile } from '../activitypub' | ||
5 | 7 | ||
6 | class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | 8 | class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { |
7 | 9 | ||
@@ -15,13 +17,13 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | |||
15 | return this.instance || (this.instance = new this()) | 17 | return this.instance || (this.instance = new this()) |
16 | } | 18 | } |
17 | 19 | ||
18 | async getFilePath (videoUUID: string) { | 20 | async getFilePathImpl (videoUUID: string) { |
19 | const video = await VideoModel.loadByUUIDWithFile(videoUUID) | 21 | const video = await VideoModel.loadByUUIDWithFile(videoUUID) |
20 | if (!video) return undefined | 22 | if (!video) return undefined |
21 | 23 | ||
22 | if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) | 24 | if (video.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreview().filename) } |
23 | 25 | ||
24 | return this.loadFromLRU(videoUUID) | 26 | return this.loadRemoteFile(videoUUID) |
25 | } | 27 | } |
26 | 28 | ||
27 | protected async loadRemoteFile (key: string) { | 29 | protected async loadRemoteFile (key: string) { |
@@ -30,10 +32,13 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | |||
30 | 32 | ||
31 | if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') | 33 | if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') |
32 | 34 | ||
33 | const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) | 35 | // FIXME: use URL |
34 | const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName()) | 36 | const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreview().filename) |
37 | const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreview().filename) | ||
35 | 38 | ||
36 | return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) | 39 | await fetchRemoteVideoStaticFile(video, remoteStaticPath, destPath) |
40 | |||
41 | return { isOwned: false, path: destPath } | ||
37 | } | 42 | } |
38 | } | 43 | } |
39 | 44 | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts new file mode 100644 index 000000000..98da4dcd8 --- /dev/null +++ b/server/lib/hls.ts | |||
@@ -0,0 +1,184 @@ | |||
1 | import { VideoModel } from '../models/video/video' | ||
2 | import { basename, dirname, join } from 'path' | ||
3 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' | ||
4 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' | ||
5 | import { getVideoFileSize } from '../helpers/ffmpeg-utils' | ||
6 | import { sha256 } from '../helpers/core-utils' | ||
7 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
8 | import { logger } from '../helpers/logger' | ||
9 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | ||
10 | import { generateRandomString } from '../helpers/utils' | ||
11 | import { flatten, uniq } from 'lodash' | ||
12 | import { VideoFileModel } from '../models/video/video-file' | ||
13 | import { CONFIG } from '../initializers/config' | ||
14 | import { sequelizeTypescript } from '../initializers/database' | ||
15 | |||
16 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | ||
17 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | ||
18 | |||
19 | // Use separate SQL queries, because we could have many videos to update | ||
20 | for (const playlist of playlistsToUpdate) { | ||
21 | await sequelizeTypescript.transaction(async t => { | ||
22 | const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) | ||
23 | |||
24 | playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) | ||
25 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | ||
26 | await playlist.save({ transaction: t }) | ||
27 | }) | ||
28 | } | ||
29 | } | ||
30 | |||
31 | async function updateMasterHLSPlaylist (video: VideoModel) { | ||
32 | const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
33 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | ||
34 | const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) | ||
35 | |||
36 | for (const file of video.VideoFiles) { | ||
37 | // If we did not generated a playlist for this resolution, skip | ||
38 | const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
39 | if (await pathExists(filePlaylistPath) === false) continue | ||
40 | |||
41 | const videoFilePath = video.getVideoFilePath(file) | ||
42 | |||
43 | const size = await getVideoFileSize(videoFilePath) | ||
44 | |||
45 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
46 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
47 | |||
48 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
49 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
50 | |||
51 | masterPlaylists.push(line) | ||
52 | masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
53 | } | ||
54 | |||
55 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
56 | } | ||
57 | |||
58 | async function updateSha256Segments (video: VideoModel) { | ||
59 | const json: { [filename: string]: { [range: string]: string } } = {} | ||
60 | |||
61 | const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
62 | |||
63 | // For all the resolutions available for this video | ||
64 | for (const file of video.VideoFiles) { | ||
65 | const rangeHashes: { [range: string]: string } = {} | ||
66 | |||
67 | const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) | ||
68 | const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) | ||
69 | |||
70 | // Maybe the playlist is not generated for this resolution yet | ||
71 | if (!await pathExists(playlistPath)) continue | ||
72 | |||
73 | const playlistContent = await readFile(playlistPath) | ||
74 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
75 | |||
76 | const fd = await open(videoPath, 'r') | ||
77 | for (const range of ranges) { | ||
78 | const buf = Buffer.alloc(range.length) | ||
79 | await read(fd, buf, 0, range.length, range.offset) | ||
80 | |||
81 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | ||
82 | } | ||
83 | await close(fd) | ||
84 | |||
85 | const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) | ||
86 | json[videoFilename] = rangeHashes | ||
87 | } | ||
88 | |||
89 | const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) | ||
90 | await outputJSON(outputPath, json) | ||
91 | } | ||
92 | |||
93 | function getRangesFromPlaylist (playlistContent: string) { | ||
94 | const ranges: { offset: number, length: number }[] = [] | ||
95 | const lines = playlistContent.split('\n') | ||
96 | const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ | ||
97 | |||
98 | for (const line of lines) { | ||
99 | const captured = regex.exec(line) | ||
100 | |||
101 | if (captured) { | ||
102 | ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) | ||
103 | } | ||
104 | } | ||
105 | |||
106 | return ranges | ||
107 | } | ||
108 | |||
109 | function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { | ||
110 | let timer | ||
111 | |||
112 | logger.info('Importing HLS playlist %s', playlistUrl) | ||
113 | |||
114 | return new Promise<string>(async (res, rej) => { | ||
115 | const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) | ||
116 | |||
117 | await ensureDir(tmpDirectory) | ||
118 | |||
119 | timer = setTimeout(() => { | ||
120 | deleteTmpDirectory(tmpDirectory) | ||
121 | |||
122 | return rej(new Error('HLS download timeout.')) | ||
123 | }, timeout) | ||
124 | |||
125 | try { | ||
126 | // Fetch master playlist | ||
127 | const subPlaylistUrls = await fetchUniqUrls(playlistUrl) | ||
128 | |||
129 | const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) | ||
130 | const fileUrls = uniq(flatten(await Promise.all(subRequests))) | ||
131 | |||
132 | logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) | ||
133 | |||
134 | for (const fileUrl of fileUrls) { | ||
135 | const destPath = join(tmpDirectory, basename(fileUrl)) | ||
136 | |||
137 | const bodyKBLimit = 10 * 1000 * 1000 // 10GB | ||
138 | await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit) | ||
139 | } | ||
140 | |||
141 | clearTimeout(timer) | ||
142 | |||
143 | await move(tmpDirectory, destinationDir, { overwrite: true }) | ||
144 | |||
145 | return res() | ||
146 | } catch (err) { | ||
147 | deleteTmpDirectory(tmpDirectory) | ||
148 | |||
149 | return rej(err) | ||
150 | } | ||
151 | }) | ||
152 | |||
153 | function deleteTmpDirectory (directory: string) { | ||
154 | remove(directory) | ||
155 | .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) | ||
156 | } | ||
157 | |||
158 | async function fetchUniqUrls (playlistUrl: string) { | ||
159 | const { body } = await doRequest<string>({ uri: playlistUrl }) | ||
160 | |||
161 | if (!body) return [] | ||
162 | |||
163 | const urls = body.split('\n') | ||
164 | .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) | ||
165 | .map(url => { | ||
166 | if (url.startsWith('http://') || url.startsWith('https://')) return url | ||
167 | |||
168 | return `${dirname(playlistUrl)}/${url}` | ||
169 | }) | ||
170 | |||
171 | return uniq(urls) | ||
172 | } | ||
173 | } | ||
174 | |||
175 | // --------------------------------------------------------------------------- | ||
176 | |||
177 | export { | ||
178 | updateMasterHLSPlaylist, | ||
179 | updateSha256Segments, | ||
180 | downloadPlaylistSegments, | ||
181 | updateStreamingPlaylistsInfohashesIfNeeded | ||
182 | } | ||
183 | |||
184 | // --------------------------------------------------------------------------- | ||
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index b4d381062..b3defb617 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { CONFIG, REMOTE_SCHEME, sequelizeTypescript } from '../../../initializers' | 3 | import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' |
4 | import { sendFollow } from '../../activitypub/send' | 4 | import { sendFollow } from '../../activitypub/send' |
5 | import { sanitizeHost } from '../../../helpers/core-utils' | 5 | import { sanitizeHost } from '../../../helpers/core-utils' |
6 | import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger' | 6 | import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger' |
@@ -9,6 +9,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
9 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 9 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
10 | import { ActorModel } from '../../../models/activitypub/actor' | 10 | import { ActorModel } from '../../../models/activitypub/actor' |
11 | import { Notifier } from '../../notifier' | 11 | import { Notifier } from '../../notifier' |
12 | import { sequelizeTypescript } from '../../../initializers/database' | ||
12 | 13 | ||
13 | export type ActivitypubFollowPayload = { | 14 | export type ActivitypubFollowPayload = { |
14 | followerActorId: number | 15 | followerActorId: number |
@@ -23,7 +24,7 @@ async function processActivityPubFollow (job: Bull.Job) { | |||
23 | logger.info('Processing ActivityPub follow in job %d.', job.id) | 24 | logger.info('Processing ActivityPub follow in job %d.', job.id) |
24 | 25 | ||
25 | let targetActor: ActorModel | 26 | let targetActor: ActorModel |
26 | if (!host || host === CONFIG.WEBSERVER.HOST) { | 27 | if (!host || host === WEBSERVER.HOST) { |
27 | targetActor = await ActorModel.loadLocalByName(payload.name) | 28 | targetActor = await ActorModel.loadLocalByName(payload.name) |
28 | } else { | 29 | } else { |
29 | const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) | 30 | const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) |
@@ -73,5 +74,5 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) { | |||
73 | return actorFollow | 74 | return actorFollow |
74 | }) | 75 | }) |
75 | 76 | ||
76 | if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewFollow(actorFollow) | 77 | if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollow) |
77 | } | 78 | } |
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 9493945ff..0ff7b44a0 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts | |||
@@ -2,10 +2,9 @@ import * as Bull from 'bull' | |||
2 | import * as Bluebird from 'bluebird' | 2 | import * as Bluebird from 'bluebird' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { doRequest } from '../../../helpers/requests' | 4 | import { doRequest } from '../../../helpers/requests' |
5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | ||
6 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' | 5 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' |
7 | import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' | 6 | import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants' |
8 | import { ActorFollowScoreCache } from '../../cache' | 7 | import { ActorFollowScoreCache } from '../../files-cache' |
9 | 8 | ||
10 | export type ActivitypubHttpBroadcastPayload = { | 9 | export type ActivitypubHttpBroadcastPayload = { |
11 | uris: string[] | 10 | uris: string[] |
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 67ccfa995..23d33c26f 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts | |||
@@ -1,17 +1,24 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import * as Bluebird from 'bluebird' | ||
2 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
3 | import { processActivities } from '../../activitypub/process' | 4 | import { processActivities } from '../../activitypub/process' |
4 | import { addVideoComments } from '../../activitypub/video-comments' | 5 | import { addVideoComments } from '../../activitypub/video-comments' |
5 | import { crawlCollectionPage } from '../../activitypub/crawl' | 6 | import { crawlCollectionPage } from '../../activitypub/crawl' |
6 | import { VideoModel } from '../../../models/video/video' | 7 | import { VideoModel } from '../../../models/video/video' |
7 | import { addVideoShares, createRates } from '../../activitypub' | 8 | import { addVideoShares, createRates } from '../../activitypub' |
9 | import { createAccountPlaylists } from '../../activitypub/playlist' | ||
10 | import { AccountModel } from '../../../models/account/account' | ||
11 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
12 | import { VideoShareModel } from '../../../models/video/video-share' | ||
13 | import { VideoCommentModel } from '../../../models/video/video-comment' | ||
8 | 14 | ||
9 | type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 15 | type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists' |
10 | 16 | ||
11 | export type ActivitypubHttpFetcherPayload = { | 17 | export type ActivitypubHttpFetcherPayload = { |
12 | uri: string | 18 | uri: string |
13 | type: FetchType | 19 | type: FetchType |
14 | videoId?: number | 20 | videoId?: number |
21 | accountId?: number | ||
15 | } | 22 | } |
16 | 23 | ||
17 | async function processActivityPubHttpFetcher (job: Bull.Job) { | 24 | async function processActivityPubHttpFetcher (job: Bull.Job) { |
@@ -22,15 +29,26 @@ async function processActivityPubHttpFetcher (job: Bull.Job) { | |||
22 | let video: VideoModel | 29 | let video: VideoModel |
23 | if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) | 30 | if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) |
24 | 31 | ||
32 | let account: AccountModel | ||
33 | if (payload.accountId) account = await AccountModel.load(payload.accountId) | ||
34 | |||
25 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { | 35 | const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { |
26 | 'activity': items => processActivities(items, { outboxUrl: payload.uri }), | 36 | 'activity': items => processActivities(items, { outboxUrl: payload.uri }), |
27 | 'video-likes': items => createRates(items, video, 'like'), | 37 | 'video-likes': items => createRates(items, video, 'like'), |
28 | 'video-dislikes': items => createRates(items, video, 'dislike'), | 38 | 'video-dislikes': items => createRates(items, video, 'dislike'), |
29 | 'video-shares': items => addVideoShares(items, video), | 39 | 'video-shares': items => addVideoShares(items, video), |
30 | 'video-comments': items => addVideoComments(items, video) | 40 | 'video-comments': items => addVideoComments(items, video), |
41 | 'account-playlists': items => createAccountPlaylists(items, account) | ||
42 | } | ||
43 | |||
44 | const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Bluebird<any> } = { | ||
45 | 'video-likes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate), | ||
46 | 'video-dislikes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate), | ||
47 | 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate), | ||
48 | 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | ||
31 | } | 49 | } |
32 | 50 | ||
33 | return crawlCollectionPage(payload.uri, fetcherType[payload.type]) | 51 | return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type]) |
34 | } | 52 | } |
35 | 53 | ||
36 | // --------------------------------------------------------------------------- | 54 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 3973dcdc8..c70ce3be9 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts | |||
@@ -2,8 +2,8 @@ import * as Bull from 'bull' | |||
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { doRequest } from '../../../helpers/requests' | 3 | import { doRequest } from '../../../helpers/requests' |
4 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' | 4 | import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' |
5 | import { JOB_REQUEST_TIMEOUT } from '../../../initializers' | 5 | import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants' |
6 | import { ActorFollowScoreCache } from '../../cache' | 6 | import { ActorFollowScoreCache } from '../../files-cache' |
7 | 7 | ||
8 | export type ActivitypubHttpUnicastPayload = { | 8 | export type ActivitypubHttpUnicastPayload = { |
9 | uri: string | 9 | uri: string |
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 454b975fe..4d6c38cfa 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts | |||
@@ -1,11 +1,12 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { fetchVideoByUrl } from '../../../helpers/video' | 3 | import { fetchVideoByUrl } from '../../../helpers/video' |
4 | import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' | 4 | import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
6 | 7 | ||
7 | export type RefreshPayload = { | 8 | export type RefreshPayload = { |
8 | type: 'video' | 'actor' | 9 | type: 'video' | 'video-playlist' | 'actor' |
9 | url: string | 10 | url: string |
10 | } | 11 | } |
11 | 12 | ||
@@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) { | |||
15 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) | 16 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) |
16 | 17 | ||
17 | if (payload.type === 'video') return refreshVideo(payload.url) | 18 | if (payload.type === 'video') return refreshVideo(payload.url) |
19 | if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) | ||
18 | if (payload.type === 'actor') return refreshActor(payload.url) | 20 | if (payload.type === 'actor') return refreshActor(payload.url) |
19 | } | 21 | } |
20 | 22 | ||
21 | // --------------------------------------------------------------------------- | 23 | // --------------------------------------------------------------------------- |
22 | 24 | ||
23 | export { | 25 | export { |
24 | refreshActor, | ||
25 | refreshAPObject | 26 | refreshAPObject |
26 | } | 27 | } |
27 | 28 | ||
@@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) { | |||
50 | if (actor) { | 51 | if (actor) { |
51 | await refreshActorIfNeeded(actor, fetchType) | 52 | await refreshActorIfNeeded(actor, fetchType) |
52 | } | 53 | } |
54 | } | ||
55 | |||
56 | async function refreshVideoPlaylist (playlistUrl: string) { | ||
57 | const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) | ||
53 | 58 | ||
59 | if (playlist) { | ||
60 | await refreshVideoPlaylistIfNeeded(playlist) | ||
61 | } | ||
54 | } | 62 | } |
diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts index 2ba39a156..62701222c 100644 --- a/server/lib/job-queue/handlers/email.ts +++ b/server/lib/job-queue/handlers/email.ts | |||
@@ -1,15 +1,8 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { Emailer } from '../../emailer' | 3 | import { Emailer, SendEmailOptions } from '../../emailer' |
4 | 4 | ||
5 | export type EmailPayload = { | 5 | export type EmailPayload = SendEmailOptions |
6 | to: string[] | ||
7 | subject: string | ||
8 | text: string | ||
9 | |||
10 | fromDisplayName?: string | ||
11 | replyTo?: string | ||
12 | } | ||
13 | 6 | ||
14 | async function processEmail (job: Bull.Job) { | 7 | async function processEmail (job: Bull.Job) { |
15 | const payload = job.data as EmailPayload | 8 | const payload = job.data as EmailPayload |
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index 4961d4502..cdee1f6fd 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts | |||
@@ -2,7 +2,7 @@ import { buildSignedActivity } from '../../../../helpers/activitypub' | |||
2 | import { getServerActor } from '../../../../helpers/utils' | 2 | import { getServerActor } from '../../../../helpers/utils' |
3 | import { ActorModel } from '../../../../models/activitypub/actor' | 3 | import { ActorModel } from '../../../../models/activitypub/actor' |
4 | import { sha256 } from '../../../../helpers/core-utils' | 4 | import { sha256 } from '../../../../helpers/core-utils' |
5 | import { HTTP_SIGNATURE } from '../../../../initializers' | 5 | import { HTTP_SIGNATURE } from '../../../../initializers/constants' |
6 | 6 | ||
7 | type Payload = { body: any, signatureActorId?: number } | 7 | type Payload = { body: any, signatureActorId?: number } |
8 | 8 | ||
@@ -28,7 +28,7 @@ async function buildSignedRequestOptions (payload: Payload) { | |||
28 | actor = await getServerActor() | 28 | actor = await getServerActor() |
29 | } | 29 | } |
30 | 30 | ||
31 | const keyId = actor.getWebfingerUrl() | 31 | const keyId = actor.url |
32 | return { | 32 | return { |
33 | algorithm: HTTP_SIGNATURE.ALGORITHM, | 33 | algorithm: HTTP_SIGNATURE.ALGORITHM, |
34 | authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, | 34 | authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, |
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts new file mode 100644 index 000000000..921d9a083 --- /dev/null +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -0,0 +1,78 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { logger } from '../../../helpers/logger' | ||
3 | import { VideoModel } from '../../../models/video/video' | ||
4 | import { publishVideoIfNeeded } from './video-transcoding' | ||
5 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' | ||
6 | import { copy, stat } from 'fs-extra' | ||
7 | import { VideoFileModel } from '../../../models/video/video-file' | ||
8 | import { extname } from 'path' | ||
9 | |||
10 | export type VideoFileImportPayload = { | ||
11 | videoUUID: string, | ||
12 | filePath: string | ||
13 | } | ||
14 | |||
15 | async function processVideoFileImport (job: Bull.Job) { | ||
16 | const payload = job.data as VideoFileImportPayload | ||
17 | logger.info('Processing video file import in job %d.', job.id) | ||
18 | |||
19 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) | ||
20 | // No video, maybe deleted? | ||
21 | if (!video) { | ||
22 | logger.info('Do not process job %d, video does not exist.', job.id) | ||
23 | return undefined | ||
24 | } | ||
25 | |||
26 | await updateVideoFile(video, payload.filePath) | ||
27 | |||
28 | await publishVideoIfNeeded(video) | ||
29 | return video | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | |||
34 | export { | ||
35 | processVideoFileImport | ||
36 | } | ||
37 | |||
38 | // --------------------------------------------------------------------------- | ||
39 | |||
40 | async function updateVideoFile (video: VideoModel, inputFilePath: string) { | ||
41 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | ||
42 | const { size } = await stat(inputFilePath) | ||
43 | const fps = await getVideoFileFPS(inputFilePath) | ||
44 | |||
45 | let updatedVideoFile = new VideoFileModel({ | ||
46 | resolution: videoFileResolution, | ||
47 | extname: extname(inputFilePath), | ||
48 | size, | ||
49 | fps, | ||
50 | videoId: video.id | ||
51 | }) | ||
52 | |||
53 | const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) | ||
54 | |||
55 | if (currentVideoFile) { | ||
56 | // Remove old file and old torrent | ||
57 | await video.removeFile(currentVideoFile) | ||
58 | await video.removeTorrent(currentVideoFile) | ||
59 | // Remove the old video file from the array | ||
60 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) | ||
61 | |||
62 | // Update the database | ||
63 | currentVideoFile.set('extname', updatedVideoFile.extname) | ||
64 | currentVideoFile.set('size', updatedVideoFile.size) | ||
65 | currentVideoFile.set('fps', updatedVideoFile.fps) | ||
66 | |||
67 | updatedVideoFile = currentVideoFile | ||
68 | } | ||
69 | |||
70 | const outputPath = video.getVideoFilePath(updatedVideoFile) | ||
71 | await copy(inputFilePath, outputPath) | ||
72 | |||
73 | await video.createTorrentAndSetInfoHash(updatedVideoFile) | ||
74 | |||
75 | await updatedVideoFile.save() | ||
76 | |||
77 | video.VideoFiles.push(updatedVideoFile) | ||
78 | } | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 12004dcd7..1650916a6 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -6,16 +6,20 @@ import { VideoImportState } from '../../../../shared/models/videos' | |||
6 | import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' | 6 | import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' |
7 | import { extname, join } from 'path' | 7 | import { extname, join } from 'path' |
8 | import { VideoFileModel } from '../../../models/video/video-file' | 8 | import { VideoFileModel } from '../../../models/video/video-file' |
9 | import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' | 9 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' |
10 | import { downloadImage } from '../../../helpers/requests' | ||
11 | import { VideoState } from '../../../../shared' | 10 | import { VideoState } from '../../../../shared' |
12 | import { JobQueue } from '../index' | 11 | import { JobQueue } from '../index' |
13 | import { federateVideoIfNeeded } from '../../activitypub' | 12 | import { federateVideoIfNeeded } from '../../activitypub' |
14 | import { VideoModel } from '../../../models/video/video' | 13 | import { VideoModel } from '../../../models/video/video' |
15 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 14 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
16 | import { getSecureTorrentName } from '../../../helpers/utils' | 15 | import { getSecureTorrentName } from '../../../helpers/utils' |
17 | import { remove, move, stat } from 'fs-extra' | 16 | import { move, remove, stat } from 'fs-extra' |
18 | import { Notifier } from '../../notifier' | 17 | import { Notifier } from '../../notifier' |
18 | import { CONFIG } from '../../../initializers/config' | ||
19 | import { sequelizeTypescript } from '../../../initializers/database' | ||
20 | import { ThumbnailModel } from '../../../models/video/thumbnail' | ||
21 | import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumbnail' | ||
22 | import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' | ||
19 | 23 | ||
20 | type VideoImportYoutubeDLPayload = { | 24 | type VideoImportYoutubeDLPayload = { |
21 | type: 'youtube-dl' | 25 | type: 'youtube-dl' |
@@ -144,25 +148,19 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
144 | tempVideoPath = null // This path is not used anymore | 148 | tempVideoPath = null // This path is not used anymore |
145 | 149 | ||
146 | // Process thumbnail | 150 | // Process thumbnail |
147 | if (options.downloadThumbnail) { | 151 | let thumbnailModel: ThumbnailModel |
148 | if (options.thumbnailUrl) { | 152 | if (options.downloadThumbnail && options.thumbnailUrl) { |
149 | await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE) | 153 | thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.MINIATURE) |
150 | } else { | 154 | } else if (options.generateThumbnail || options.downloadThumbnail) { |
151 | await videoImport.Video.createThumbnail(videoFile) | 155 | thumbnailModel = await generateVideoMiniature(videoImport.Video, videoFile, ThumbnailType.MINIATURE) |
152 | } | ||
153 | } else if (options.generateThumbnail) { | ||
154 | await videoImport.Video.createThumbnail(videoFile) | ||
155 | } | 156 | } |
156 | 157 | ||
157 | // Process preview | 158 | // Process preview |
158 | if (options.downloadPreview) { | 159 | let previewModel: ThumbnailModel |
159 | if (options.thumbnailUrl) { | 160 | if (options.downloadPreview && options.thumbnailUrl) { |
160 | await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE) | 161 | previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.PREVIEW) |
161 | } else { | 162 | } else if (options.generatePreview || options.downloadPreview) { |
162 | await videoImport.Video.createPreview(videoFile) | 163 | previewModel = await generateVideoMiniature(videoImport.Video, videoFile, ThumbnailType.PREVIEW) |
163 | } | ||
164 | } else if (options.generatePreview) { | ||
165 | await videoImport.Video.createPreview(videoFile) | ||
166 | } | 164 | } |
167 | 165 | ||
168 | // Create torrent | 166 | // Create torrent |
@@ -182,6 +180,9 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
182 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED | 180 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED |
183 | await video.save({ transaction: t }) | 181 | await video.save({ transaction: t }) |
184 | 182 | ||
183 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) | ||
184 | if (previewModel) await video.addAndSaveThumbnail(previewModel, t) | ||
185 | |||
185 | // Now we can federate the video (reload from database, we need more attributes) | 186 | // Now we can federate the video (reload from database, we need more attributes) |
186 | const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | 187 | const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
187 | await federateVideoIfNeeded(videoForFederation, true, t) | 188 | await federateVideoIfNeeded(videoForFederation, true, t) |
@@ -196,9 +197,14 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
196 | return videoImportUpdated | 197 | return videoImportUpdated |
197 | }) | 198 | }) |
198 | 199 | ||
199 | Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) | ||
200 | Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true) | 200 | Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true) |
201 | 201 | ||
202 | if (videoImportUpdated.Video.VideoBlacklist) { | ||
203 | Notifier.Instance.notifyOnVideoAutoBlacklist(videoImportUpdated.Video) | ||
204 | } else { | ||
205 | Notifier.Instance.notifyOnNewVideo(videoImportUpdated.Video) | ||
206 | } | ||
207 | |||
202 | // Create transcoding jobs? | 208 | // Create transcoding jobs? |
203 | if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { | 209 | if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { |
204 | // Put uuid because we don't have id auto incremented for now | 210 | // Put uuid because we don't have id auto incremented for now |
@@ -207,7 +213,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
207 | isNewVideo: true | 213 | isNewVideo: true |
208 | } | 214 | } |
209 | 215 | ||
210 | await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) | 216 | await JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }) |
211 | } | 217 | } |
212 | 218 | ||
213 | } catch (err) { | 219 | } catch (err) { |
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 593e43cc5..48cac517e 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -8,40 +8,20 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
8 | import { sequelizeTypescript } from '../../../initializers' | 8 | import { sequelizeTypescript } from '../../../initializers' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' | 10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' |
11 | import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' | 11 | import { generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' |
12 | import { Notifier } from '../../notifier' | 12 | import { Notifier } from '../../notifier' |
13 | import { CONFIG } from '../../../initializers/config' | ||
13 | 14 | ||
14 | export type VideoFilePayload = { | 15 | export type VideoTranscodingPayload = { |
15 | videoUUID: string | 16 | videoUUID: string |
16 | isNewVideo?: boolean | ||
17 | resolution?: VideoResolution | 17 | resolution?: VideoResolution |
18 | isNewVideo?: boolean | ||
18 | isPortraitMode?: boolean | 19 | isPortraitMode?: boolean |
20 | generateHlsPlaylist?: boolean | ||
19 | } | 21 | } |
20 | 22 | ||
21 | export type VideoFileImportPayload = { | 23 | async function processVideoTranscoding (job: Bull.Job) { |
22 | videoUUID: string, | 24 | const payload = job.data as VideoTranscodingPayload |
23 | filePath: string | ||
24 | } | ||
25 | |||
26 | async function processVideoFileImport (job: Bull.Job) { | ||
27 | const payload = job.data as VideoFileImportPayload | ||
28 | logger.info('Processing video file import in job %d.', job.id) | ||
29 | |||
30 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) | ||
31 | // No video, maybe deleted? | ||
32 | if (!video) { | ||
33 | logger.info('Do not process job %d, video does not exist.', job.id) | ||
34 | return undefined | ||
35 | } | ||
36 | |||
37 | await importVideoFile(video, payload.filePath) | ||
38 | |||
39 | await onVideoFileTranscoderOrImportSuccess(video) | ||
40 | return video | ||
41 | } | ||
42 | |||
43 | async function processVideoFile (job: Bull.Job) { | ||
44 | const payload = job.data as VideoFilePayload | ||
45 | logger.info('Processing video file in job %d.', job.id) | 25 | logger.info('Processing video file in job %d.', job.id) |
46 | 26 | ||
47 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) | 27 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) |
@@ -51,23 +31,38 @@ async function processVideoFile (job: Bull.Job) { | |||
51 | return undefined | 31 | return undefined |
52 | } | 32 | } |
53 | 33 | ||
54 | // Transcoding in other resolution | 34 | if (payload.generateHlsPlaylist) { |
55 | if (payload.resolution) { | 35 | await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) |
36 | |||
37 | await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) | ||
38 | } else if (payload.resolution) { // Transcoding in other resolution | ||
56 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) | 39 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) |
57 | 40 | ||
58 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) | 41 | await retryTransactionWrapper(publishVideoIfNeeded, video, payload) |
59 | } else { | 42 | } else { |
60 | await optimizeVideofile(video) | 43 | await optimizeVideofile(video) |
61 | 44 | ||
62 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) | 45 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) |
63 | } | 46 | } |
64 | 47 | ||
65 | return video | 48 | return video |
66 | } | 49 | } |
67 | 50 | ||
68 | async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | 51 | async function onHlsPlaylistGenerationSuccess (video: VideoModel) { |
69 | if (video === undefined) return undefined | 52 | if (video === undefined) return undefined |
70 | 53 | ||
54 | await sequelizeTypescript.transaction(async t => { | ||
55 | // Maybe the video changed in database, refresh it | ||
56 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
57 | // Video does not exist anymore | ||
58 | if (!videoDatabase) return undefined | ||
59 | |||
60 | // If the video was not published, we consider it is a new one for other instances | ||
61 | await federateVideoIfNeeded(videoDatabase, false, t) | ||
62 | }) | ||
63 | } | ||
64 | |||
65 | async function publishVideoIfNeeded (video: VideoModel, payload?: VideoTranscodingPayload) { | ||
71 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { | 66 | const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { |
72 | // Maybe the video changed in database, refresh it | 67 | // Maybe the video changed in database, refresh it |
73 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | 68 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
@@ -93,11 +88,13 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | |||
93 | 88 | ||
94 | if (videoPublished) { | 89 | if (videoPublished) { |
95 | Notifier.Instance.notifyOnNewVideo(videoDatabase) | 90 | Notifier.Instance.notifyOnNewVideo(videoDatabase) |
96 | Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 91 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) |
97 | } | 92 | } |
93 | |||
94 | await createHlsJobIfEnabled(payload) | ||
98 | } | 95 | } |
99 | 96 | ||
100 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { | 97 | async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoTranscodingPayload) { |
101 | if (videoArg === undefined) return undefined | 98 | if (videoArg === undefined) return undefined |
102 | 99 | ||
103 | // Outside the transaction (IO on disk) | 100 | // Outside the transaction (IO on disk) |
@@ -119,7 +116,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
119 | let videoPublished = false | 116 | let videoPublished = false |
120 | 117 | ||
121 | if (resolutionsEnabled.length !== 0) { | 118 | if (resolutionsEnabled.length !== 0) { |
122 | const tasks: Bluebird<Bull.Job<any>>[] = [] | 119 | const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = [] |
123 | 120 | ||
124 | for (const resolution of resolutionsEnabled) { | 121 | for (const resolution of resolutionsEnabled) { |
125 | const dataInput = { | 122 | const dataInput = { |
@@ -127,7 +124,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
127 | resolution | 124 | resolution |
128 | } | 125 | } |
129 | 126 | ||
130 | const p = JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) | 127 | const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }) |
131 | tasks.push(p) | 128 | tasks.push(p) |
132 | } | 129 | } |
133 | 130 | ||
@@ -144,18 +141,37 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo | |||
144 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) | 141 | logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) |
145 | } | 142 | } |
146 | 143 | ||
147 | await federateVideoIfNeeded(videoDatabase, isNewVideo, t) | 144 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) |
148 | 145 | ||
149 | return { videoDatabase, videoPublished } | 146 | return { videoDatabase, videoPublished } |
150 | }) | 147 | }) |
151 | 148 | ||
152 | if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) | 149 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) |
153 | if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) | 150 | if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) |
151 | |||
152 | await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })) | ||
154 | } | 153 | } |
155 | 154 | ||
156 | // --------------------------------------------------------------------------- | 155 | // --------------------------------------------------------------------------- |
157 | 156 | ||
158 | export { | 157 | export { |
159 | processVideoFile, | 158 | processVideoTranscoding, |
160 | processVideoFileImport | 159 | publishVideoIfNeeded |
160 | } | ||
161 | |||
162 | // --------------------------------------------------------------------------- | ||
163 | |||
164 | function createHlsJobIfEnabled (payload?: VideoTranscodingPayload) { | ||
165 | // Generate HLS playlist? | ||
166 | if (payload && CONFIG.TRANSCODING.HLS.ENABLED) { | ||
167 | const hlsTranscodingPayload = { | ||
168 | videoUUID: payload.videoUUID, | ||
169 | resolution: payload.resolution, | ||
170 | isPortraitMode: payload.isPortraitMode, | ||
171 | |||
172 | generateHlsPlaylist: true | ||
173 | } | ||
174 | |||
175 | return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }) | ||
176 | } | ||
161 | } | 177 | } |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ba9cbe0d9..3c810da98 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -2,16 +2,17 @@ import * as Bull from 'bull' | |||
2 | import { JobState, JobType } from '../../../shared/models' | 2 | import { JobState, JobType } from '../../../shared/models' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { Redis } from '../redis' | 4 | import { Redis } from '../redis' |
5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS } from '../../initializers' | 5 | import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' |
6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' | 6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | 7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' |
8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | 8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' |
9 | import { EmailPayload, processEmail } from './handlers/email' | 9 | import { EmailPayload, processEmail } from './handlers/email' |
10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' | 10 | import { processVideoTranscoding, VideoTranscodingPayload } from './handlers/video-transcoding' |
11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' | 11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' | 12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' |
13 | import { processVideosViews } from './handlers/video-views' | 13 | import { processVideosViews } from './handlers/video-views' |
14 | import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' | 14 | import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' |
15 | import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' | ||
15 | 16 | ||
16 | type CreateJobArgument = | 17 | type CreateJobArgument = |
17 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 18 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -19,19 +20,20 @@ type CreateJobArgument = | |||
19 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | 20 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | |
20 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | 21 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
21 | { type: 'video-file-import', payload: VideoFileImportPayload } | | 22 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
22 | { type: 'video-file', payload: VideoFilePayload } | | 23 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | |
23 | { type: 'email', payload: EmailPayload } | | 24 | { type: 'email', payload: EmailPayload } | |
24 | { type: 'video-import', payload: VideoImportPayload } | | 25 | { type: 'video-import', payload: VideoImportPayload } | |
25 | { type: 'activitypub-refresher', payload: RefreshPayload } | | 26 | { type: 'activitypub-refresher', payload: RefreshPayload } | |
26 | { type: 'videos-views', payload: {} } | 27 | { type: 'videos-views', payload: {} } |
27 | 28 | ||
28 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | 29 | const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = { |
29 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, | 30 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
30 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | 31 | 'activitypub-http-unicast': processActivityPubHttpUnicast, |
31 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | 32 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, |
32 | 'activitypub-follow': processActivityPubFollow, | 33 | 'activitypub-follow': processActivityPubFollow, |
33 | 'video-file-import': processVideoFileImport, | 34 | 'video-file-import': processVideoFileImport, |
34 | 'video-file': processVideoFile, | 35 | 'video-transcoding': processVideoTranscoding, |
36 | 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3) | ||
35 | 'email': processEmail, | 37 | 'email': processEmail, |
36 | 'video-import': processVideoImport, | 38 | 'video-import': processVideoImport, |
37 | 'videos-views': processVideosViews, | 39 | 'videos-views': processVideosViews, |
@@ -44,7 +46,7 @@ const jobTypes: JobType[] = [ | |||
44 | 'activitypub-http-fetcher', | 46 | 'activitypub-http-fetcher', |
45 | 'activitypub-http-unicast', | 47 | 'activitypub-http-unicast', |
46 | 'email', | 48 | 'email', |
47 | 'video-file', | 49 | 'video-transcoding', |
48 | 'video-file-import', | 50 | 'video-file-import', |
49 | 'video-import', | 51 | 'video-import', |
50 | 'videos-views', | 52 | 'videos-views', |
@@ -66,10 +68,10 @@ class JobQueue { | |||
66 | if (this.initialized === true) return | 68 | if (this.initialized === true) return |
67 | this.initialized = true | 69 | this.initialized = true |
68 | 70 | ||
69 | this.jobRedisPrefix = 'bull-' + CONFIG.WEBSERVER.HOST | 71 | this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST |
70 | const queueOptions = { | 72 | const queueOptions = { |
71 | prefix: this.jobRedisPrefix, | 73 | prefix: this.jobRedisPrefix, |
72 | redis: Redis.getRedisClient(), | 74 | redis: Redis.getRedisClientOptions(), |
73 | settings: { | 75 | settings: { |
74 | maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts | 76 | maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts |
75 | } | 77 | } |
diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts index 2fa320cd7..c1e63fa8f 100644 --- a/server/lib/notifier.ts +++ b/server/lib/notifier.ts | |||
@@ -6,7 +6,7 @@ import { UserNotificationModel } from '../models/account/user-notification' | |||
6 | import { VideoCommentModel } from '../models/video/video-comment' | 6 | import { VideoCommentModel } from '../models/video/video-comment' |
7 | import { UserModel } from '../models/account/user' | 7 | import { UserModel } from '../models/account/user' |
8 | import { PeerTubeSocket } from './peertube-socket' | 8 | import { PeerTubeSocket } from './peertube-socket' |
9 | import { CONFIG } from '../initializers/constants' | 9 | import { CONFIG } from '../initializers/config' |
10 | import { VideoPrivacy, VideoState } from '../../shared/models/videos' | 10 | import { VideoPrivacy, VideoState } from '../../shared/models/videos' |
11 | import { VideoAbuseModel } from '../models/video/video-abuse' | 11 | import { VideoAbuseModel } from '../models/video/video-abuse' |
12 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | 12 | import { VideoBlacklistModel } from '../models/video/video-blacklist' |
@@ -23,19 +23,35 @@ class Notifier { | |||
23 | private constructor () {} | 23 | private constructor () {} |
24 | 24 | ||
25 | notifyOnNewVideo (video: VideoModel): void { | 25 | notifyOnNewVideo (video: VideoModel): void { |
26 | // Only notify on public and published videos | 26 | // Only notify on public and published videos which are not blacklisted |
27 | if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED) return | 27 | if (video.privacy !== VideoPrivacy.PUBLIC || video.state !== VideoState.PUBLISHED || video.VideoBlacklist) return |
28 | 28 | ||
29 | this.notifySubscribersOfNewVideo(video) | 29 | this.notifySubscribersOfNewVideo(video) |
30 | .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) | 30 | .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) |
31 | } | 31 | } |
32 | 32 | ||
33 | notifyOnPendingVideoPublished (video: VideoModel): void { | 33 | notifyOnVideoPublishedAfterTranscoding (video: VideoModel): void { |
34 | // Only notify on public videos that has been published while the user waited transcoding/scheduled update | 34 | // don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update |
35 | if (video.waitTranscoding === false && !video.ScheduleVideoUpdate) return | 35 | if (!video.waitTranscoding || video.VideoBlacklist || video.ScheduleVideoUpdate) return |
36 | 36 | ||
37 | this.notifyOwnedVideoHasBeenPublished(video) | 37 | this.notifyOwnedVideoHasBeenPublished(video) |
38 | .catch(err => logger.error('Cannot notify owner that its video %s has been published.', video.url, { err })) | 38 | .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err })) |
39 | } | ||
40 | |||
41 | notifyOnVideoPublishedAfterScheduledUpdate (video: VideoModel): void { | ||
42 | // don't notify if video is still blacklisted or waiting for transcoding | ||
43 | if (video.VideoBlacklist || (video.waitTranscoding && video.state !== VideoState.PUBLISHED)) return | ||
44 | |||
45 | this.notifyOwnedVideoHasBeenPublished(video) | ||
46 | .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err })) | ||
47 | } | ||
48 | |||
49 | notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: VideoModel): void { | ||
50 | // don't notify if video is still waiting for transcoding or scheduled update | ||
51 | if (video.ScheduleVideoUpdate || (video.waitTranscoding && video.state !== VideoState.PUBLISHED)) return | ||
52 | |||
53 | this.notifyOwnedVideoHasBeenPublished(video) | ||
54 | .catch(err => logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err })) // tslint:disable-line:max-line-length | ||
39 | } | 55 | } |
40 | 56 | ||
41 | notifyOnNewComment (comment: VideoCommentModel): void { | 57 | notifyOnNewComment (comment: VideoCommentModel): void { |
@@ -51,6 +67,11 @@ class Notifier { | |||
51 | .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err })) | 67 | .catch(err => logger.error('Cannot notify of new video abuse of video %s.', videoAbuse.Video.url, { err })) |
52 | } | 68 | } |
53 | 69 | ||
70 | notifyOnVideoAutoBlacklist (video: VideoModel): void { | ||
71 | this.notifyModeratorsOfVideoAutoBlacklist(video) | ||
72 | .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', video.url, { err })) | ||
73 | } | ||
74 | |||
54 | notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void { | 75 | notifyOnVideoBlacklist (videoBlacklist: VideoBlacklistModel): void { |
55 | this.notifyVideoOwnerOfBlacklist(videoBlacklist) | 76 | this.notifyVideoOwnerOfBlacklist(videoBlacklist) |
56 | .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) | 77 | .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) |
@@ -58,7 +79,7 @@ class Notifier { | |||
58 | 79 | ||
59 | notifyOnVideoUnblacklist (video: VideoModel): void { | 80 | notifyOnVideoUnblacklist (video: VideoModel): void { |
60 | this.notifyVideoOwnerOfUnblacklist(video) | 81 | this.notifyVideoOwnerOfUnblacklist(video) |
61 | .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', video.url, { err })) | 82 | .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err })) |
62 | } | 83 | } |
63 | 84 | ||
64 | notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void { | 85 | notifyOnFinishedVideoImport (videoImport: VideoImportModel, success: boolean): void { |
@@ -71,18 +92,25 @@ class Notifier { | |||
71 | .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) | 92 | .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) |
72 | } | 93 | } |
73 | 94 | ||
74 | notifyOfNewFollow (actorFollow: ActorFollowModel): void { | 95 | notifyOfNewUserFollow (actorFollow: ActorFollowModel): void { |
75 | this.notifyUserOfNewActorFollow(actorFollow) | 96 | this.notifyUserOfNewActorFollow(actorFollow) |
76 | .catch(err => { | 97 | .catch(err => { |
77 | logger.error( | 98 | logger.error( |
78 | 'Cannot notify owner of channel %s of a new follow by %s.', | 99 | 'Cannot notify owner of channel %s of a new follow by %s.', |
79 | actorFollow.ActorFollowing.VideoChannel.getDisplayName(), | 100 | actorFollow.ActorFollowing.VideoChannel.getDisplayName(), |
80 | actorFollow.ActorFollower.Account.getDisplayName(), | 101 | actorFollow.ActorFollower.Account.getDisplayName(), |
81 | err | 102 | { err } |
82 | ) | 103 | ) |
83 | }) | 104 | }) |
84 | } | 105 | } |
85 | 106 | ||
107 | notifyOfNewInstanceFollow (actorFollow: ActorFollowModel): void { | ||
108 | this.notifyAdminsOfNewInstanceFollow(actorFollow) | ||
109 | .catch(err => { | ||
110 | logger.error('Cannot notify administrators of new follower %s.', actorFollow.ActorFollower.url, { err }) | ||
111 | }) | ||
112 | } | ||
113 | |||
86 | private async notifySubscribersOfNewVideo (video: VideoModel) { | 114 | private async notifySubscribersOfNewVideo (video: VideoModel) { |
87 | // List all followers that are users | 115 | // List all followers that are users |
88 | const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) | 116 | const users = await UserModel.listUserSubscribersOf(video.VideoChannel.actorId) |
@@ -147,10 +175,13 @@ class Notifier { | |||
147 | } | 175 | } |
148 | 176 | ||
149 | private async notifyOfCommentMention (comment: VideoCommentModel) { | 177 | private async notifyOfCommentMention (comment: VideoCommentModel) { |
150 | const usernames = comment.extractMentions() | 178 | const extractedUsernames = comment.extractMentions() |
151 | logger.debug('Extracted %d username from comment %s.', usernames.length, comment.url, { usernames, text: comment.text }) | 179 | logger.debug( |
180 | 'Extracted %d username from comment %s.', extractedUsernames.length, comment.url, | ||
181 | { usernames: extractedUsernames, text: comment.text } | ||
182 | ) | ||
152 | 183 | ||
153 | let users = await UserModel.listByUsernames(usernames) | 184 | let users = await UserModel.listByUsernames(extractedUsernames) |
154 | 185 | ||
155 | if (comment.Video.isOwned()) { | 186 | if (comment.Video.isOwned()) { |
156 | const userException = await UserModel.loadByVideoId(comment.videoId) | 187 | const userException = await UserModel.loadByVideoId(comment.videoId) |
@@ -237,6 +268,33 @@ class Notifier { | |||
237 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) | 268 | return this.notify({ users: [ user ], settingGetter, notificationCreator, emailSender }) |
238 | } | 269 | } |
239 | 270 | ||
271 | private async notifyAdminsOfNewInstanceFollow (actorFollow: ActorFollowModel) { | ||
272 | const admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) | ||
273 | |||
274 | logger.info('Notifying %d administrators of new instance follower: %s.', admins.length, actorFollow.ActorFollower.url) | ||
275 | |||
276 | function settingGetter (user: UserModel) { | ||
277 | return user.NotificationSetting.newInstanceFollower | ||
278 | } | ||
279 | |||
280 | async function notificationCreator (user: UserModel) { | ||
281 | const notification = await UserNotificationModel.create({ | ||
282 | type: UserNotificationType.NEW_INSTANCE_FOLLOWER, | ||
283 | userId: user.id, | ||
284 | actorFollowId: actorFollow.id | ||
285 | }) | ||
286 | notification.ActorFollow = actorFollow | ||
287 | |||
288 | return notification | ||
289 | } | ||
290 | |||
291 | function emailSender (emails: string[]) { | ||
292 | return Emailer.Instance.addNewInstanceFollowerNotification(emails, actorFollow) | ||
293 | } | ||
294 | |||
295 | return this.notify({ users: admins, settingGetter, notificationCreator, emailSender }) | ||
296 | } | ||
297 | |||
240 | private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { | 298 | private async notifyModeratorsOfNewVideoAbuse (videoAbuse: VideoAbuseModel) { |
241 | const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) | 299 | const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_ABUSES) |
242 | if (moderators.length === 0) return | 300 | if (moderators.length === 0) return |
@@ -265,6 +323,34 @@ class Notifier { | |||
265 | return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) | 323 | return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) |
266 | } | 324 | } |
267 | 325 | ||
326 | private async notifyModeratorsOfVideoAutoBlacklist (video: VideoModel) { | ||
327 | const moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST) | ||
328 | if (moderators.length === 0) return | ||
329 | |||
330 | logger.info('Notifying %s moderators of video auto-blacklist %s.', moderators.length, video.url) | ||
331 | |||
332 | function settingGetter (user: UserModel) { | ||
333 | return user.NotificationSetting.videoAutoBlacklistAsModerator | ||
334 | } | ||
335 | async function notificationCreator (user: UserModel) { | ||
336 | |||
337 | const notification = await UserNotificationModel.create({ | ||
338 | type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS, | ||
339 | userId: user.id, | ||
340 | videoId: video.id | ||
341 | }) | ||
342 | notification.Video = video | ||
343 | |||
344 | return notification | ||
345 | } | ||
346 | |||
347 | function emailSender (emails: string[]) { | ||
348 | return Emailer.Instance.addVideoAutoBlacklistModeratorsNotification(emails, video) | ||
349 | } | ||
350 | |||
351 | return this.notify({ users: moderators, settingGetter, notificationCreator, emailSender }) | ||
352 | } | ||
353 | |||
268 | private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { | 354 | private async notifyVideoOwnerOfBlacklist (videoBlacklist: VideoBlacklistModel) { |
269 | const user = await UserModel.loadByVideoId(videoBlacklist.videoId) | 355 | const user = await UserModel.loadByVideoId(videoBlacklist.videoId) |
270 | if (!user) return | 356 | if (!user) return |
@@ -436,7 +522,7 @@ class Notifier { | |||
436 | } | 522 | } |
437 | 523 | ||
438 | private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) { | 524 | private isEmailEnabled (user: UserModel, value: UserNotificationSettingValue) { |
439 | if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified !== true) return false | 525 | if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified === false) return false |
440 | 526 | ||
441 | return value & UserNotificationSettingValue.EMAIL | 527 | return value & UserNotificationSettingValue.EMAIL |
442 | } | 528 | } |
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 2cd2ae97c..45ac3e7c4 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts | |||
@@ -4,12 +4,13 @@ import { logger } from '../helpers/logger' | |||
4 | import { UserModel } from '../models/account/user' | 4 | import { UserModel } from '../models/account/user' |
5 | import { OAuthClientModel } from '../models/oauth/oauth-client' | 5 | import { OAuthClientModel } from '../models/oauth/oauth-client' |
6 | import { OAuthTokenModel } from '../models/oauth/oauth-token' | 6 | import { OAuthTokenModel } from '../models/oauth/oauth-token' |
7 | import { CONFIG } from '../initializers/constants' | 7 | import { CACHE } from '../initializers/constants' |
8 | import { Transaction } from 'sequelize' | 8 | import { Transaction } from 'sequelize' |
9 | import { CONFIG } from '../initializers/config' | ||
9 | 10 | ||
10 | type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } | 11 | type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } |
11 | const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} | 12 | let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} |
12 | const userHavingToken: { [ userId: number ]: string } = {} | 13 | let userHavingToken: { [ userId: number ]: string } = {} |
13 | 14 | ||
14 | // --------------------------------------------------------------------------- | 15 | // --------------------------------------------------------------------------- |
15 | 16 | ||
@@ -38,11 +39,19 @@ function clearCacheByToken (token: string) { | |||
38 | function getAccessToken (bearerToken: string) { | 39 | function getAccessToken (bearerToken: string) { |
39 | logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') | 40 | logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') |
40 | 41 | ||
42 | if (!bearerToken) return Bluebird.resolve(undefined) | ||
43 | |||
41 | if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) | 44 | if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) |
42 | 45 | ||
43 | return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) | 46 | return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) |
44 | .then(tokenModel => { | 47 | .then(tokenModel => { |
45 | if (tokenModel) { | 48 | if (tokenModel) { |
49 | // Reinit our cache | ||
50 | if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) { | ||
51 | accessTokenCache = {} | ||
52 | userHavingToken = {} | ||
53 | } | ||
54 | |||
46 | accessTokenCache[ bearerToken ] = tokenModel | 55 | accessTokenCache[ bearerToken ] = tokenModel |
47 | userHavingToken[ tokenModel.userId ] = tokenModel.accessToken | 56 | userHavingToken[ tokenModel.userId ] = tokenModel.accessToken |
48 | } | 57 | } |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 3628c0583..f77d0b62c 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -3,12 +3,13 @@ import { createClient, RedisClient } from 'redis' | |||
3 | import { logger } from '../helpers/logger' | 3 | import { logger } from '../helpers/logger' |
4 | import { generateRandomString } from '../helpers/utils' | 4 | import { generateRandomString } from '../helpers/utils' |
5 | import { | 5 | import { |
6 | CONFIG, | ||
7 | CONTACT_FORM_LIFETIME, | 6 | CONTACT_FORM_LIFETIME, |
8 | USER_EMAIL_VERIFY_LIFETIME, | 7 | USER_EMAIL_VERIFY_LIFETIME, |
9 | USER_PASSWORD_RESET_LIFETIME, | 8 | USER_PASSWORD_RESET_LIFETIME, |
10 | VIDEO_VIEW_LIFETIME | 9 | VIDEO_VIEW_LIFETIME, |
11 | } from '../initializers' | 10 | WEBSERVER |
11 | } from '../initializers/constants' | ||
12 | import { CONFIG } from '../initializers/config' | ||
12 | 13 | ||
13 | type CachedRoute = { | 14 | type CachedRoute = { |
14 | body: string, | 15 | body: string, |
@@ -30,7 +31,7 @@ class Redis { | |||
30 | if (this.initialized === true) return | 31 | if (this.initialized === true) return |
31 | this.initialized = true | 32 | this.initialized = true |
32 | 33 | ||
33 | this.client = createClient(Redis.getRedisClient()) | 34 | this.client = createClient(Redis.getRedisClientOptions()) |
34 | 35 | ||
35 | this.client.on('error', err => { | 36 | this.client.on('error', err => { |
36 | logger.error('Error in Redis client.', { err }) | 37 | logger.error('Error in Redis client.', { err }) |
@@ -41,10 +42,10 @@ class Redis { | |||
41 | this.client.auth(CONFIG.REDIS.AUTH) | 42 | this.client.auth(CONFIG.REDIS.AUTH) |
42 | } | 43 | } |
43 | 44 | ||
44 | this.prefix = 'redis-' + CONFIG.WEBSERVER.HOST + '-' | 45 | this.prefix = 'redis-' + WEBSERVER.HOST + '-' |
45 | } | 46 | } |
46 | 47 | ||
47 | static getRedisClient () { | 48 | static getRedisClientOptions () { |
48 | return Object.assign({}, | 49 | return Object.assign({}, |
49 | (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, | 50 | (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, |
50 | (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, | 51 | (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, |
@@ -54,6 +55,14 @@ class Redis { | |||
54 | ) | 55 | ) |
55 | } | 56 | } |
56 | 57 | ||
58 | getClient () { | ||
59 | return this.client | ||
60 | } | ||
61 | |||
62 | getPrefix () { | ||
63 | return this.prefix | ||
64 | } | ||
65 | |||
57 | /************* Forgot password *************/ | 66 | /************* Forgot password *************/ |
58 | 67 | ||
59 | async setResetPasswordVerificationString (userId: number) { | 68 | async setResetPasswordVerificationString (userId: number) { |
@@ -88,7 +97,7 @@ class Redis { | |||
88 | return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) | 97 | return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) |
89 | } | 98 | } |
90 | 99 | ||
91 | async isContactFormIpExists (ip: string) { | 100 | async doesContactFormIpExist (ip: string) { |
92 | return this.exists(this.generateContactFormKey(ip)) | 101 | return this.exists(this.generateContactFormKey(ip)) |
93 | } | 102 | } |
94 | 103 | ||
@@ -98,7 +107,7 @@ class Redis { | |||
98 | return this.setValue(this.generateViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME) | 107 | return this.setValue(this.generateViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME) |
99 | } | 108 | } |
100 | 109 | ||
101 | async isVideoIPViewExists (ip: string, videoUUID: string) { | 110 | async doesVideoIPViewExist (ip: string, videoUUID: string) { |
102 | return this.exists(this.generateViewKey(ip, videoUUID)) | 111 | return this.exists(this.generateViewKey(ip, videoUUID)) |
103 | } | 112 | } |
104 | 113 | ||
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts index 86ea7aa38..0e6088911 100644 --- a/server/lib/schedulers/abstract-scheduler.ts +++ b/server/lib/schedulers/abstract-scheduler.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | import { logger } from '../../helpers/logger' | 1 | import { logger } from '../../helpers/logger' |
2 | import * as Bluebird from 'bluebird' | ||
2 | 3 | ||
3 | export abstract class AbstractScheduler { | 4 | export abstract class AbstractScheduler { |
4 | 5 | ||
@@ -30,5 +31,5 @@ export abstract class AbstractScheduler { | |||
30 | } | 31 | } |
31 | } | 32 | } |
32 | 33 | ||
33 | protected abstract internalExecute (): Promise<any> | 34 | protected abstract internalExecute (): Promise<any> | Bluebird<any> |
34 | } | 35 | } |
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts index 3967be7f8..fdd3ad5fa 100644 --- a/server/lib/schedulers/actor-follow-scheduler.ts +++ b/server/lib/schedulers/actor-follow-scheduler.ts | |||
@@ -2,8 +2,8 @@ import { isTestInstance } from '../../helpers/core-utils' | |||
2 | import { logger } from '../../helpers/logger' | 2 | import { logger } from '../../helpers/logger' |
3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../models/activitypub/actor-follow' |
4 | import { AbstractScheduler } from './abstract-scheduler' | 4 | import { AbstractScheduler } from './abstract-scheduler' |
5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' | 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
6 | import { ActorFollowScoreCache } from '../cache' | 6 | import { ActorFollowScoreCache } from '../files-cache' |
7 | 7 | ||
8 | export class ActorFollowScheduler extends AbstractScheduler { | 8 | export class ActorFollowScheduler extends AbstractScheduler { |
9 | 9 | ||
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts new file mode 100644 index 000000000..1b5ff8394 --- /dev/null +++ b/server/lib/schedulers/remove-old-history-scheduler.ts | |||
@@ -0,0 +1,32 @@ | |||
1 | import { logger } from '../../helpers/logger' | ||
2 | import { AbstractScheduler } from './abstract-scheduler' | ||
3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
4 | import { UserVideoHistoryModel } from '../../models/account/user-video-history' | ||
5 | import { CONFIG } from '../../initializers/config' | ||
6 | import { isTestInstance } from '../../helpers/core-utils' | ||
7 | |||
8 | export class RemoveOldHistoryScheduler extends AbstractScheduler { | ||
9 | |||
10 | private static instance: AbstractScheduler | ||
11 | |||
12 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory | ||
13 | |||
14 | private constructor () { | ||
15 | super() | ||
16 | } | ||
17 | |||
18 | protected internalExecute () { | ||
19 | if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return | ||
20 | |||
21 | logger.info('Removing old videos history.') | ||
22 | |||
23 | const now = new Date() | ||
24 | const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString() | ||
25 | |||
26 | return UserVideoHistoryModel.removeOldHistory(beforeDate) | ||
27 | } | ||
28 | |||
29 | static get Instance () { | ||
30 | return this.instance || (this.instance = new this()) | ||
31 | } | ||
32 | } | ||
diff --git a/server/lib/schedulers/remove-old-jobs-scheduler.ts b/server/lib/schedulers/remove-old-jobs-scheduler.ts index 4a4341ba9..0179a7618 100644 --- a/server/lib/schedulers/remove-old-jobs-scheduler.ts +++ b/server/lib/schedulers/remove-old-jobs-scheduler.ts | |||
@@ -2,7 +2,7 @@ import { isTestInstance } from '../../helpers/core-utils' | |||
2 | import { logger } from '../../helpers/logger' | 2 | import { logger } from '../../helpers/logger' |
3 | import { JobQueue } from '../job-queue' | 3 | import { JobQueue } from '../job-queue' |
4 | import { AbstractScheduler } from './abstract-scheduler' | 4 | import { AbstractScheduler } from './abstract-scheduler' |
5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' | 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
6 | 6 | ||
7 | export class RemoveOldJobsScheduler extends AbstractScheduler { | 7 | export class RemoveOldJobsScheduler extends AbstractScheduler { |
8 | 8 | ||
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts new file mode 100644 index 000000000..39fbb9163 --- /dev/null +++ b/server/lib/schedulers/remove-old-views-scheduler.ts | |||
@@ -0,0 +1,33 @@ | |||
1 | import { logger } from '../../helpers/logger' | ||
2 | import { AbstractScheduler } from './abstract-scheduler' | ||
3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
4 | import { UserVideoHistoryModel } from '../../models/account/user-video-history' | ||
5 | import { CONFIG } from '../../initializers/config' | ||
6 | import { isTestInstance } from '../../helpers/core-utils' | ||
7 | import { VideoViewModel } from '../../models/video/video-views' | ||
8 | |||
9 | export class RemoveOldViewsScheduler extends AbstractScheduler { | ||
10 | |||
11 | private static instance: AbstractScheduler | ||
12 | |||
13 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldViews | ||
14 | |||
15 | private constructor () { | ||
16 | super() | ||
17 | } | ||
18 | |||
19 | protected internalExecute () { | ||
20 | if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return | ||
21 | |||
22 | logger.info('Removing old videos views.') | ||
23 | |||
24 | const now = new Date() | ||
25 | const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString() | ||
26 | |||
27 | return VideoViewModel.removeOldRemoteViewsHistory(beforeDate) | ||
28 | } | ||
29 | |||
30 | static get Instance () { | ||
31 | return this.instance || (this.instance = new this()) | ||
32 | } | ||
33 | } | ||
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 2618a5857..80080a132 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts | |||
@@ -3,10 +3,11 @@ import { AbstractScheduler } from './abstract-scheduler' | |||
3 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' | 3 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' |
4 | import { retryTransactionWrapper } from '../../helpers/database-utils' | 4 | import { retryTransactionWrapper } from '../../helpers/database-utils' |
5 | import { federateVideoIfNeeded } from '../activitypub' | 5 | import { federateVideoIfNeeded } from '../activitypub' |
6 | import { SCHEDULER_INTERVALS_MS, sequelizeTypescript } from '../../initializers' | 6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | 7 | import { VideoPrivacy } from '../../../shared/models/videos' |
8 | import { Notifier } from '../notifier' | 8 | import { Notifier } from '../notifier' |
9 | import { VideoModel } from '../../models/video/video' | 9 | import { VideoModel } from '../../models/video/video' |
10 | import { sequelizeTypescript } from '../../initializers/database' | ||
10 | 11 | ||
11 | export class UpdateVideosScheduler extends AbstractScheduler { | 12 | export class UpdateVideosScheduler extends AbstractScheduler { |
12 | 13 | ||
@@ -57,7 +58,7 @@ export class UpdateVideosScheduler extends AbstractScheduler { | |||
57 | 58 | ||
58 | for (const v of publishedVideos) { | 59 | for (const v of publishedVideos) { |
59 | Notifier.Instance.notifyOnNewVideo(v) | 60 | Notifier.Instance.notifyOnNewVideo(v) |
60 | Notifier.Instance.notifyOnPendingVideoPublished(v) | 61 | Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v) |
61 | } | 62 | } |
62 | } | 63 | } |
63 | 64 | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f643ee226..01af1e9d2 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' | 2 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
@@ -9,9 +9,20 @@ import { join } from 'path' | |||
9 | import { move } from 'fs-extra' | 9 | import { move } from 'fs-extra' |
10 | import { getServerActor } from '../../helpers/utils' | 10 | import { getServerActor } from '../../helpers/utils' |
11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
12 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 12 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
13 | import { removeVideoRedundancy } from '../redundancy' | 13 | import { removeVideoRedundancy } from '../redundancy' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | 14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { downloadPlaylistSegments } from '../hls' | ||
18 | import { CONFIG } from '../../initializers/config' | ||
19 | |||
20 | type CandidateToDuplicate = { | ||
21 | redundancy: VideosRedundancy, | ||
22 | video: VideoModel, | ||
23 | files: VideoFileModel[], | ||
24 | streamingPlaylists: VideoStreamingPlaylistModel[] | ||
25 | } | ||
15 | 26 | ||
16 | export class VideosRedundancyScheduler extends AbstractScheduler { | 27 | export class VideosRedundancyScheduler extends AbstractScheduler { |
17 | 28 | ||
@@ -24,28 +35,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
24 | } | 35 | } |
25 | 36 | ||
26 | protected async internalExecute () { | 37 | protected async internalExecute () { |
27 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 38 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
28 | logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) | 39 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) |
29 | 40 | ||
30 | try { | 41 | try { |
31 | const videoToDuplicate = await this.findVideoToDuplicate(obj) | 42 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) |
32 | if (!videoToDuplicate) continue | 43 | if (!videoToDuplicate) continue |
33 | 44 | ||
34 | const videoFiles = videoToDuplicate.VideoFiles | 45 | const candidateToDuplicate = { |
35 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 46 | video: videoToDuplicate, |
47 | redundancy: redundancyConfig, | ||
48 | files: videoToDuplicate.VideoFiles, | ||
49 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
50 | } | ||
36 | 51 | ||
37 | await this.purgeCacheIfNeeded(obj, videoFiles) | 52 | await this.purgeCacheIfNeeded(candidateToDuplicate) |
38 | 53 | ||
39 | if (await this.isTooHeavy(obj, videoFiles)) { | 54 | if (await this.isTooHeavy(candidateToDuplicate)) { |
40 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 55 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
41 | continue | 56 | continue |
42 | } | 57 | } |
43 | 58 | ||
44 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | 59 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) |
45 | 60 | ||
46 | await this.createVideoRedundancy(obj, videoFiles) | 61 | await this.createVideoRedundancies(candidateToDuplicate) |
47 | } catch (err) { | 62 | } catch (err) { |
48 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | 63 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) |
49 | } | 64 | } |
50 | } | 65 | } |
51 | 66 | ||
@@ -63,25 +78,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
63 | 78 | ||
64 | for (const redundancyModel of expired) { | 79 | for (const redundancyModel of expired) { |
65 | try { | 80 | try { |
66 | await this.extendsOrDeleteRedundancy(redundancyModel) | 81 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
82 | const candidate = { | ||
83 | redundancy: redundancyConfig, | ||
84 | video: null, | ||
85 | files: [], | ||
86 | streamingPlaylists: [] | ||
87 | } | ||
88 | |||
89 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | ||
90 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { | ||
91 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | ||
92 | await removeVideoRedundancy(redundancyModel) | ||
93 | } else { | ||
94 | await this.extendsRedundancy(redundancyModel) | ||
95 | } | ||
67 | } catch (err) { | 96 | } catch (err) { |
68 | logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) | 97 | logger.error( |
98 | 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), | ||
99 | { err } | ||
100 | ) | ||
69 | } | 101 | } |
70 | } | 102 | } |
71 | } | 103 | } |
72 | 104 | ||
73 | private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { | 105 | private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { |
74 | // Refresh the video, maybe it was deleted | ||
75 | const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url) | ||
76 | |||
77 | if (!video) { | ||
78 | logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url) | ||
79 | |||
80 | await redundancyModel.destroy() | ||
81 | return | ||
82 | } | ||
83 | |||
84 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 106 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
107 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | ||
108 | if (!redundancy) await removeVideoRedundancy(redundancyModel) | ||
109 | |||
85 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | 110 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) |
86 | } | 111 | } |
87 | 112 | ||
@@ -112,49 +137,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
112 | } | 137 | } |
113 | } | 138 | } |
114 | 139 | ||
115 | private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 140 | private async createVideoRedundancies (data: CandidateToDuplicate) { |
116 | const serverActor = await getServerActor() | 141 | const video = await this.loadAndRefreshVideo(data.video.url) |
142 | |||
143 | if (!video) { | ||
144 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) | ||
117 | 145 | ||
118 | for (const file of filesToDuplicate) { | 146 | return |
119 | const video = await this.loadAndRefreshVideo(file.Video.url) | 147 | } |
120 | 148 | ||
149 | for (const file of data.files) { | ||
121 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) | 150 | const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) |
122 | if (existingRedundancy) { | 151 | if (existingRedundancy) { |
123 | await this.extendsOrDeleteRedundancy(existingRedundancy) | 152 | await this.extendsRedundancy(existingRedundancy) |
124 | 153 | ||
125 | continue | 154 | continue |
126 | } | 155 | } |
127 | 156 | ||
128 | if (!video) { | 157 | await this.createVideoFileRedundancy(data.redundancy, video, file) |
129 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) | 158 | } |
159 | |||
160 | for (const streamingPlaylist of data.streamingPlaylists) { | ||
161 | const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) | ||
162 | if (existingRedundancy) { | ||
163 | await this.extendsRedundancy(existingRedundancy) | ||
130 | 164 | ||
131 | continue | 165 | continue |
132 | } | 166 | } |
133 | 167 | ||
134 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 168 | await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) |
169 | } | ||
170 | } | ||
135 | 171 | ||
136 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 172 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { |
137 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | 173 | file.Video = video |
138 | 174 | ||
139 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | 175 | const serverActor = await getServerActor() |
140 | 176 | ||
141 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) | 177 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
142 | await move(tmpPath, destPath) | ||
143 | 178 | ||
144 | const createdModel = await VideoRedundancyModel.create({ | 179 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
145 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 180 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) |
146 | url: getVideoCacheFileActivityPubUrl(file), | ||
147 | fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL), | ||
148 | strategy: redundancy.strategy, | ||
149 | videoFileId: file.id, | ||
150 | actorId: serverActor.id | ||
151 | }) | ||
152 | createdModel.VideoFile = file | ||
153 | 181 | ||
154 | await sendCreateCacheFile(serverActor, createdModel) | 182 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) |
155 | 183 | ||
156 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 184 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) |
157 | } | 185 | await move(tmpPath, destPath) |
186 | |||
187 | const createdModel = await VideoRedundancyModel.create({ | ||
188 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
189 | url: getVideoCacheFileActivityPubUrl(file), | ||
190 | fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), | ||
191 | strategy: redundancy.strategy, | ||
192 | videoFileId: file.id, | ||
193 | actorId: serverActor.id | ||
194 | }) | ||
195 | |||
196 | createdModel.VideoFile = file | ||
197 | |||
198 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
199 | |||
200 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | ||
201 | } | ||
202 | |||
203 | private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { | ||
204 | playlist.Video = video | ||
205 | |||
206 | const serverActor = await getServerActor() | ||
207 | |||
208 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) | ||
209 | |||
210 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | ||
211 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | ||
212 | |||
213 | const createdModel = await VideoRedundancyModel.create({ | ||
214 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | ||
215 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | ||
216 | fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), | ||
217 | strategy: redundancy.strategy, | ||
218 | videoStreamingPlaylistId: playlist.id, | ||
219 | actorId: serverActor.id | ||
220 | }) | ||
221 | |||
222 | createdModel.VideoStreamingPlaylist = playlist | ||
223 | |||
224 | await sendCreateCacheFile(serverActor, video, createdModel) | ||
225 | |||
226 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) | ||
158 | } | 227 | } |
159 | 228 | ||
160 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { | 229 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { |
@@ -168,8 +237,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
168 | await sendUpdateCacheFile(serverActor, redundancy) | 237 | await sendUpdateCacheFile(serverActor, redundancy) |
169 | } | 238 | } |
170 | 239 | ||
171 | private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 240 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { |
172 | while (this.isTooHeavy(redundancy, filesToDuplicate)) { | 241 | while (this.isTooHeavy(candidateToDuplicate)) { |
242 | const redundancy = candidateToDuplicate.redundancy | ||
173 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | 243 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) |
174 | if (!toDelete) return | 244 | if (!toDelete) return |
175 | 245 | ||
@@ -177,11 +247,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
177 | } | 247 | } |
178 | } | 248 | } |
179 | 249 | ||
180 | private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | 250 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { |
181 | const maxSize = redundancy.size | 251 | const maxSize = candidateToDuplicate.redundancy.size |
182 | 252 | ||
183 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) | 253 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy) |
184 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) | 254 | const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) |
185 | 255 | ||
186 | return totalWillDuplicate > maxSize | 256 | return totalWillDuplicate > maxSize |
187 | } | 257 | } |
@@ -191,13 +261,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
191 | } | 261 | } |
192 | 262 | ||
193 | private buildEntryLogId (object: VideoRedundancyModel) { | 263 | private buildEntryLogId (object: VideoRedundancyModel) { |
194 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | 264 | if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` |
265 | |||
266 | return `${object.VideoStreamingPlaylist.playlistUrl}` | ||
195 | } | 267 | } |
196 | 268 | ||
197 | private getTotalFileSizes (files: VideoFileModel[]) { | 269 | private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { |
198 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | 270 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size |
199 | 271 | ||
200 | return files.reduce(fileReducer, 0) | 272 | return files.reduce(fileReducer, 0) * playlists.length |
201 | } | 273 | } |
202 | 274 | ||
203 | private async loadAndRefreshVideo (videoUrl: string) { | 275 | private async loadAndRefreshVideo (videoUrl: string) { |
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index aa027116d..aefe6aba4 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' | 2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
3 | import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' | 3 | import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' |
4 | 4 | ||
5 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { | 5 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts new file mode 100644 index 000000000..950b14c3b --- /dev/null +++ b/server/lib/thumbnail.ts | |||
@@ -0,0 +1,151 @@ | |||
1 | import { VideoFileModel } from '../models/video/video-file' | ||
2 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' | ||
3 | import { CONFIG } from '../initializers/config' | ||
4 | import { PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' | ||
5 | import { VideoModel } from '../models/video/video' | ||
6 | import { ThumbnailModel } from '../models/video/thumbnail' | ||
7 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' | ||
8 | import { processImage } from '../helpers/image-utils' | ||
9 | import { join } from 'path' | ||
10 | import { downloadImage } from '../helpers/requests' | ||
11 | import { VideoPlaylistModel } from '../models/video/video-playlist' | ||
12 | |||
13 | type ImageSize = { height: number, width: number } | ||
14 | |||
15 | function createPlaylistMiniatureFromExisting (inputPath: string, playlist: VideoPlaylistModel, keepOriginal = false, size?: ImageSize) { | ||
16 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) | ||
17 | const type = ThumbnailType.MINIATURE | ||
18 | |||
19 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | ||
20 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail }) | ||
21 | } | ||
22 | |||
23 | function createPlaylistMiniatureFromUrl (fileUrl: string, playlist: VideoPlaylistModel, size?: ImageSize) { | ||
24 | const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) | ||
25 | const type = ThumbnailType.MINIATURE | ||
26 | |||
27 | const thumbnailCreator = () => downloadImage(fileUrl, basePath, filename, { width, height }) | ||
28 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | ||
29 | } | ||
30 | |||
31 | function createVideoMiniatureFromUrl (fileUrl: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) { | ||
32 | const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
33 | const thumbnailCreator = () => downloadImage(fileUrl, basePath, filename, { width, height }) | ||
34 | |||
35 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | ||
36 | } | ||
37 | |||
38 | function createVideoMiniatureFromExisting (inputPath: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) { | ||
39 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
40 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }) | ||
41 | |||
42 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail }) | ||
43 | } | ||
44 | |||
45 | function generateVideoMiniature (video: VideoModel, videoFile: VideoFileModel, type: ThumbnailType) { | ||
46 | const input = video.getVideoFilePath(videoFile) | ||
47 | |||
48 | const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type) | ||
49 | const thumbnailCreator = () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | ||
50 | |||
51 | return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail }) | ||
52 | } | ||
53 | |||
54 | function createPlaceholderThumbnail (fileUrl: string, video: VideoModel, type: ThumbnailType, size: ImageSize) { | ||
55 | const { filename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
56 | |||
57 | const thumbnail = existingThumbnail ? existingThumbnail : new ThumbnailModel() | ||
58 | |||
59 | thumbnail.filename = filename | ||
60 | thumbnail.height = height | ||
61 | thumbnail.width = width | ||
62 | thumbnail.type = type | ||
63 | thumbnail.fileUrl = fileUrl | ||
64 | |||
65 | return thumbnail | ||
66 | } | ||
67 | |||
68 | // --------------------------------------------------------------------------- | ||
69 | |||
70 | export { | ||
71 | generateVideoMiniature, | ||
72 | createVideoMiniatureFromUrl, | ||
73 | createVideoMiniatureFromExisting, | ||
74 | createPlaceholderThumbnail, | ||
75 | createPlaylistMiniatureFromUrl, | ||
76 | createPlaylistMiniatureFromExisting | ||
77 | } | ||
78 | |||
79 | function buildMetadataFromPlaylist (playlist: VideoPlaylistModel, size: ImageSize) { | ||
80 | const filename = playlist.generateThumbnailName() | ||
81 | const basePath = CONFIG.STORAGE.THUMBNAILS_DIR | ||
82 | |||
83 | return { | ||
84 | filename, | ||
85 | basePath, | ||
86 | existingThumbnail: playlist.Thumbnail, | ||
87 | outputPath: join(basePath, filename), | ||
88 | height: size ? size.height : THUMBNAILS_SIZE.height, | ||
89 | width: size ? size.width : THUMBNAILS_SIZE.width | ||
90 | } | ||
91 | } | ||
92 | |||
93 | function buildMetadataFromVideo (video: VideoModel, type: ThumbnailType, size?: ImageSize) { | ||
94 | const existingThumbnail = Array.isArray(video.Thumbnails) | ||
95 | ? video.Thumbnails.find(t => t.type === type) | ||
96 | : undefined | ||
97 | |||
98 | if (type === ThumbnailType.MINIATURE) { | ||
99 | const filename = video.generateThumbnailName() | ||
100 | const basePath = CONFIG.STORAGE.THUMBNAILS_DIR | ||
101 | |||
102 | return { | ||
103 | filename, | ||
104 | basePath, | ||
105 | existingThumbnail, | ||
106 | outputPath: join(basePath, filename), | ||
107 | height: size ? size.height : THUMBNAILS_SIZE.height, | ||
108 | width: size ? size.width : THUMBNAILS_SIZE.width | ||
109 | } | ||
110 | } | ||
111 | |||
112 | if (type === ThumbnailType.PREVIEW) { | ||
113 | const filename = video.generatePreviewName() | ||
114 | const basePath = CONFIG.STORAGE.PREVIEWS_DIR | ||
115 | |||
116 | return { | ||
117 | filename, | ||
118 | basePath, | ||
119 | existingThumbnail, | ||
120 | outputPath: join(basePath, filename), | ||
121 | height: size ? size.height : PREVIEWS_SIZE.height, | ||
122 | width: size ? size.width : PREVIEWS_SIZE.width | ||
123 | } | ||
124 | } | ||
125 | |||
126 | return undefined | ||
127 | } | ||
128 | |||
129 | async function createThumbnailFromFunction (parameters: { | ||
130 | thumbnailCreator: () => Promise<any>, | ||
131 | filename: string, | ||
132 | height: number, | ||
133 | width: number, | ||
134 | type: ThumbnailType, | ||
135 | fileUrl?: string, | ||
136 | existingThumbnail?: ThumbnailModel | ||
137 | }) { | ||
138 | const { thumbnailCreator, filename, width, height, type, existingThumbnail, fileUrl = null } = parameters | ||
139 | |||
140 | const thumbnail = existingThumbnail ? existingThumbnail : new ThumbnailModel() | ||
141 | |||
142 | thumbnail.filename = filename | ||
143 | thumbnail.height = height | ||
144 | thumbnail.width = width | ||
145 | thumbnail.type = type | ||
146 | thumbnail.fileUrl = fileUrl | ||
147 | |||
148 | await thumbnailCreator() | ||
149 | |||
150 | return thumbnail | ||
151 | } | ||
diff --git a/server/lib/user.ts b/server/lib/user.ts index a39ef6c3d..7badb3e72 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts | |||
@@ -1,18 +1,19 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import * as uuidv4 from 'uuid/v4' | 2 | import * as uuidv4 from 'uuid/v4' |
3 | import { ActivityPubActorType } from '../../shared/models/activitypub' | 3 | import { ActivityPubActorType } from '../../shared/models/activitypub' |
4 | import { sequelizeTypescript, SERVER_ACTOR_NAME } from '../initializers' | 4 | import { SERVER_ACTOR_NAME } from '../initializers/constants' |
5 | import { AccountModel } from '../models/account/account' | 5 | import { AccountModel } from '../models/account/account' |
6 | import { UserModel } from '../models/account/user' | 6 | import { UserModel } from '../models/account/user' |
7 | import { buildActorInstance, getAccountActivityPubUrl, setAsyncActorKeys } from './activitypub' | 7 | import { buildActorInstance, getAccountActivityPubUrl, setAsyncActorKeys } from './activitypub' |
8 | import { createVideoChannel } from './video-channel' | 8 | import { createVideoChannel } from './video-channel' |
9 | import { VideoChannelModel } from '../models/video/video-channel' | 9 | import { VideoChannelModel } from '../models/video/video-channel' |
10 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | ||
11 | import { ActorModel } from '../models/activitypub/actor' | 10 | import { ActorModel } from '../models/activitypub/actor' |
12 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' | 11 | import { UserNotificationSettingModel } from '../models/account/user-notification-setting' |
13 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' | 12 | import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' |
13 | import { createWatchLaterPlaylist } from './video-playlist' | ||
14 | import { sequelizeTypescript } from '../initializers/database' | ||
14 | 15 | ||
15 | async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) { | 16 | async function createUserAccountAndChannelAndPlaylist (userToCreate: UserModel, validateUser = true) { |
16 | const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { | 17 | const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { |
17 | const userOptions = { | 18 | const userOptions = { |
18 | transaction: t, | 19 | transaction: t, |
@@ -38,7 +39,9 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse | |||
38 | } | 39 | } |
39 | const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) | 40 | const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) |
40 | 41 | ||
41 | return { user: userCreated, account: accountCreated, videoChannel } | 42 | const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) |
43 | |||
44 | return { user: userCreated, account: accountCreated, videoChannel, videoPlaylist } | ||
42 | }) | 45 | }) |
43 | 46 | ||
44 | const [ accountKeys, channelKeys ] = await Promise.all([ | 47 | const [ accountKeys, channelKeys ] = await Promise.all([ |
@@ -69,7 +72,7 @@ async function createLocalAccountWithoutKeys ( | |||
69 | userId, | 72 | userId, |
70 | applicationId, | 73 | applicationId, |
71 | actorId: actorInstanceCreated.id | 74 | actorId: actorInstanceCreated.id |
72 | } as FilteredModelAttributes<AccountModel>) | 75 | }) |
73 | 76 | ||
74 | const accountInstanceCreated = await accountInstance.save({ transaction: t }) | 77 | const accountInstanceCreated = await accountInstance.save({ transaction: t }) |
75 | accountInstanceCreated.Actor = actorInstanceCreated | 78 | accountInstanceCreated.Actor = actorInstanceCreated |
@@ -89,7 +92,7 @@ async function createApplicationActor (applicationId: number) { | |||
89 | 92 | ||
90 | export { | 93 | export { |
91 | createApplicationActor, | 94 | createApplicationActor, |
92 | createUserAccountAndChannel, | 95 | createUserAccountAndChannelAndPlaylist, |
93 | createLocalAccountWithoutKeys | 96 | createLocalAccountWithoutKeys |
94 | } | 97 | } |
95 | 98 | ||
@@ -103,10 +106,12 @@ function createDefaultUserNotificationSettings (user: UserModel, t: Sequelize.Tr | |||
103 | myVideoImportFinished: UserNotificationSettingValue.WEB, | 106 | myVideoImportFinished: UserNotificationSettingValue.WEB, |
104 | myVideoPublished: UserNotificationSettingValue.WEB, | 107 | myVideoPublished: UserNotificationSettingValue.WEB, |
105 | videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | 108 | videoAbuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, |
109 | videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | ||
106 | blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, | 110 | blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, |
107 | newUserRegistration: UserNotificationSettingValue.WEB, | 111 | newUserRegistration: UserNotificationSettingValue.WEB, |
108 | commentMention: UserNotificationSettingValue.WEB, | 112 | commentMention: UserNotificationSettingValue.WEB, |
109 | newFollow: UserNotificationSettingValue.WEB | 113 | newFollow: UserNotificationSettingValue.WEB, |
114 | newInstanceFollower: UserNotificationSettingValue.WEB | ||
110 | } | 115 | } |
111 | 116 | ||
112 | return UserNotificationSettingModel.create(values, { transaction: t }) | 117 | return UserNotificationSettingModel.create(values, { transaction: t }) |
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts new file mode 100644 index 000000000..985b89e31 --- /dev/null +++ b/server/lib/video-blacklist.ts | |||
@@ -0,0 +1,33 @@ | |||
1 | import * as sequelize from 'sequelize' | ||
2 | import { CONFIG } from '../initializers/config' | ||
3 | import { UserRight, VideoBlacklistType } from '../../shared/models' | ||
4 | import { VideoBlacklistModel } from '../models/video/video-blacklist' | ||
5 | import { UserModel } from '../models/account/user' | ||
6 | import { VideoModel } from '../models/video/video' | ||
7 | import { logger } from '../helpers/logger' | ||
8 | import { UserAdminFlag } from '../../shared/models/users/user-flag.model' | ||
9 | |||
10 | async function autoBlacklistVideoIfNeeded (video: VideoModel, user: UserModel, transaction: sequelize.Transaction) { | ||
11 | if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED) return false | ||
12 | |||
13 | if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BY_PASS_VIDEO_AUTO_BLACKLIST)) return false | ||
14 | |||
15 | const sequelizeOptions = { transaction } | ||
16 | const videoBlacklistToCreate = { | ||
17 | videoId: video.id, | ||
18 | unfederated: true, | ||
19 | reason: 'Auto-blacklisted. Moderator review required.', | ||
20 | type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED | ||
21 | } | ||
22 | await VideoBlacklistModel.create(videoBlacklistToCreate, sequelizeOptions) | ||
23 | |||
24 | logger.info('Video %s auto-blacklisted.', video.uuid) | ||
25 | |||
26 | return true | ||
27 | } | ||
28 | |||
29 | // --------------------------------------------------------------------------- | ||
30 | |||
31 | export { | ||
32 | autoBlacklistVideoIfNeeded | ||
33 | } | ||
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts index 59bce7520..bfe22d225 100644 --- a/server/lib/video-comment.ts +++ b/server/lib/video-comment.ts | |||
@@ -28,7 +28,7 @@ async function createVideoComment (obj: { | |||
28 | videoId: obj.video.id, | 28 | videoId: obj.video.id, |
29 | accountId: obj.account.id, | 29 | accountId: obj.account.id, |
30 | url: 'fake url' | 30 | url: 'fake url' |
31 | }, { transaction: t, validate: false }) | 31 | }, { transaction: t, validate: false } as any) // FIXME: sequelize typings |
32 | 32 | ||
33 | comment.set('url', getVideoCommentActivityPubUrl(obj.video, comment)) | 33 | comment.set('url', getVideoCommentActivityPubUrl(obj.video, comment)) |
34 | 34 | ||
diff --git a/server/lib/video-playlist.ts b/server/lib/video-playlist.ts new file mode 100644 index 000000000..6e214e60f --- /dev/null +++ b/server/lib/video-playlist.ts | |||
@@ -0,0 +1,29 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { AccountModel } from '../models/account/account' | ||
3 | import { VideoPlaylistModel } from '../models/video/video-playlist' | ||
4 | import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model' | ||
5 | import { getVideoPlaylistActivityPubUrl } from './activitypub' | ||
6 | import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model' | ||
7 | |||
8 | async function createWatchLaterPlaylist (account: AccountModel, t: Sequelize.Transaction) { | ||
9 | const videoPlaylist = new VideoPlaylistModel({ | ||
10 | name: 'Watch later', | ||
11 | privacy: VideoPlaylistPrivacy.PRIVATE, | ||
12 | type: VideoPlaylistType.WATCH_LATER, | ||
13 | ownerAccountId: account.id | ||
14 | }) | ||
15 | |||
16 | videoPlaylist.url = getVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object | ||
17 | |||
18 | await videoPlaylist.save({ transaction: t }) | ||
19 | |||
20 | videoPlaylist.OwnerAccount = account | ||
21 | |||
22 | return videoPlaylist | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
26 | |||
27 | export { | ||
28 | createWatchLaterPlaylist | ||
29 | } | ||
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 4460f46e4..0fe0ff12a 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts | |||
@@ -1,11 +1,15 @@ | |||
1 | import { CONFIG } from '../initializers' | 1 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants' |
2 | import { extname, join } from 'path' | 2 | import { join } from 'path' |
3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' | 3 | import { getVideoFileFPS, transcode } from '../helpers/ffmpeg-utils' |
4 | import { copy, remove, move, stat } from 'fs-extra' | 4 | import { ensureDir, move, remove, stat } from 'fs-extra' |
5 | import { logger } from '../helpers/logger' | 5 | import { logger } from '../helpers/logger' |
6 | import { VideoResolution } from '../../shared/models/videos' | 6 | import { VideoResolution } from '../../shared/models/videos' |
7 | import { VideoFileModel } from '../models/video/video-file' | 7 | import { VideoFileModel } from '../models/video/video-file' |
8 | import { VideoModel } from '../models/video/video' | 8 | import { VideoModel } from '../models/video/video' |
9 | import { updateMasterHLSPlaylist, updateSha256Segments } from './hls' | ||
10 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | ||
11 | import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' | ||
12 | import { CONFIG } from '../initializers/config' | ||
9 | 13 | ||
10 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { | 14 | async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { |
11 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 15 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
@@ -17,7 +21,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
17 | 21 | ||
18 | const transcodeOptions = { | 22 | const transcodeOptions = { |
19 | inputPath: videoInputPath, | 23 | inputPath: videoInputPath, |
20 | outputPath: videoTranscodedPath | 24 | outputPath: videoTranscodedPath, |
25 | resolution: inputVideoFile.resolution | ||
21 | } | 26 | } |
22 | 27 | ||
23 | // Could be very long! | 28 | // Could be very long! |
@@ -47,7 +52,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi | |||
47 | } | 52 | } |
48 | } | 53 | } |
49 | 54 | ||
50 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | 55 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) { |
51 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | 56 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR |
52 | const extname = '.mp4' | 57 | const extname = '.mp4' |
53 | 58 | ||
@@ -60,13 +65,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
60 | size: 0, | 65 | size: 0, |
61 | videoId: video.id | 66 | videoId: video.id |
62 | }) | 67 | }) |
63 | const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) | 68 | const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) |
64 | 69 | ||
65 | const transcodeOptions = { | 70 | const transcodeOptions = { |
66 | inputPath: videoInputPath, | 71 | inputPath: videoInputPath, |
67 | outputPath: videoOutputPath, | 72 | outputPath: videoOutputPath, |
68 | resolution, | 73 | resolution, |
69 | isPortraitMode | 74 | isPortraitMode: isPortrait |
70 | } | 75 | } |
71 | 76 | ||
72 | await transcode(transcodeOptions) | 77 | await transcode(transcodeOptions) |
@@ -84,48 +89,44 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR | |||
84 | video.VideoFiles.push(newVideoFile) | 89 | video.VideoFiles.push(newVideoFile) |
85 | } | 90 | } |
86 | 91 | ||
87 | async function importVideoFile (video: VideoModel, inputFilePath: string) { | 92 | async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { |
88 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | 93 | const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) |
89 | const { size } = await stat(inputFilePath) | 94 | await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)) |
90 | const fps = await getVideoFileFPS(inputFilePath) | ||
91 | 95 | ||
92 | let updatedVideoFile = new VideoFileModel({ | 96 | const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile())) |
93 | resolution: videoFileResolution, | 97 | const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) |
94 | extname: extname(inputFilePath), | ||
95 | size, | ||
96 | fps, | ||
97 | videoId: video.id | ||
98 | }) | ||
99 | |||
100 | const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) | ||
101 | 98 | ||
102 | if (currentVideoFile) { | 99 | const transcodeOptions = { |
103 | // Remove old file and old torrent | 100 | inputPath: videoInputPath, |
104 | await video.removeFile(currentVideoFile) | 101 | outputPath, |
105 | await video.removeTorrent(currentVideoFile) | 102 | resolution, |
106 | // Remove the old video file from the array | 103 | isPortraitMode, |
107 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) | ||
108 | |||
109 | // Update the database | ||
110 | currentVideoFile.set('extname', updatedVideoFile.extname) | ||
111 | currentVideoFile.set('size', updatedVideoFile.size) | ||
112 | currentVideoFile.set('fps', updatedVideoFile.fps) | ||
113 | 104 | ||
114 | updatedVideoFile = currentVideoFile | 105 | hlsPlaylist: { |
106 | videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) | ||
107 | } | ||
115 | } | 108 | } |
116 | 109 | ||
117 | const outputPath = video.getVideoFilePath(updatedVideoFile) | 110 | await transcode(transcodeOptions) |
118 | await copy(inputFilePath, outputPath) | ||
119 | 111 | ||
120 | await video.createTorrentAndSetInfoHash(updatedVideoFile) | 112 | await updateMasterHLSPlaylist(video) |
113 | await updateSha256Segments(video) | ||
121 | 114 | ||
122 | await updatedVideoFile.save() | 115 | const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) |
123 | 116 | ||
124 | video.VideoFiles.push(updatedVideoFile) | 117 | await VideoStreamingPlaylistModel.upsert({ |
118 | videoId: video.id, | ||
119 | playlistUrl, | ||
120 | segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), | ||
121 | p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles), | ||
122 | p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, | ||
123 | |||
124 | type: VideoStreamingPlaylistType.HLS | ||
125 | }) | ||
125 | } | 126 | } |
126 | 127 | ||
127 | export { | 128 | export { |
129 | generateHlsPlaylist, | ||
128 | optimizeVideofile, | 130 | optimizeVideofile, |
129 | transcodeOriginalVideofile, | 131 | transcodeOriginalVideofile |
130 | importVideoFile | ||
131 | } | 132 | } |