aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts594
-rw-r--r--server/lib/activitypub/actors/get.ts122
-rw-r--r--server/lib/activitypub/actors/image.ts94
-rw-r--r--server/lib/activitypub/actors/index.ts6
-rw-r--r--server/lib/activitypub/actors/keys.ts16
-rw-r--r--server/lib/activitypub/actors/refresh.ts81
-rw-r--r--server/lib/activitypub/actors/shared/creator.ts149
-rw-r--r--server/lib/activitypub/actors/shared/index.ts3
-rw-r--r--server/lib/activitypub/actors/shared/object-to-model-attributes.ts70
-rw-r--r--server/lib/activitypub/actors/shared/url-to-object.ts54
-rw-r--r--server/lib/activitypub/actors/updater.ts90
-rw-r--r--server/lib/activitypub/actors/webfinger.ts67
-rw-r--r--server/lib/activitypub/audience.ts2
-rw-r--r--server/lib/activitypub/cache-file.ts91
-rw-r--r--server/lib/activitypub/crawl.ts7
-rw-r--r--server/lib/activitypub/follow.ts13
-rw-r--r--server/lib/activitypub/outbox.ts24
-rw-r--r--server/lib/activitypub/playlist.ts204
-rw-r--r--server/lib/activitypub/playlists/create-update.ts156
-rw-r--r--server/lib/activitypub/playlists/get.ts35
-rw-r--r--server/lib/activitypub/playlists/index.ts3
-rw-r--r--server/lib/activitypub/playlists/refresh.ts53
-rw-r--r--server/lib/activitypub/playlists/shared/index.ts2
-rw-r--r--server/lib/activitypub/playlists/shared/object-to-model-attributes.ts40
-rw-r--r--server/lib/activitypub/playlists/shared/url-to-object.ts47
-rw-r--r--server/lib/activitypub/process/process-accept.ts4
-rw-r--r--server/lib/activitypub/process/process-announce.ts4
-rw-r--r--server/lib/activitypub/process/process-create.ts12
-rw-r--r--server/lib/activitypub/process/process-delete.ts11
-rw-r--r--server/lib/activitypub/process/process-dislike.ts4
-rw-r--r--server/lib/activitypub/process/process-follow.ts21
-rw-r--r--server/lib/activitypub/process/process-like.ts4
-rw-r--r--server/lib/activitypub/process/process-reject.ts2
-rw-r--r--server/lib/activitypub/process/process-undo.ts16
-rw-r--r--server/lib/activitypub/process/process-update.ts104
-rw-r--r--server/lib/activitypub/process/process-view.ts13
-rw-r--r--server/lib/activitypub/process/process.ts14
-rw-r--r--server/lib/activitypub/send/send-delete.ts2
-rw-r--r--server/lib/activitypub/send/send-view.ts2
-rw-r--r--server/lib/activitypub/send/utils.ts16
-rw-r--r--server/lib/activitypub/share.ts40
-rw-r--r--server/lib/activitypub/video-comments.ts41
-rw-r--r--server/lib/activitypub/video-rates.ts64
-rw-r--r--server/lib/activitypub/videos.ts931
-rw-r--r--server/lib/activitypub/videos/federate.ts36
-rw-r--r--server/lib/activitypub/videos/get.ts113
-rw-r--r--server/lib/activitypub/videos/index.ts4
-rw-r--r--server/lib/activitypub/videos/refresh.ts68
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts173
-rw-r--r--server/lib/activitypub/videos/shared/creator.ts88
-rw-r--r--server/lib/activitypub/videos/shared/index.ts6
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts256
-rw-r--r--server/lib/activitypub/videos/shared/trackers.ts43
-rw-r--r--server/lib/activitypub/videos/shared/url-to-object.ts25
-rw-r--r--server/lib/activitypub/videos/shared/video-sync-attributes.ts94
-rw-r--r--server/lib/activitypub/videos/updater.ts166
-rw-r--r--server/lib/auth/oauth-model.ts4
-rw-r--r--server/lib/client-html.ts48
-rw-r--r--server/lib/config.ts255
-rw-r--r--server/lib/job-queue/handlers/activitypub-follow.ts21
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-broadcast.ts3
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts10
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-unicast.ts2
-rw-r--r--server/lib/job-queue/handlers/activitypub-refresher.ts16
-rw-r--r--server/lib/job-queue/handlers/actor-keys.ts4
-rw-r--r--server/lib/job-queue/handlers/utils/activitypub-http-utils.ts8
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts6
-rw-r--r--server/lib/job-queue/handlers/video-import.ts11
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts21
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts53
-rw-r--r--server/lib/job-queue/handlers/video-views.ts4
-rw-r--r--server/lib/live-manager.ts621
-rw-r--r--server/lib/live/index.ts4
-rw-r--r--server/lib/live/live-manager.ts419
-rw-r--r--server/lib/live/live-quota-store.ts48
-rw-r--r--server/lib/live/live-segment-sha-store.ts64
-rw-r--r--server/lib/live/live-utils.ts23
-rw-r--r--server/lib/live/shared/index.ts1
-rw-r--r--server/lib/live/shared/muxing-session.ts346
-rw-r--r--server/lib/local-actor.ts (renamed from server/lib/actor-image.ts)38
-rw-r--r--server/lib/model-loaders/actor.ts17
-rw-r--r--server/lib/model-loaders/index.ts2
-rw-r--r--server/lib/model-loaders/video.ts73
-rw-r--r--server/lib/moderation.ts8
-rw-r--r--server/lib/notifier.ts4
-rw-r--r--server/lib/plugins/hooks.ts6
-rw-r--r--server/lib/plugins/plugin-helpers-builder.ts6
-rw-r--r--server/lib/plugins/plugin-index.ts20
-rw-r--r--server/lib/plugins/plugin-manager.ts41
-rw-r--r--server/lib/plugins/register-helpers.ts34
-rw-r--r--server/lib/redundancy.ts12
-rw-r--r--server/lib/schedulers/actor-follow-scheduler.ts4
-rw-r--r--server/lib/schedulers/auto-follow-index-instances.ts2
-rw-r--r--server/lib/schedulers/remove-old-history-scheduler.ts2
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts32
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts4
-rw-r--r--server/lib/schedulers/youtube-dl-update-scheduler.ts6
-rw-r--r--server/lib/search.ts50
-rw-r--r--server/lib/server-config-manager.ts304
-rw-r--r--server/lib/signup.ts62
-rw-r--r--server/lib/stat-manager.ts4
-rw-r--r--server/lib/thumbnail.ts32
-rw-r--r--server/lib/transcoding/video-transcoding-profiles.ts (renamed from server/lib/video-transcoding-profiles.ts)8
-rw-r--r--server/lib/transcoding/video-transcoding.ts (renamed from server/lib/video-transcoding.ts)42
-rw-r--r--server/lib/user.ts36
-rw-r--r--server/lib/video-blacklist.ts2
-rw-r--r--server/lib/video-channel.ts6
-rw-r--r--server/lib/video-comment.ts15
-rw-r--r--server/lib/video.ts6
109 files changed, 4120 insertions, 3150 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
deleted file mode 100644
index 5fe7381c9..000000000
--- a/server/lib/activitypub/actor.ts
+++ /dev/null
@@ -1,594 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { extname } from 'path'
3import { Op, Transaction } from 'sequelize'
4import { URL } from 'url'
5import { v4 as uuidv4 } from 'uuid'
6import { getServerActor } from '@server/models/application/application'
7import { ActorImageType } from '@shared/models'
8import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
9import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
10import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
11import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
12import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
13import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor'
14import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
15import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
16import { logger } from '../../helpers/logger'
17import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
18import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
19import { getUrlFromWebfinger } from '../../helpers/webfinger'
20import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
21import { sequelizeTypescript } from '../../initializers/database'
22import { AccountModel } from '../../models/account/account'
23import { ActorImageModel } from '../../models/account/actor-image'
24import { ActorModel } from '../../models/activitypub/actor'
25import { ServerModel } from '../../models/server/server'
26import { VideoChannelModel } from '../../models/video/video-channel'
27import {
28 MAccount,
29 MAccountDefault,
30 MActor,
31 MActorAccountChannelId,
32 MActorAccountChannelIdActor,
33 MActorAccountId,
34 MActorFull,
35 MActorFullActor,
36 MActorId,
37 MActorImage,
38 MActorImages,
39 MChannel
40} from '../../types/models'
41import { JobQueue } from '../job-queue'
42
43// Set account keys, this could be long so process after the account creation and do not block the client
44async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
45 const { publicKey, privateKey } = await createPrivateAndPublicKeys()
46
47 actor.publicKey = publicKey
48 actor.privateKey = privateKey
49
50 return actor.save()
51}
52
53function getOrCreateActorAndServerAndModel (
54 activityActor: string | ActivityPubActor,
55 fetchType: 'all',
56 recurseIfNeeded?: boolean,
57 updateCollections?: boolean
58): Promise<MActorFullActor>
59
60function getOrCreateActorAndServerAndModel (
61 activityActor: string | ActivityPubActor,
62 fetchType?: 'association-ids',
63 recurseIfNeeded?: boolean,
64 updateCollections?: boolean
65): Promise<MActorAccountChannelId>
66
67async function getOrCreateActorAndServerAndModel (
68 activityActor: string | ActivityPubActor,
69 fetchType: ActorFetchByUrlType = 'association-ids',
70 recurseIfNeeded = true,
71 updateCollections = false
72): Promise<MActorFullActor | MActorAccountChannelId> {
73 const actorUrl = getAPId(activityActor)
74 let created = false
75 let accountPlaylistsUrl: string
76
77 let actor = await fetchActorByUrl(actorUrl, fetchType)
78 // Orphan actor (not associated to an account of channel) so recreate it
79 if (actor && (!actor.Account && !actor.VideoChannel)) {
80 await actor.destroy()
81 actor = null
82 }
83
84 // We don't have this actor in our database, fetch it on remote
85 if (!actor) {
86 const { result } = await fetchRemoteActor(actorUrl)
87 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
88
89 // Create the attributed to actor
90 // In PeerTube a video channel is owned by an account
91 let ownerActor: MActorFullActor
92 if (recurseIfNeeded === true && result.actor.type === 'Group') {
93 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
94 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
95
96 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
97 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
98 }
99
100 try {
101 // Don't recurse another time
102 const recurseIfNeeded = false
103 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
104 } catch (err) {
105 logger.error('Cannot get or create account attributed to video channel ' + actorUrl)
106 throw new Error(err)
107 }
108 }
109
110 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
111 created = true
112 accountPlaylistsUrl = result.playlists
113 }
114
115 if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor
116 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
117
118 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
119 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
120
121 if ((created === true || refreshed === true) && updateCollections === true) {
122 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
123 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
124 }
125
126 // We created a new account: fetch the playlists
127 if (created === true && actor.Account && accountPlaylistsUrl) {
128 const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
129 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
130 }
131
132 return actorRefreshed
133}
134
135function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
136 return new ActorModel({
137 type,
138 url,
139 preferredUsername,
140 uuid,
141 publicKey: null,
142 privateKey: null,
143 followersCount: 0,
144 followingCount: 0,
145 inboxUrl: url + '/inbox',
146 outboxUrl: url + '/outbox',
147 sharedInboxUrl: WEBSERVER.URL + '/inbox',
148 followersUrl: url + '/followers',
149 followingUrl: url + '/following'
150 }) as MActor
151}
152
153async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
154 const followersCount = await fetchActorTotalItems(attributes.followers)
155 const followingCount = await fetchActorTotalItems(attributes.following)
156
157 actorInstance.type = attributes.type
158 actorInstance.preferredUsername = attributes.preferredUsername
159 actorInstance.url = attributes.id
160 actorInstance.publicKey = attributes.publicKey.publicKeyPem
161 actorInstance.followersCount = followersCount
162 actorInstance.followingCount = followingCount
163 actorInstance.inboxUrl = attributes.inbox
164 actorInstance.outboxUrl = attributes.outbox
165 actorInstance.followersUrl = attributes.followers
166 actorInstance.followingUrl = attributes.following
167
168 if (attributes.published) actorInstance.remoteCreatedAt = new Date(attributes.published)
169
170 if (attributes.endpoints?.sharedInbox) {
171 actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox
172 }
173}
174
175type ImageInfo = {
176 name: string
177 fileUrl: string
178 height: number
179 width: number
180 onDisk?: boolean
181}
182async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) {
183 const oldImageModel = type === ActorImageType.AVATAR
184 ? actor.Avatar
185 : actor.Banner
186
187 if (oldImageModel) {
188 // Don't update the avatar if the file URL did not change
189 if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor
190
191 try {
192 await oldImageModel.destroy({ transaction: t })
193
194 setActorImage(actor, type, null)
195 } catch (err) {
196 logger.error('Cannot remove old actor image of actor %s.', actor.url, { err })
197 }
198 }
199
200 if (imageInfo) {
201 const imageModel = await ActorImageModel.create({
202 filename: imageInfo.name,
203 onDisk: imageInfo.onDisk ?? false,
204 fileUrl: imageInfo.fileUrl,
205 height: imageInfo.height,
206 width: imageInfo.width,
207 type
208 }, { transaction: t })
209
210 setActorImage(actor, type, imageModel)
211 }
212
213 return actor
214}
215
216async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) {
217 try {
218 if (type === ActorImageType.AVATAR) {
219 await actor.Avatar.destroy({ transaction: t })
220
221 actor.avatarId = null
222 actor.Avatar = null
223 } else {
224 await actor.Banner.destroy({ transaction: t })
225
226 actor.bannerId = null
227 actor.Banner = null
228 }
229 } catch (err) {
230 logger.error('Cannot remove old image of actor %s.', actor.url, { err })
231 }
232
233 return actor
234}
235
236async function fetchActorTotalItems (url: string) {
237 try {
238 const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
239
240 return body.totalItems || 0
241 } catch (err) {
242 logger.warn('Cannot fetch remote actor count %s.', url, { err })
243 return 0
244 }
245}
246
247function getImageInfoIfExists (actorJSON: ActivityPubActor, type: ActorImageType) {
248 const mimetypes = MIMETYPES.IMAGE
249 const icon = type === ActorImageType.AVATAR
250 ? actorJSON.icon
251 : actorJSON.image
252
253 if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined
254
255 let extension: string
256
257 if (icon.mediaType) {
258 extension = mimetypes.MIMETYPE_EXT[icon.mediaType]
259 } else {
260 const tmp = extname(icon.url)
261
262 if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp
263 }
264
265 if (!extension) return undefined
266
267 return {
268 name: uuidv4() + extension,
269 fileUrl: icon.url,
270 height: icon.height,
271 width: icon.width,
272 type
273 }
274}
275
276async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) {
277 // Don't fetch ourselves
278 const serverActor = await getServerActor()
279 if (serverActor.id === actor.id) {
280 logger.error('Cannot fetch our own outbox!')
281 return undefined
282 }
283
284 const payload = {
285 uri: actor.outboxUrl,
286 type: 'activity' as 'activity'
287 }
288
289 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
290}
291
292async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (
293 actorArg: T,
294 fetchedType: ActorFetchByUrlType
295): Promise<{ actor: T | MActorFull, refreshed: boolean }> {
296 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
297
298 // We need more attributes
299 const actor = fetchedType === 'all'
300 ? actorArg as MActorFull
301 : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
302
303 try {
304 let actorUrl: string
305 try {
306 actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
307 } catch (err) {
308 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
309 actorUrl = actor.url
310 }
311
312 const { result } = await fetchRemoteActor(actorUrl)
313
314 if (result === undefined) {
315 logger.warn('Cannot fetch remote actor in refresh actor.')
316 return { actor, refreshed: false }
317 }
318
319 return sequelizeTypescript.transaction(async t => {
320 updateInstanceWithAnother(actor, result.actor)
321
322 await updateActorImageInstance(actor, ActorImageType.AVATAR, result.avatar, t)
323 await updateActorImageInstance(actor, ActorImageType.BANNER, result.banner, t)
324
325 // Force update
326 actor.setDataValue('updatedAt', new Date())
327 await actor.save({ transaction: t })
328
329 if (actor.Account) {
330 actor.Account.name = result.name
331 actor.Account.description = result.summary
332
333 await actor.Account.save({ transaction: t })
334 } else if (actor.VideoChannel) {
335 actor.VideoChannel.name = result.name
336 actor.VideoChannel.description = result.summary
337 actor.VideoChannel.support = result.support
338
339 await actor.VideoChannel.save({ transaction: t })
340 }
341
342 return { refreshed: true, actor }
343 })
344 } catch (err) {
345 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
346 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
347 actor.Account
348 ? await actor.Account.destroy()
349 : await actor.VideoChannel.destroy()
350
351 return { actor: undefined, refreshed: false }
352 }
353
354 logger.warn('Cannot refresh actor %s.', actor.url, { err })
355 return { actor, refreshed: false }
356 }
357}
358
359export {
360 getOrCreateActorAndServerAndModel,
361 buildActorInstance,
362 generateAndSaveActorKeys,
363 fetchActorTotalItems,
364 getImageInfoIfExists,
365 updateActorInstance,
366 deleteActorImageInstance,
367 refreshActorIfNeeded,
368 updateActorImageInstance,
369 addFetchOutboxJob
370}
371
372// ---------------------------------------------------------------------------
373
374function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) {
375 const id = imageModel
376 ? imageModel.id
377 : null
378
379 if (type === ActorImageType.AVATAR) {
380 actorModel.avatarId = id
381 actorModel.Avatar = imageModel
382 } else {
383 actorModel.bannerId = id
384 actorModel.Banner = imageModel
385 }
386
387 return actorModel
388}
389
390function saveActorAndServerAndModelIfNotExist (
391 result: FetchRemoteActorResult,
392 ownerActor?: MActorFullActor,
393 t?: Transaction
394): Bluebird<MActorFullActor> | Promise<MActorFullActor> {
395 const actor = result.actor
396
397 if (t !== undefined) return save(t)
398
399 return sequelizeTypescript.transaction(t => save(t))
400
401 async function save (t: Transaction) {
402 const actorHost = new URL(actor.url).host
403
404 const serverOptions = {
405 where: {
406 host: actorHost
407 },
408 defaults: {
409 host: actorHost
410 },
411 transaction: t
412 }
413 const [ server ] = await ServerModel.findOrCreate(serverOptions)
414
415 // Save our new account in database
416 actor.serverId = server.id
417
418 // Avatar?
419 if (result.avatar) {
420 const avatar = await ActorImageModel.create({
421 filename: result.avatar.name,
422 fileUrl: result.avatar.fileUrl,
423 width: result.avatar.width,
424 height: result.avatar.height,
425 onDisk: false,
426 type: ActorImageType.AVATAR
427 }, { transaction: t })
428
429 actor.avatarId = avatar.id
430 }
431
432 // Banner?
433 if (result.banner) {
434 const banner = await ActorImageModel.create({
435 filename: result.banner.name,
436 fileUrl: result.banner.fileUrl,
437 width: result.banner.width,
438 height: result.banner.height,
439 onDisk: false,
440 type: ActorImageType.BANNER
441 }, { transaction: t })
442
443 actor.bannerId = banner.id
444 }
445
446 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
447 // (which could be false in a retried query)
448 const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({
449 defaults: actor.toJSON(),
450 where: {
451 [Op.or]: [
452 {
453 url: actor.url
454 },
455 {
456 serverId: actor.serverId,
457 preferredUsername: actor.preferredUsername
458 }
459 ]
460 },
461 transaction: t
462 })
463
464 // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards
465 if (created !== true && actorCreated.url !== actor.url) {
466 // Only fix http://example.com/account/djidane to https://example.com/account/djidane
467 if (actorCreated.url.replace(/^http:\/\//, '') !== actor.url.replace(/^https:\/\//, '')) {
468 throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${actor.url}`)
469 }
470
471 actorCreated.url = actor.url
472 await actorCreated.save({ transaction: t })
473 }
474
475 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
476 actorCreated.Account = await saveAccount(actorCreated, result, t) as MAccountDefault
477 actorCreated.Account.Actor = actorCreated
478 } else if (actorCreated.type === 'Group') { // Video channel
479 const channel = await saveVideoChannel(actorCreated, result, ownerActor, t)
480 actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: ownerActor.Account })
481 }
482
483 actorCreated.Server = server
484
485 return actorCreated
486 }
487}
488
489type ImageResult = {
490 name: string
491 fileUrl: string
492 height: number
493 width: number
494}
495
496type FetchRemoteActorResult = {
497 actor: MActor
498 name: string
499 summary: string
500 support?: string
501 playlists?: string
502 avatar?: ImageResult
503 banner?: ImageResult
504 attributedTo: ActivityPubAttributedTo[]
505}
506async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
507 logger.info('Fetching remote actor %s.', actorUrl)
508
509 const requestResult = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
510 const actorJSON = requestResult.body
511
512 if (sanitizeAndCheckActorObject(actorJSON) === false) {
513 logger.debug('Remote actor JSON is not valid.', { actorJSON })
514 return { result: undefined, statusCode: requestResult.statusCode }
515 }
516
517 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
518 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
519 return { result: undefined, statusCode: requestResult.statusCode }
520 }
521
522 const followersCount = await fetchActorTotalItems(actorJSON.followers)
523 const followingCount = await fetchActorTotalItems(actorJSON.following)
524
525 const actor = new ActorModel({
526 type: actorJSON.type,
527 preferredUsername: actorJSON.preferredUsername,
528 url: actorJSON.id,
529 publicKey: actorJSON.publicKey.publicKeyPem,
530 privateKey: null,
531 followersCount: followersCount,
532 followingCount: followingCount,
533 inboxUrl: actorJSON.inbox,
534 outboxUrl: actorJSON.outbox,
535 followersUrl: actorJSON.followers,
536 followingUrl: actorJSON.following,
537
538 sharedInboxUrl: actorJSON.endpoints?.sharedInbox
539 ? actorJSON.endpoints.sharedInbox
540 : null
541 })
542
543 const avatarInfo = getImageInfoIfExists(actorJSON, ActorImageType.AVATAR)
544 const bannerInfo = getImageInfoIfExists(actorJSON, ActorImageType.BANNER)
545
546 const name = actorJSON.name || actorJSON.preferredUsername
547 return {
548 statusCode: requestResult.statusCode,
549 result: {
550 actor,
551 name,
552 avatar: avatarInfo,
553 banner: bannerInfo,
554 summary: actorJSON.summary,
555 support: actorJSON.support,
556 playlists: actorJSON.playlists,
557 attributedTo: actorJSON.attributedTo
558 }
559 }
560}
561
562async function saveAccount (actor: MActorId, result: FetchRemoteActorResult, t: Transaction) {
563 const [ accountCreated ] = await AccountModel.findOrCreate({
564 defaults: {
565 name: result.name,
566 description: result.summary,
567 actorId: actor.id
568 },
569 where: {
570 actorId: actor.id
571 },
572 transaction: t
573 })
574
575 return accountCreated as MAccount
576}
577
578async function saveVideoChannel (actor: MActorId, result: FetchRemoteActorResult, ownerActor: MActorAccountId, t: Transaction) {
579 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
580 defaults: {
581 name: result.name,
582 description: result.summary,
583 support: result.support,
584 actorId: actor.id,
585 accountId: ownerActor.Account.id
586 },
587 where: {
588 actorId: actor.id
589 },
590 transaction: t
591 })
592
593 return videoChannelCreated as MChannel
594}
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts
new file mode 100644
index 000000000..8681ea02a
--- /dev/null
+++ b/server/lib/activitypub/actors/get.ts
@@ -0,0 +1,122 @@
1
2import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub'
3import { retryTransactionWrapper } from '@server/helpers/database-utils'
4import { logger } from '@server/helpers/logger'
5import { JobQueue } from '@server/lib/job-queue'
6import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders'
7import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models'
8import { ActivityPubActor } from '@shared/models'
9import { refreshActorIfNeeded } from './refresh'
10import { APActorCreator, fetchRemoteActor } from './shared'
11
12function getOrCreateAPActor (
13 activityActor: string | ActivityPubActor,
14 fetchType: 'all',
15 recurseIfNeeded?: boolean,
16 updateCollections?: boolean
17): Promise<MActorFullActor>
18
19function getOrCreateAPActor (
20 activityActor: string | ActivityPubActor,
21 fetchType?: 'association-ids',
22 recurseIfNeeded?: boolean,
23 updateCollections?: boolean
24): Promise<MActorAccountChannelId>
25
26async function getOrCreateAPActor (
27 activityActor: string | ActivityPubActor,
28 fetchType: ActorLoadByUrlType = 'association-ids',
29 recurseIfNeeded = true,
30 updateCollections = false
31): Promise<MActorFullActor | MActorAccountChannelId> {
32 const actorUrl = getAPId(activityActor)
33 let actor = await loadActorFromDB(actorUrl, fetchType)
34
35 let created = false
36 let accountPlaylistsUrl: string
37
38 // We don't have this actor in our database, fetch it on remote
39 if (!actor) {
40 const { actorObject } = await fetchRemoteActor(actorUrl)
41 if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
42
43 // actorUrl is just an alias/rediraction, so process object id instead
44 if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections)
45
46 // Create the attributed to actor
47 // In PeerTube a video channel is owned by an account
48 let ownerActor: MActorFullActor
49 if (recurseIfNeeded === true && actorObject.type === 'Group') {
50 ownerActor = await getOrCreateAPOwner(actorObject, actorUrl)
51 }
52
53 const creator = new APActorCreator(actorObject, ownerActor)
54 actor = await retryTransactionWrapper(creator.create.bind(creator))
55 created = true
56 accountPlaylistsUrl = actorObject.playlists
57 }
58
59 if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor
60 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
61
62 const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType })
63 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
64
65 await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections)
66 await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl)
67
68 return actorRefreshed
69}
70
71// ---------------------------------------------------------------------------
72
73export {
74 getOrCreateAPActor
75}
76
77// ---------------------------------------------------------------------------
78
79async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) {
80 let actor = await loadActorByUrl(actorUrl, fetchType)
81
82 // Orphan actor (not associated to an account of channel) so recreate it
83 if (actor && (!actor.Account && !actor.VideoChannel)) {
84 await actor.destroy()
85 actor = null
86 }
87
88 return actor
89}
90
91function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) {
92 const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person')
93 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl)
94
95 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
96 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
97 }
98
99 try {
100 // Don't recurse another time
101 const recurseIfNeeded = false
102 return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded)
103 } catch (err) {
104 logger.error('Cannot get or create account attributed to video channel ' + actorUrl)
105 throw new Error(err)
106 }
107}
108
109async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) {
110 if ((created === true || refreshed === true) && updateCollections === true) {
111 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
112 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
113 }
114}
115
116async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) {
117 // We created a new account: fetch the playlists
118 if (created === true && actor.Account && accountPlaylistsUrl) {
119 const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' }
120 await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
121 }
122}
diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts
new file mode 100644
index 000000000..443ad0a63
--- /dev/null
+++ b/server/lib/activitypub/actors/image.ts
@@ -0,0 +1,94 @@
1import { Transaction } from 'sequelize/types'
2import { logger } from '@server/helpers/logger'
3import { ActorImageModel } from '@server/models/actor/actor-image'
4import { MActorImage, MActorImages } from '@server/types/models'
5import { ActorImageType } from '@shared/models'
6
7type ImageInfo = {
8 name: string
9 fileUrl: string
10 height: number
11 width: number
12 onDisk?: boolean
13}
14
15async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) {
16 const oldImageModel = type === ActorImageType.AVATAR
17 ? actor.Avatar
18 : actor.Banner
19
20 if (oldImageModel) {
21 // Don't update the avatar if the file URL did not change
22 if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor
23
24 try {
25 await oldImageModel.destroy({ transaction: t })
26
27 setActorImage(actor, type, null)
28 } catch (err) {
29 logger.error('Cannot remove old actor image of actor %s.', actor.url, { err })
30 }
31 }
32
33 if (imageInfo) {
34 const imageModel = await ActorImageModel.create({
35 filename: imageInfo.name,
36 onDisk: imageInfo.onDisk ?? false,
37 fileUrl: imageInfo.fileUrl,
38 height: imageInfo.height,
39 width: imageInfo.width,
40 type
41 }, { transaction: t })
42
43 setActorImage(actor, type, imageModel)
44 }
45
46 return actor
47}
48
49async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) {
50 try {
51 if (type === ActorImageType.AVATAR) {
52 await actor.Avatar.destroy({ transaction: t })
53
54 actor.avatarId = null
55 actor.Avatar = null
56 } else {
57 await actor.Banner.destroy({ transaction: t })
58
59 actor.bannerId = null
60 actor.Banner = null
61 }
62 } catch (err) {
63 logger.error('Cannot remove old image of actor %s.', actor.url, { err })
64 }
65
66 return actor
67}
68
69// ---------------------------------------------------------------------------
70
71export {
72 ImageInfo,
73
74 updateActorImageInstance,
75 deleteActorImageInstance
76}
77
78// ---------------------------------------------------------------------------
79
80function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) {
81 const id = imageModel
82 ? imageModel.id
83 : null
84
85 if (type === ActorImageType.AVATAR) {
86 actorModel.avatarId = id
87 actorModel.Avatar = imageModel
88 } else {
89 actorModel.bannerId = id
90 actorModel.Banner = imageModel
91 }
92
93 return actorModel
94}
diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts
new file mode 100644
index 000000000..5ee2a6f1a
--- /dev/null
+++ b/server/lib/activitypub/actors/index.ts
@@ -0,0 +1,6 @@
1export * from './get'
2export * from './image'
3export * from './keys'
4export * from './refresh'
5export * from './updater'
6export * from './webfinger'
diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts
new file mode 100644
index 000000000..c3d18abd8
--- /dev/null
+++ b/server/lib/activitypub/actors/keys.ts
@@ -0,0 +1,16 @@
1import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto'
2import { MActor } from '@server/types/models'
3
4// Set account keys, this could be long so process after the account creation and do not block the client
5async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
6 const { publicKey, privateKey } = await createPrivateAndPublicKeys()
7
8 actor.publicKey = publicKey
9 actor.privateKey = privateKey
10
11 return actor.save()
12}
13
14export {
15 generateAndSaveActorKeys
16}
diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts
new file mode 100644
index 000000000..b2fe3932f
--- /dev/null
+++ b/server/lib/activitypub/actors/refresh.ts
@@ -0,0 +1,81 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { PromiseCache } from '@server/helpers/promise-cache'
3import { PeerTubeRequestError } from '@server/helpers/requests'
4import { ActorLoadByUrlType } from '@server/lib/model-loaders'
5import { ActorModel } from '@server/models/actor/actor'
6import { MActorAccountChannelId, MActorFull } from '@server/types/models'
7import { HttpStatusCode } from '@shared/core-utils'
8import { fetchRemoteActor } from './shared'
9import { APActorUpdater } from './updater'
10import { getUrlFromWebfinger } from './webfinger'
11
12type RefreshResult <T> = Promise<{ actor: T | MActorFull, refreshed: boolean }>
13
14type RefreshOptions <T> = {
15 actor: T
16 fetchedType: ActorLoadByUrlType
17}
18
19const promiseCache = new PromiseCache(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url)
20
21function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> {
22 const actorArg = options.actor
23 if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false })
24
25 return promiseCache.run(options)
26}
27
28export {
29 refreshActorIfNeeded
30}
31
32// ---------------------------------------------------------------------------
33
34async function doRefresh <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <MActorFull> {
35 const { actor: actorArg, fetchedType } = options
36
37 // We need more attributes
38 const actor = fetchedType === 'all'
39 ? actorArg as MActorFull
40 : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
41
42 const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url)
43
44 logger.info('Refreshing actor %s.', actor.url, lTags())
45
46 try {
47 const actorUrl = await getActorUrl(actor)
48 const { actorObject } = await fetchRemoteActor(actorUrl)
49
50 if (actorObject === undefined) {
51 logger.warn('Cannot fetch remote actor in refresh actor.')
52 return { actor, refreshed: false }
53 }
54
55 const updater = new APActorUpdater(actorObject, actor)
56 await updater.update()
57
58 return { refreshed: true, actor }
59 } catch (err) {
60 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
61 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags())
62
63 actor.Account
64 ? await actor.Account.destroy()
65 : await actor.VideoChannel.destroy()
66
67 return { actor: undefined, refreshed: false }
68 }
69
70 logger.warn('Cannot refresh actor %s.', actor.url, { err, ...lTags() })
71 return { actor, refreshed: false }
72 }
73}
74
75function getActorUrl (actor: MActorFull) {
76 return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
77 .catch(err => {
78 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
79 return actor.url
80 })
81}
diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts
new file mode 100644
index 000000000..999aed97d
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/creator.ts
@@ -0,0 +1,149 @@
1import { Op, Transaction } from 'sequelize'
2import { sequelizeTypescript } from '@server/initializers/database'
3import { AccountModel } from '@server/models/account/account'
4import { ActorModel } from '@server/models/actor/actor'
5import { ServerModel } from '@server/models/server/server'
6import { VideoChannelModel } from '@server/models/video/video-channel'
7import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models'
8import { ActivityPubActor, ActorImageType } from '@shared/models'
9import { updateActorImageInstance } from '../image'
10import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImageInfoFromObject } from './object-to-model-attributes'
11import { fetchActorFollowsCount } from './url-to-object'
12
13export class APActorCreator {
14
15 constructor (
16 private readonly actorObject: ActivityPubActor,
17 private readonly ownerActor?: MActorFullActor
18 ) {
19
20 }
21
22 async create (): Promise<MActorFullActor> {
23 const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject)
24
25 const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount))
26
27 return sequelizeTypescript.transaction(async t => {
28 const server = await this.setServer(actorInstance, t)
29
30 await this.setImageIfNeeded(actorInstance, ActorImageType.AVATAR, t)
31 await this.setImageIfNeeded(actorInstance, ActorImageType.BANNER, t)
32
33 const { actorCreated, created } = await this.saveActor(actorInstance, t)
34
35 await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t)
36
37 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance
38 actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault
39 actorCreated.Account.Actor = actorCreated
40 }
41
42 if (actorCreated.type === 'Group') { // Video channel
43 const channel = await this.saveVideoChannel(actorCreated, t)
44 actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account })
45 }
46
47 actorCreated.Server = server
48
49 return actorCreated
50 })
51 }
52
53 private async setServer (actor: MActor, t: Transaction) {
54 const actorHost = new URL(actor.url).host
55
56 const serverOptions = {
57 where: {
58 host: actorHost
59 },
60 defaults: {
61 host: actorHost
62 },
63 transaction: t
64 }
65 const [ server ] = await ServerModel.findOrCreate(serverOptions)
66
67 // Save our new account in database
68 actor.serverId = server.id
69
70 return server as MServer
71 }
72
73 private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) {
74 const imageInfo = getImageInfoFromObject(this.actorObject, type)
75 if (!imageInfo) return
76
77 return updateActorImageInstance(actor as MActorImages, type, imageInfo, t)
78 }
79
80 private async saveActor (actor: MActor, t: Transaction) {
81 // Force the actor creation using findOrCreate() instead of save()
82 // Sometimes Sequelize skips the save() when it thinks the instance already exists
83 // (which could be false in a retried query)
84 const [ actorCreated, created ] = await ActorModel.findOrCreate<MActorFullActor>({
85 defaults: actor.toJSON(),
86 where: {
87 [Op.or]: [
88 {
89 url: actor.url
90 },
91 {
92 serverId: actor.serverId,
93 preferredUsername: actor.preferredUsername
94 }
95 ]
96 },
97 transaction: t
98 })
99
100 return { actorCreated, created }
101 }
102
103 private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) {
104 // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards
105 if (created !== true && actorCreated.url !== newActor.url) {
106 // Only fix http://example.com/account/djidane to https://example.com/account/djidane
107 if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) {
108 throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`)
109 }
110
111 actorCreated.url = newActor.url
112 await actorCreated.save({ transaction: t })
113 }
114 }
115
116 private async saveAccount (actor: MActorId, t: Transaction) {
117 const [ accountCreated ] = await AccountModel.findOrCreate({
118 defaults: {
119 name: getActorDisplayNameFromObject(this.actorObject),
120 description: this.actorObject.summary,
121 actorId: actor.id
122 },
123 where: {
124 actorId: actor.id
125 },
126 transaction: t
127 })
128
129 return accountCreated as MAccount
130 }
131
132 private async saveVideoChannel (actor: MActorId, t: Transaction) {
133 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
134 defaults: {
135 name: getActorDisplayNameFromObject(this.actorObject),
136 description: this.actorObject.summary,
137 support: this.actorObject.support,
138 actorId: actor.id,
139 accountId: this.ownerActor.Account.id
140 },
141 where: {
142 actorId: actor.id
143 },
144 transaction: t
145 })
146
147 return videoChannelCreated as MChannel
148 }
149}
diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts
new file mode 100644
index 000000000..52af1a8e1
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/index.ts
@@ -0,0 +1,3 @@
1export * from './creator'
2export * from './object-to-model-attributes'
3export * from './url-to-object'
diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..1612b3ad0
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
@@ -0,0 +1,70 @@
1import { getLowercaseExtension } from '@server/helpers/core-utils'
2import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
3import { buildUUID } from '@server/helpers/uuid'
4import { MIMETYPES } from '@server/initializers/constants'
5import { ActorModel } from '@server/models/actor/actor'
6import { FilteredModelAttributes } from '@server/types'
7import { ActivityPubActor, ActorImageType } from '@shared/models'
8
9function getActorAttributesFromObject (
10 actorObject: ActivityPubActor,
11 followersCount: number,
12 followingCount: number
13): FilteredModelAttributes<ActorModel> {
14 return {
15 type: actorObject.type,
16 preferredUsername: actorObject.preferredUsername,
17 url: actorObject.id,
18 publicKey: actorObject.publicKey.publicKeyPem,
19 privateKey: null,
20 followersCount,
21 followingCount,
22 inboxUrl: actorObject.inbox,
23 outboxUrl: actorObject.outbox,
24 followersUrl: actorObject.followers,
25 followingUrl: actorObject.following,
26
27 sharedInboxUrl: actorObject.endpoints?.sharedInbox
28 ? actorObject.endpoints.sharedInbox
29 : null
30 }
31}
32
33function getImageInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) {
34 const mimetypes = MIMETYPES.IMAGE
35 const icon = type === ActorImageType.AVATAR
36 ? actorObject.icon
37 : actorObject.image
38
39 if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined
40
41 let extension: string
42
43 if (icon.mediaType) {
44 extension = mimetypes.MIMETYPE_EXT[icon.mediaType]
45 } else {
46 const tmp = getLowercaseExtension(icon.url)
47
48 if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp
49 }
50
51 if (!extension) return undefined
52
53 return {
54 name: buildUUID() + extension,
55 fileUrl: icon.url,
56 height: icon.height,
57 width: icon.width,
58 type
59 }
60}
61
62function getActorDisplayNameFromObject (actorObject: ActivityPubActor) {
63 return actorObject.name || actorObject.preferredUsername
64}
65
66export {
67 getActorAttributesFromObject,
68 getImageInfoFromObject,
69 getActorDisplayNameFromObject
70}
diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts
new file mode 100644
index 000000000..f4f16b044
--- /dev/null
+++ b/server/lib/activitypub/actors/shared/url-to-object.ts
@@ -0,0 +1,54 @@
1
2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor'
4import { logger } from '@server/helpers/logger'
5import { doJSONRequest } from '@server/helpers/requests'
6import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models'
7
8async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> {
9 logger.info('Fetching remote actor %s.', actorUrl)
10
11 const { body, statusCode } = await doJSONRequest<ActivityPubActor>(actorUrl, { activityPub: true })
12
13 if (sanitizeAndCheckActorObject(body) === false) {
14 logger.debug('Remote actor JSON is not valid.', { actorJSON: body })
15 return { actorObject: undefined, statusCode: statusCode }
16 }
17
18 if (checkUrlsSameHost(body.id, actorUrl) !== true) {
19 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id)
20 return { actorObject: undefined, statusCode: statusCode }
21 }
22
23 return {
24 statusCode,
25
26 actorObject: body
27 }
28}
29
30async function fetchActorFollowsCount (actorObject: ActivityPubActor) {
31 const followersCount = await fetchActorTotalItems(actorObject.followers)
32 const followingCount = await fetchActorTotalItems(actorObject.following)
33
34 return { followersCount, followingCount }
35}
36
37// ---------------------------------------------------------------------------
38export {
39 fetchActorFollowsCount,
40 fetchRemoteActor
41}
42
43// ---------------------------------------------------------------------------
44
45async function fetchActorTotalItems (url: string) {
46 try {
47 const { body } = await doJSONRequest<ActivityPubOrderedCollection<unknown>>(url, { activityPub: true })
48
49 return body.totalItems || 0
50 } catch (err) {
51 logger.warn('Cannot fetch remote actor count %s.', url, { err })
52 return 0
53 }
54}
diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts
new file mode 100644
index 000000000..de5e03eee
--- /dev/null
+++ b/server/lib/activitypub/actors/updater.ts
@@ -0,0 +1,90 @@
1import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
2import { logger } from '@server/helpers/logger'
3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models'
5import { ActivityPubActor, ActorImageType } from '@shared/models'
6import { updateActorImageInstance } from './image'
7import { fetchActorFollowsCount } from './shared'
8import { getImageInfoFromObject } from './shared/object-to-model-attributes'
9
10export class APActorUpdater {
11
12 private accountOrChannel: MAccount | MChannel
13
14 private readonly actorFieldsSave: object
15 private readonly accountOrChannelFieldsSave: object
16
17 constructor (
18 private readonly actorObject: ActivityPubActor,
19 private readonly actor: MActorFull
20 ) {
21 this.actorFieldsSave = this.actor.toJSON()
22
23 if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel
24 else this.accountOrChannel = this.actor.Account
25
26 this.accountOrChannelFieldsSave = this.accountOrChannel.toJSON()
27 }
28
29 async update () {
30 const avatarInfo = getImageInfoFromObject(this.actorObject, ActorImageType.AVATAR)
31 const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER)
32
33 try {
34 await this.updateActorInstance(this.actor, this.actorObject)
35
36 this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername
37 this.accountOrChannel.description = this.actorObject.summary
38
39 if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support
40
41 await runInReadCommittedTransaction(async t => {
42 await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t)
43 await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t)
44 })
45
46 await runInReadCommittedTransaction(async t => {
47 await this.actor.save({ transaction: t })
48 await this.accountOrChannel.save({ transaction: t })
49 })
50
51 logger.info('Remote account %s updated', this.actorObject.url)
52 } catch (err) {
53 if (this.actor !== undefined && this.actorFieldsSave !== undefined) {
54 resetSequelizeInstance(this.actor, this.actorFieldsSave)
55 }
56
57 if (this.accountOrChannel !== undefined && this.accountOrChannelFieldsSave !== undefined) {
58 resetSequelizeInstance(this.accountOrChannel, this.accountOrChannelFieldsSave)
59 }
60
61 // This is just a debug because we will retry the insert
62 logger.debug('Cannot update the remote account.', { err })
63 throw err
64 }
65 }
66
67 private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) {
68 const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject)
69
70 actorInstance.type = actorObject.type
71 actorInstance.preferredUsername = actorObject.preferredUsername
72 actorInstance.url = actorObject.id
73 actorInstance.publicKey = actorObject.publicKey.publicKeyPem
74 actorInstance.followersCount = followersCount
75 actorInstance.followingCount = followingCount
76 actorInstance.inboxUrl = actorObject.inbox
77 actorInstance.outboxUrl = actorObject.outbox
78 actorInstance.followersUrl = actorObject.followers
79 actorInstance.followingUrl = actorObject.following
80
81 if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published)
82
83 if (actorObject.endpoints?.sharedInbox) {
84 actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox
85 }
86
87 // Force actor update
88 actorInstance.changed('updatedAt', true)
89 }
90}
diff --git a/server/lib/activitypub/actors/webfinger.ts b/server/lib/activitypub/actors/webfinger.ts
new file mode 100644
index 000000000..1c7ec4717
--- /dev/null
+++ b/server/lib/activitypub/actors/webfinger.ts
@@ -0,0 +1,67 @@
1import * as WebFinger from 'webfinger.js'
2import { isProdInstance } from '@server/helpers/core-utils'
3import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
4import { REQUEST_TIMEOUT, WEBSERVER } from '@server/initializers/constants'
5import { ActorModel } from '@server/models/actor/actor'
6import { MActorFull } from '@server/types/models'
7import { WebFingerData } from '@shared/models'
8
9const webfinger = new WebFinger({
10 webfist_fallback: false,
11 tls_only: isProdInstance(),
12 uri_fallback: false,
13 request_timeout: REQUEST_TIMEOUT
14})
15
16async function loadActorUrlOrGetFromWebfinger (uriArg: string) {
17 // Handle strings like @toto@example.com
18 const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg
19
20 const [ name, host ] = uri.split('@')
21 let actor: MActorFull
22
23 if (!host || host === WEBSERVER.HOST) {
24 actor = await ActorModel.loadLocalByName(name)
25 } else {
26 actor = await ActorModel.loadByNameAndHost(name, host)
27 }
28
29 if (actor) return actor.url
30
31 return getUrlFromWebfinger(uri)
32}
33
34async function getUrlFromWebfinger (uri: string) {
35 const webfingerData: WebFingerData = await webfingerLookup(uri)
36 return getLinkOrThrow(webfingerData)
37}
38
39// ---------------------------------------------------------------------------
40
41export {
42 getUrlFromWebfinger,
43 loadActorUrlOrGetFromWebfinger
44}
45
46// ---------------------------------------------------------------------------
47
48function getLinkOrThrow (webfingerData: WebFingerData) {
49 if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.')
50
51 const selfLink = webfingerData.links.find(l => l.rel === 'self')
52 if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) {
53 throw new Error('Cannot find self link or href is not a valid URL.')
54 }
55
56 return selfLink.href
57}
58
59function webfingerLookup (nameWithHost: string) {
60 return new Promise<WebFingerData>((res, rej) => {
61 webfinger.lookup(nameWithHost, (err, p) => {
62 if (err) return rej(err)
63
64 return res(p.object)
65 })
66 })
67}
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index 2986714d3..d0558f191 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -1,7 +1,7 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience } from '../../../shared/models/activitypub' 2import { ActivityAudience } from '../../../shared/models/activitypub'
3import { ACTIVITY_PUB } from '../../initializers/constants' 3import { ACTIVITY_PUB } from '../../initializers/constants'
4import { ActorModel } from '../../models/activitypub/actor' 4import { ActorModel } from '../../models/actor/actor'
5import { VideoModel } from '../../models/video/video' 5import { VideoModel } from '../../models/video/video'
6import { VideoShareModel } from '../../models/video/video-share' 6import { VideoShareModel } from '../../models/video/video-share'
7import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '../../types/models' 7import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '../../types/models'
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 2e6dd34e0..a16d2cd93 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,54 +1,27 @@
1import { CacheFileObject } from '../../../shared/index'
2import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
3import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
4import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' 2import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models'
3import { CacheFileObject } from '../../../shared/index'
4import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6 6
7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { 7async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
8 8 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14 9
15 return { 10 if (redundancyModel) {
16 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, 11 return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 } 12 }
24 13
25 const url = cacheFileObject.url 14 return createCacheFile(cacheFileObject, video, byActor, t)
26 const videoFile = video.VideoFiles.find(f => {
27 return f.resolution === url.height && f.fps === url.fps
28 })
29
30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
31
32 return {
33 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
34 url: cacheFileObject.id,
35 fileUrl: url.href,
36 strategy: null,
37 videoFileId: videoFile.id,
38 actorId: byActor.id
39 }
40} 15}
41 16
42async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { 17// ---------------------------------------------------------------------------
43 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
44 18
45 if (!redundancyModel) { 19export {
46 await createCacheFile(cacheFileObject, video, byActor, t) 20 createOrUpdateCacheFile
47 } else {
48 await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
49 }
50} 21}
51 22
23// ---------------------------------------------------------------------------
24
52function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { 25function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
53 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) 26 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
54 27
@@ -74,9 +47,37 @@ function updateCacheFile (
74 return redundancyModel.save({ transaction: t }) 47 return redundancyModel.save({ transaction: t })
75} 48}
76 49
77export { 50function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) {
78 createOrUpdateCacheFile, 51
79 createCacheFile, 52 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
80 updateCacheFile, 53 const url = cacheFileObject.url
81 cacheFileActivityObjectToDBAttributes 54
55 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
56 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
57
58 return {
59 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
60 url: cacheFileObject.id,
61 fileUrl: url.href,
62 strategy: null,
63 videoStreamingPlaylistId: playlist.id,
64 actorId: byActor.id
65 }
66 }
67
68 const url = cacheFileObject.url
69 const videoFile = video.VideoFiles.find(f => {
70 return f.resolution === url.height && f.fps === url.fps
71 })
72
73 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
74
75 return {
76 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
77 url: cacheFileObject.id,
78 fileUrl: url.href,
79 strategy: null,
80 videoFileId: videoFile.id,
81 actorId: byActor.id
82 }
82} 83}
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 278abf7de..cd117f571 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -3,7 +3,7 @@ import { URL } from 'url'
3import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' 3import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
4import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
5import { doJSONRequest } from '../../helpers/requests' 5import { doJSONRequest } from '../../helpers/requests'
6import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' 6import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants'
7 7
8type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) 8type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
9type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) 9type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
@@ -13,10 +13,7 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
13 13
14 logger.info('Crawling ActivityPub data on %s.', url) 14 logger.info('Crawling ActivityPub data on %s.', url)
15 15
16 const options = { 16 const options = { activityPub: true }
17 activityPub: true,
18 timeout: REQUEST_TIMEOUT
19 }
20 17
21 const startDate = new Date() 18 const startDate = new Date()
22 19
diff --git a/server/lib/activitypub/follow.ts b/server/lib/activitypub/follow.ts
index 351499bd1..c1bd667e0 100644
--- a/server/lib/activitypub/follow.ts
+++ b/server/lib/activitypub/follow.ts
@@ -1,12 +1,13 @@
1import { MActorFollowActors } from '../../types/models' 1import { Transaction } from 'sequelize'
2import { getServerActor } from '@server/models/application/application'
3import { logger } from '../../helpers/logger'
2import { CONFIG } from '../../initializers/config' 4import { CONFIG } from '../../initializers/config'
3import { SERVER_ACTOR_NAME } from '../../initializers/constants' 5import { SERVER_ACTOR_NAME } from '../../initializers/constants'
4import { JobQueue } from '../job-queue'
5import { logger } from '../../helpers/logger'
6import { ServerModel } from '../../models/server/server' 6import { ServerModel } from '../../models/server/server'
7import { getServerActor } from '@server/models/application/application' 7import { MActorFollowActors } from '../../types/models'
8import { JobQueue } from '../job-queue'
8 9
9async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) { 10async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) {
10 if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return 11 if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return
11 12
12 const follower = actorFollow.ActorFollower 13 const follower = actorFollow.ActorFollower
@@ -16,7 +17,7 @@ async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) {
16 17
17 const me = await getServerActor() 18 const me = await getServerActor()
18 19
19 const server = await ServerModel.load(follower.serverId) 20 const server = await ServerModel.load(follower.serverId, transaction)
20 const host = server.host 21 const host = server.host
21 22
22 const payload = { 23 const payload = {
diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts
new file mode 100644
index 000000000..ecdc33a77
--- /dev/null
+++ b/server/lib/activitypub/outbox.ts
@@ -0,0 +1,24 @@
1import { logger } from '@server/helpers/logger'
2import { ActorModel } from '@server/models/actor/actor'
3import { getServerActor } from '@server/models/application/application'
4import { JobQueue } from '../job-queue'
5
6async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) {
7 // Don't fetch ourselves
8 const serverActor = await getServerActor()
9 if (serverActor.id === actor.id) {
10 logger.error('Cannot fetch our own outbox!')
11 return undefined
12 }
13
14 const payload = {
15 uri: actor.outboxUrl,
16 type: 'activity' as 'activity'
17 }
18
19 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
20}
21
22export {
23 addFetchOutboxJob
24}
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
deleted file mode 100644
index 7166c68a6..000000000
--- a/server/lib/activitypub/playlist.ts
+++ /dev/null
@@ -1,204 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
3import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
4import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
5import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
6import { checkUrlsSameHost } from '../../helpers/activitypub'
7import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
8import { isArray } from '../../helpers/custom-validators/misc'
9import { logger } from '../../helpers/logger'
10import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
11import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
12import { sequelizeTypescript } from '../../initializers/database'
13import { VideoPlaylistModel } from '../../models/video/video-playlist'
14import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
15import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
16import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
17import { FilteredModelAttributes } from '../../types/sequelize'
18import { createPlaylistMiniatureFromUrl } from '../thumbnail'
19import { getOrCreateActorAndServerAndModel } from './actor'
20import { crawlCollectionPage } from './crawl'
21import { getOrCreateVideoAndAccountAndChannel } from './videos'
22
23function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
24 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
25 ? VideoPlaylistPrivacy.PUBLIC
26 : VideoPlaylistPrivacy.UNLISTED
27
28 return {
29 name: playlistObject.name,
30 description: playlistObject.content,
31 privacy,
32 url: playlistObject.id,
33 uuid: playlistObject.uuid,
34 ownerAccountId: byAccount.id,
35 videoChannelId: null,
36 createdAt: new Date(playlistObject.published),
37 updatedAt: new Date(playlistObject.updated)
38 }
39}
40
41function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
42 return {
43 position: elementObject.position,
44 url: elementObject.id,
45 startTimestamp: elementObject.startTimestamp || null,
46 stopTimestamp: elementObject.stopTimestamp || null,
47 videoPlaylistId: videoPlaylist.id,
48 videoId: video.id
49 }
50}
51
52async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
53 await Bluebird.map(playlistUrls, async playlistUrl => {
54 try {
55 const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
56 if (exists === true) return
57
58 // Fetch url
59 const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
60
61 if (!isPlaylistObjectValid(body)) {
62 throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
63 }
64
65 if (!isArray(body.to)) {
66 throw new Error('Playlist does not have an audience.')
67 }
68
69 return createOrUpdateVideoPlaylist(body, account, body.to)
70 } catch (err) {
71 logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
72 }
73 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
74}
75
76async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
77 const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
78
79 if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
80 const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
81
82 if (actor.VideoChannel) {
83 playlistAttributes.videoChannelId = actor.VideoChannel.id
84 } else {
85 logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
86 }
87 }
88
89 const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
90
91 let accItems: string[] = []
92 await crawlCollectionPage<string>(playlistObject.id, items => {
93 accItems = accItems.concat(items)
94
95 return Promise.resolve()
96 })
97
98 const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
99
100 if (playlistObject.icon) {
101 try {
102 const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
103 await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
104 } catch (err) {
105 logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
106 }
107 } else if (refreshedPlaylist.hasThumbnail()) {
108 await refreshedPlaylist.Thumbnail.destroy()
109 refreshedPlaylist.Thumbnail = null
110 }
111
112 return resetVideoPlaylistElements(accItems, refreshedPlaylist)
113}
114
115async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
116 if (!videoPlaylist.isOutdated()) return videoPlaylist
117
118 try {
119 const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
120
121 if (playlistObject === undefined) {
122 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
123
124 await videoPlaylist.setAsRefreshed()
125 return videoPlaylist
126 }
127
128 const byAccount = videoPlaylist.OwnerAccount
129 await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
130
131 return videoPlaylist
132 } catch (err) {
133 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
134 logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
135
136 await videoPlaylist.destroy()
137 return undefined
138 }
139
140 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
141
142 await videoPlaylist.setAsRefreshed()
143 return videoPlaylist
144 }
145}
146
147// ---------------------------------------------------------------------------
148
149export {
150 createAccountPlaylists,
151 playlistObjectToDBAttributes,
152 playlistElementObjectToDBAttributes,
153 createOrUpdateVideoPlaylist,
154 refreshVideoPlaylistIfNeeded
155}
156
157// ---------------------------------------------------------------------------
158
159async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
160 const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
161
162 await Bluebird.map(elementUrls, async elementUrl => {
163 try {
164 const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
165
166 if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
167
168 if (checkUrlsSameHost(body.id, elementUrl) !== true) {
169 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
170 }
171
172 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
173
174 elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
175 } catch (err) {
176 logger.warn('Cannot add playlist element %s.', elementUrl, { err })
177 }
178 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
179
180 await sequelizeTypescript.transaction(async t => {
181 await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
182
183 for (const element of elementsToCreate) {
184 await VideoPlaylistElementModel.create(element, { transaction: t })
185 }
186 })
187
188 logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
189
190 return undefined
191}
192
193async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
194 logger.info('Fetching remote playlist %s.', playlistUrl)
195
196 const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
197
198 if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
199 logger.debug('Remote video playlist JSON is not valid.', { body })
200 return { statusCode, playlistObject: undefined }
201 }
202
203 return { statusCode, playlistObject: body }
204}
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts
new file mode 100644
index 000000000..ea3e61ac5
--- /dev/null
+++ b/server/lib/activitypub/playlists/create-update.ts
@@ -0,0 +1,156 @@
1import * as Bluebird from 'bluebird'
2import { getAPId } from '@server/helpers/activitypub'
3import { isArray } from '@server/helpers/custom-validators/misc'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
6import { sequelizeTypescript } from '@server/initializers/database'
7import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
8import { VideoPlaylistModel } from '@server/models/video/video-playlist'
9import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
10import { FilteredModelAttributes } from '@server/types'
11import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
12import { AttributesOnly } from '@shared/core-utils'
13import { PlaylistObject } from '@shared/models'
14import { getOrCreateAPActor } from '../actors'
15import { crawlCollectionPage } from '../crawl'
16import { getOrCreateAPVideo } from '../videos'
17import {
18 fetchRemotePlaylistElement,
19 fetchRemoteVideoPlaylist,
20 playlistElementObjectToDBAttributes,
21 playlistObjectToDBAttributes
22} from './shared'
23
24const lTags = loggerTagsFactory('ap', 'video-playlist')
25
26async function createAccountPlaylists (playlistUrls: string[]) {
27 await Bluebird.map(playlistUrls, async playlistUrl => {
28 try {
29 const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
30 if (exists === true) return
31
32 const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
33
34 if (playlistObject === undefined) {
35 throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
36 }
37
38 return createOrUpdateVideoPlaylist(playlistObject)
39 } catch (err) {
40 logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
41 }
42 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
43}
44
45async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
46 const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
47
48 await setVideoChannel(playlistObject, playlistAttributes)
49
50 const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
51
52 const playlistElementUrls = await fetchElementUrls(playlistObject)
53
54 // Refetch playlist from DB since elements fetching could be long in time
55 const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
56
57 await updatePlaylistThumbnail(playlistObject, playlist)
58
59 const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist)
60 playlist.setVideosLength(elementsLength)
61
62 return playlist
63}
64
65// ---------------------------------------------------------------------------
66
67export {
68 createAccountPlaylists,
69 createOrUpdateVideoPlaylist
70}
71
72// ---------------------------------------------------------------------------
73
74async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
75 if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
76 throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
77 }
78
79 const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all')
80
81 if (!actor.VideoChannel) {
82 logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
83 return
84 }
85
86 playlistAttributes.videoChannelId = actor.VideoChannel.id
87 playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
88}
89
90async function fetchElementUrls (playlistObject: PlaylistObject) {
91 let accItems: string[] = []
92 await crawlCollectionPage<string>(playlistObject.id, items => {
93 accItems = accItems.concat(items)
94
95 return Promise.resolve()
96 })
97
98 return accItems
99}
100
101async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
102 if (playlistObject.icon) {
103 let thumbnailModel: MThumbnail
104
105 try {
106 thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
107 await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
108 } catch (err) {
109 logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
110
111 if (thumbnailModel) await thumbnailModel.removeThumbnail()
112 }
113
114 return
115 }
116
117 // Playlist does not have an icon, destroy existing one
118 if (playlist.hasThumbnail()) {
119 await playlist.Thumbnail.destroy()
120 playlist.Thumbnail = null
121 }
122}
123
124async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
125 const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
126
127 await sequelizeTypescript.transaction(async t => {
128 await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
129
130 for (const element of elementsToCreate) {
131 await VideoPlaylistElementModel.create(element, { transaction: t })
132 }
133 })
134
135 logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
136
137 return elementsToCreate.length
138}
139
140async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
141 const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
142
143 await Bluebird.map(elementUrls, async elementUrl => {
144 try {
145 const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
146
147 const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
148
149 elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
150 } catch (err) {
151 logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
152 }
153 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
154
155 return elementsToCreate
156}
diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts
new file mode 100644
index 000000000..2c19c503a
--- /dev/null
+++ b/server/lib/activitypub/playlists/get.ts
@@ -0,0 +1,35 @@
1import { getAPId } from '@server/helpers/activitypub'
2import { VideoPlaylistModel } from '@server/models/video/video-playlist'
3import { MVideoPlaylistFullSummary } from '@server/types/models'
4import { APObject } from '@shared/models'
5import { createOrUpdateVideoPlaylist } from './create-update'
6import { scheduleRefreshIfNeeded } from './refresh'
7import { fetchRemoteVideoPlaylist } from './shared'
8
9async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObject): Promise<MVideoPlaylistFullSummary> {
10 const playlistUrl = getAPId(playlistObjectArg)
11
12 const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl)
13
14 if (playlistFromDatabase) {
15 scheduleRefreshIfNeeded(playlistFromDatabase)
16
17 return playlistFromDatabase
18 }
19
20 const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
21 if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl)
22
23 // playlistUrl is just an alias/rediraction, so process object id instead
24 if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject)
25
26 const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject)
27
28 return playlistCreated
29}
30
31// ---------------------------------------------------------------------------
32
33export {
34 getOrCreateAPVideoPlaylist
35}
diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts
new file mode 100644
index 000000000..e2470a674
--- /dev/null
+++ b/server/lib/activitypub/playlists/index.ts
@@ -0,0 +1,3 @@
1export * from './get'
2export * from './create-update'
3export * from './refresh'
diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts
new file mode 100644
index 000000000..ef3cb3fe4
--- /dev/null
+++ b/server/lib/activitypub/playlists/refresh.ts
@@ -0,0 +1,53 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { PeerTubeRequestError } from '@server/helpers/requests'
3import { JobQueue } from '@server/lib/job-queue'
4import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models'
5import { HttpStatusCode } from '@shared/core-utils'
6import { createOrUpdateVideoPlaylist } from './create-update'
7import { fetchRemoteVideoPlaylist } from './shared'
8
9function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) {
10 if (!playlist.isOutdated()) return
11
12 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } })
13}
14
15async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
16 if (!videoPlaylist.isOutdated()) return videoPlaylist
17
18 const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
19
20 logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags())
21
22 try {
23 const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
24
25 if (playlistObject === undefined) {
26 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags())
27
28 await videoPlaylist.setAsRefreshed()
29 return videoPlaylist
30 }
31
32 await createOrUpdateVideoPlaylist(playlistObject)
33
34 return videoPlaylist
35 } catch (err) {
36 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
37 logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags())
38
39 await videoPlaylist.destroy()
40 return undefined
41 }
42
43 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() })
44
45 await videoPlaylist.setAsRefreshed()
46 return videoPlaylist
47 }
48}
49
50export {
51 scheduleRefreshIfNeeded,
52 refreshVideoPlaylistIfNeeded
53}
diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts
new file mode 100644
index 000000000..a217f2291
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/index.ts
@@ -0,0 +1,2 @@
1export * from './object-to-model-attributes'
2export * from './url-to-object'
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..70fd335bc
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
@@ -0,0 +1,40 @@
1import { ACTIVITY_PUB } from '@server/initializers/constants'
2import { VideoPlaylistModel } from '@server/models/video/video-playlist'
3import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
4import { MVideoId, MVideoPlaylistId } from '@server/types/models'
5import { AttributesOnly } from '@shared/core-utils'
6import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
7
8function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) {
9 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
10 ? VideoPlaylistPrivacy.PUBLIC
11 : VideoPlaylistPrivacy.UNLISTED
12
13 return {
14 name: playlistObject.name,
15 description: playlistObject.content,
16 privacy,
17 url: playlistObject.id,
18 uuid: playlistObject.uuid,
19 ownerAccountId: null,
20 videoChannelId: null,
21 createdAt: new Date(playlistObject.published),
22 updatedAt: new Date(playlistObject.updated)
23 } as AttributesOnly<VideoPlaylistModel>
24}
25
26function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
27 return {
28 position: elementObject.position,
29 url: elementObject.id,
30 startTimestamp: elementObject.startTimestamp || null,
31 stopTimestamp: elementObject.stopTimestamp || null,
32 videoPlaylistId: videoPlaylist.id,
33 videoId: video.id
34 } as AttributesOnly<VideoPlaylistElementModel>
35}
36
37export {
38 playlistObjectToDBAttributes,
39 playlistElementObjectToDBAttributes
40}
diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts
new file mode 100644
index 000000000..ec8c01255
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/url-to-object.ts
@@ -0,0 +1,47 @@
1import { isArray } from 'lodash'
2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { doJSONRequest } from '@server/helpers/requests'
6import { PlaylistElementObject, PlaylistObject } from '@shared/models'
7
8async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
9 const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl)
10
11 logger.info('Fetching remote playlist %s.', playlistUrl, lTags())
12
13 const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
14
15 if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
16 logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() })
17 return { statusCode, playlistObject: undefined }
18 }
19
20 if (!isArray(body.to)) {
21 logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() })
22 return { statusCode, playlistObject: undefined }
23 }
24
25 return { statusCode, playlistObject: body }
26}
27
28async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> {
29 const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl)
30
31 logger.debug('Fetching remote playlist element %s.', elementUrl, lTags())
32
33 const { body, statusCode } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
34
35 if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`)
36
37 if (checkUrlsSameHost(body.id, elementUrl) !== true) {
38 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
39 }
40
41 return { statusCode, elementObject: body }
42}
43
44export {
45 fetchRemoteVideoPlaylist,
46 fetchRemotePlaylistElement
47}
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts
index 1799829f8..077b01eda 100644
--- a/server/lib/activitypub/process/process-accept.ts
+++ b/server/lib/activitypub/process/process-accept.ts
@@ -1,8 +1,8 @@
1import { ActivityAccept } from '../../../../shared/models/activitypub' 1import { ActivityAccept } from '../../../../shared/models/activitypub'
2import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 2import { ActorFollowModel } from '../../../models/actor/actor-follow'
3import { addFetchOutboxJob } from '../actor'
4import { APProcessorOptions } from '../../../types/activitypub-processor.model' 3import { APProcessorOptions } from '../../../types/activitypub-processor.model'
5import { MActorDefault, MActorSignature } from '../../../types/models' 4import { MActorDefault, MActorSignature } from '../../../types/models'
5import { addFetchOutboxJob } from '../outbox'
6 6
7async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) { 7async function processAcceptActivity (options: APProcessorOptions<ActivityAccept>) {
8 const { byActor: targetActor, inboxActor } = options 8 const { byActor: targetActor, inboxActor } = options
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 63082466e..ec23c705e 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -3,7 +3,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers/database' 3import { sequelizeTypescript } from '../../../initializers/database'
4import { VideoShareModel } from '../../../models/video/video-share' 4import { VideoShareModel } from '../../../models/video/video-share'
5import { forwardVideoRelatedActivity } from '../send/utils' 5import { forwardVideoRelatedActivity } from '../send/utils'
6import { getOrCreateVideoAndAccountAndChannel } from '../videos' 6import { getOrCreateAPVideo } from '../videos'
7import { Notifier } from '../../notifier' 7import { Notifier } from '../../notifier'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9import { APProcessorOptions } from '../../../types/activitypub-processor.model' 9import { APProcessorOptions } from '../../../types/activitypub-processor.model'
@@ -32,7 +32,7 @@ async function processVideoShare (actorAnnouncer: MActorSignature, activity: Act
32 let videoCreated: boolean 32 let videoCreated: boolean
33 33
34 try { 34 try {
35 const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) 35 const result = await getOrCreateAPVideo({ videoObject: objectUri })
36 video = result.video 36 video = result.video
37 videoCreated = result.created 37 videoCreated = result.created
38 } catch (err) { 38 } catch (err) {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 9cded4dec..70e048d6e 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -1,3 +1,4 @@
1import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
1import { isRedundancyAccepted } from '@server/lib/redundancy' 2import { isRedundancyAccepted } from '@server/lib/redundancy'
2import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared' 3import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared'
3import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' 4import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
@@ -9,11 +10,10 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
9import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' 10import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models'
10import { Notifier } from '../../notifier' 11import { Notifier } from '../../notifier'
11import { createOrUpdateCacheFile } from '../cache-file' 12import { createOrUpdateCacheFile } from '../cache-file'
12import { createOrUpdateVideoPlaylist } from '../playlist' 13import { createOrUpdateVideoPlaylist } from '../playlists'
13import { forwardVideoRelatedActivity } from '../send/utils' 14import { forwardVideoRelatedActivity } from '../send/utils'
14import { resolveThread } from '../video-comments' 15import { resolveThread } from '../video-comments'
15import { getOrCreateVideoAndAccountAndChannel } from '../videos' 16import { getOrCreateAPVideo } from '../videos'
16import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
17 17
18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { 18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
19 const { activity, byActor } = options 19 const { activity, byActor } = options
@@ -55,7 +55,7 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) {
55 const videoToCreateData = activity.object as VideoObject 55 const videoToCreateData = activity.object as VideoObject
56 56
57 const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } 57 const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false }
58 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData, syncParam }) 58 const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam })
59 59
60 if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) 60 if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video)
61 61
@@ -67,7 +67,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor
67 67
68 const cacheFile = activity.object as CacheFileObject 68 const cacheFile = activity.object as CacheFileObject
69 69
70 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 70 const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object })
71 71
72 await sequelizeTypescript.transaction(async t => { 72 await sequelizeTypescript.transaction(async t => {
73 return createOrUpdateCacheFile(cacheFile, video, byActor, t) 73 return createOrUpdateCacheFile(cacheFile, video, byActor, t)
@@ -128,5 +128,5 @@ async function processCreatePlaylist (activity: ActivityCreate, byActor: MActorS
128 128
129 if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) 129 if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
130 130
131 await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) 131 await createOrUpdateVideoPlaylist(playlistObject, activity.to)
132} 132}
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts
index 88a968318..1d2279df5 100644
--- a/server/lib/activitypub/process/process-delete.ts
+++ b/server/lib/activitypub/process/process-delete.ts
@@ -2,7 +2,7 @@ import { ActivityDelete } from '../../../../shared/models/activitypub'
2import { retryTransactionWrapper } from '../../../helpers/database-utils' 2import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers/database' 4import { sequelizeTypescript } from '../../../initializers/database'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/actor/actor'
6import { VideoModel } from '../../../models/video/video' 6import { VideoModel } from '../../../models/video/video'
7import { VideoCommentModel } from '../../../models/video/video-comment' 7import { VideoCommentModel } from '../../../models/video/video-comment'
8import { VideoPlaylistModel } from '../../../models/video/video-playlist' 8import { VideoPlaylistModel } from '../../../models/video/video-playlist'
@@ -16,7 +16,6 @@ import {
16 MChannelActor, 16 MChannelActor,
17 MCommentOwnerVideo 17 MCommentOwnerVideo
18} from '../../../types/models' 18} from '../../../types/models'
19import { markCommentAsDeleted } from '../../video-comment'
20import { forwardVideoRelatedActivity } from '../send/utils' 19import { forwardVideoRelatedActivity } from '../send/utils'
21 20
22async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) { 21async function processDeleteActivity (options: APProcessorOptions<ActivityDelete>) {
@@ -130,7 +129,7 @@ async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) {
130 129
131function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { 130function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) {
132 // Already deleted 131 // Already deleted
133 if (videoComment.isDeleted()) return 132 if (videoComment.isDeleted()) return Promise.resolve()
134 133
135 logger.debug('Removing remote video comment "%s".', videoComment.url) 134 logger.debug('Removing remote video comment "%s".', videoComment.url)
136 135
@@ -139,11 +138,9 @@ function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCom
139 throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) 138 throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`)
140 } 139 }
141 140
142 await sequelizeTypescript.transaction(async t => { 141 videoComment.markAsDeleted()
143 markCommentAsDeleted(videoComment)
144 142
145 await videoComment.save() 143 await videoComment.save({ transaction: t })
146 })
147 144
148 if (videoComment.Video.isOwned()) { 145 if (videoComment.Video.isOwned()) {
149 // Don't resend the activity to the sender 146 // Don't resend the activity to the sender
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts
index 089c7b881..ecc57cd10 100644
--- a/server/lib/activitypub/process/process-dislike.ts
+++ b/server/lib/activitypub/process/process-dislike.ts
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
6import { APProcessorOptions } from '../../../types/activitypub-processor.model' 6import { APProcessorOptions } from '../../../types/activitypub-processor.model'
7import { MActorSignature } from '../../../types/models' 7import { MActorSignature } from '../../../types/models'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 9import { getOrCreateAPVideo } from '../videos'
10 10
11async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { 11async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) {
12 const { activity, byActor } = options 12 const { activity, byActor } = options
@@ -30,7 +30,7 @@ async function processDislike (activity: ActivityCreate | ActivityDislike, byAct
30 30
31 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 31 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
32 32
33 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) 33 const { video } = await getOrCreateAPVideo({ videoObject: dislikeObject })
34 34
35 return sequelizeTypescript.transaction(async t => { 35 return sequelizeTypescript.transaction(async t => {
36 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) 36 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t)
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index 38d684512..f85238f8e 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -1,17 +1,17 @@
1import { getServerActor } from '@server/models/application/application'
1import { ActivityFollow } from '../../../../shared/models/activitypub' 2import { ActivityFollow } from '../../../../shared/models/activitypub'
3import { getAPId } from '../../../helpers/activitypub'
2import { retryTransactionWrapper } from '../../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers/database'
5import { ActorModel } from '../../../models/activitypub/actor'
6import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { sendAccept, sendReject } from '../send'
8import { Notifier } from '../../notifier'
9import { getAPId } from '../../../helpers/activitypub'
10import { CONFIG } from '../../../initializers/config' 6import { CONFIG } from '../../../initializers/config'
7import { sequelizeTypescript } from '../../../initializers/database'
8import { ActorModel } from '../../../models/actor/actor'
9import { ActorFollowModel } from '../../../models/actor/actor-follow'
11import { APProcessorOptions } from '../../../types/activitypub-processor.model' 10import { APProcessorOptions } from '../../../types/activitypub-processor.model'
12import { MActorFollowActors, MActorSignature } from '../../../types/models' 11import { MActorFollowActors, MActorSignature } from '../../../types/models'
12import { Notifier } from '../../notifier'
13import { autoFollowBackIfNeeded } from '../follow' 13import { autoFollowBackIfNeeded } from '../follow'
14import { getServerActor } from '@server/models/application/application' 14import { sendAccept, sendReject } from '../send'
15 15
16async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) { 16async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) {
17 const { activity, byActor } = options 17 const { activity, byActor } = options
@@ -43,7 +43,7 @@ async function processFollow (byActor: MActorSignature, activityId: string, targ
43 if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { 43 if (isFollowingInstance && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) {
44 logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) 44 logger.info('Rejecting %s because instance followers are disabled.', targetActor.url)
45 45
46 await sendReject(activityId, byActor, targetActor) 46 sendReject(activityId, byActor, targetActor)
47 47
48 return { actorFollow: undefined as MActorFollowActors } 48 return { actorFollow: undefined as MActorFollowActors }
49 } 49 }
@@ -84,8 +84,9 @@ async function processFollow (byActor: MActorSignature, activityId: string, targ
84 84
85 // Target sends to actor he accepted the follow request 85 // Target sends to actor he accepted the follow request
86 if (actorFollow.state === 'accepted') { 86 if (actorFollow.state === 'accepted') {
87 await sendAccept(actorFollow) 87 sendAccept(actorFollow)
88 await autoFollowBackIfNeeded(actorFollow) 88
89 await autoFollowBackIfNeeded(actorFollow, t)
89 } 90 }
90 91
91 return { actorFollow, created, isFollowingInstance, targetActor } 92 return { actorFollow, created, isFollowingInstance, targetActor }
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 8688b3b47..cd4e86cbb 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
6import { APProcessorOptions } from '../../../types/activitypub-processor.model' 6import { APProcessorOptions } from '../../../types/activitypub-processor.model'
7import { MActorSignature } from '../../../types/models' 7import { MActorSignature } from '../../../types/models'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 9import { getOrCreateAPVideo } from '../videos'
10 10
11async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { 11async function processLikeActivity (options: APProcessorOptions<ActivityLike>) {
12 const { activity, byActor } = options 12 const { activity, byActor } = options
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: MActorSignature, activity: ActivityLik
27 const byAccount = byActor.Account 27 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 29
30 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) 30 const { video } = await getOrCreateAPVideo({ videoObject: videoUrl })
31 31
32 return sequelizeTypescript.transaction(async t => { 32 return sequelizeTypescript.transaction(async t => {
33 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) 33 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t)
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts
index 03b669fd9..7f7ab305f 100644
--- a/server/lib/activitypub/process/process-reject.ts
+++ b/server/lib/activitypub/process/process-reject.ts
@@ -1,6 +1,6 @@
1import { ActivityReject } from '../../../../shared/models/activitypub/activity' 1import { ActivityReject } from '../../../../shared/models/activitypub/activity'
2import { sequelizeTypescript } from '../../../initializers/database' 2import { sequelizeTypescript } from '../../../initializers/database'
3import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../../models/actor/actor-follow'
4import { APProcessorOptions } from '../../../types/activitypub-processor.model' 4import { APProcessorOptions } from '../../../types/activitypub-processor.model'
5import { MActor } from '../../../types/models' 5import { MActor } from '../../../types/models'
6 6
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index e520c2f0d..d4b2a795f 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -4,14 +4,14 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { sequelizeTypescript } from '../../../initializers/database' 5import { sequelizeTypescript } from '../../../initializers/database'
6import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 6import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/actor/actor'
8import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 8import { ActorFollowModel } from '../../../models/actor/actor-follow'
9import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 9import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
10import { VideoShareModel } from '../../../models/video/video-share' 10import { VideoShareModel } from '../../../models/video/video-share'
11import { APProcessorOptions } from '../../../types/activitypub-processor.model' 11import { APProcessorOptions } from '../../../types/activitypub-processor.model'
12import { MActorSignature } from '../../../types/models' 12import { MActorSignature } from '../../../types/models'
13import { forwardVideoRelatedActivity } from '../send/utils' 13import { forwardVideoRelatedActivity } from '../send/utils'
14import { getOrCreateVideoAndAccountAndChannel } from '../videos' 14import { getOrCreateAPVideo } from '../videos'
15 15
16async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { 16async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) {
17 const { activity, byActor } = options 17 const { activity, byActor } = options
@@ -55,7 +55,7 @@ export {
55async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { 55async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) {
56 const likeActivity = activity.object as ActivityLike 56 const likeActivity = activity.object as ActivityLike
57 57
58 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) 58 const { video } = await getOrCreateAPVideo({ videoObject: likeActivity.object })
59 59
60 return sequelizeTypescript.transaction(async t => { 60 return sequelizeTypescript.transaction(async t => {
61 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 61 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
@@ -80,7 +80,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU
80 ? activity.object 80 ? activity.object
81 : activity.object.object as DislikeObject 81 : activity.object.object as DislikeObject
82 82
83 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) 83 const { video } = await getOrCreateAPVideo({ videoObject: dislike.object })
84 84
85 return sequelizeTypescript.transaction(async t => { 85 return sequelizeTypescript.transaction(async t => {
86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
@@ -103,10 +103,10 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU
103async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { 103async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) {
104 const cacheFileObject = activity.object.object as CacheFileObject 104 const cacheFileObject = activity.object.object as CacheFileObject
105 105
106 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) 106 const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object })
107 107
108 return sequelizeTypescript.transaction(async t => { 108 return sequelizeTypescript.transaction(async t => {
109 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 109 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
110 if (!cacheFile) { 110 if (!cacheFile) {
111 logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) 111 logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id)
112 return 112 return
@@ -114,7 +114,7 @@ async function processUndoCacheFile (byActor: MActorSignature, activity: Activit
114 114
115 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') 115 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
116 116
117 await cacheFile.destroy() 117 await cacheFile.destroy({ transaction: t })
118 118
119 if (video.isOwned()) { 119 if (video.isOwned()) {
120 // Don't resend the activity to the sender 120 // Don't resend the activity to the sender
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 6df9b93b2..f40008a6b 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -1,23 +1,20 @@
1import { isRedundancyAccepted } from '@server/lib/redundancy'
1import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' 2import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub'
2import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' 3import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
3import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' 4import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
5import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
6import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
7import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
5import { sequelizeTypescript } from '../../../initializers/database' 9import { sequelizeTypescript } from '../../../initializers/database'
6import { AccountModel } from '../../../models/account/account' 10import { ActorModel } from '../../../models/actor/actor'
7import { ActorModel } from '../../../models/activitypub/actor' 11import { APProcessorOptions } from '../../../types/activitypub-processor.model'
8import { VideoChannelModel } from '../../../models/video/video-channel' 12import { MActorFull, MActorSignature } from '../../../types/models'
9import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor' 13import { APActorUpdater } from '../actors/updater'
10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { createOrUpdateCacheFile } from '../cache-file' 14import { createOrUpdateCacheFile } from '../cache-file'
15import { createOrUpdateVideoPlaylist } from '../playlists'
14import { forwardVideoRelatedActivity } from '../send/utils' 16import { forwardVideoRelatedActivity } from '../send/utils'
15import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' 17import { APVideoUpdater, getOrCreateAPVideo } from '../videos'
16import { createOrUpdateVideoPlaylist } from '../playlist'
17import { APProcessorOptions } from '../../../types/activitypub-processor.model'
18import { MActorSignature, MAccountIdActor } from '../../../types/models'
19import { isRedundancyAccepted } from '@server/lib/redundancy'
20import { ActorImageType } from '@shared/models'
21 18
22async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { 19async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
23 const { activity, byActor } = options 20 const { activity, byActor } = options
@@ -25,7 +22,7 @@ async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate
25 const objectType = activity.object.type 22 const objectType = activity.object.type
26 23
27 if (objectType === 'Video') { 24 if (objectType === 'Video') {
28 return retryTransactionWrapper(processUpdateVideo, byActor, activity) 25 return retryTransactionWrapper(processUpdateVideo, activity)
29 } 26 }
30 27
31 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { 28 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
@@ -55,7 +52,7 @@ export {
55 52
56// --------------------------------------------------------------------------- 53// ---------------------------------------------------------------------------
57 54
58async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpdate) { 55async function processUpdateVideo (activity: ActivityUpdate) {
59 const videoObject = activity.object as VideoObject 56 const videoObject = activity.object as VideoObject
60 57
61 if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { 58 if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) {
@@ -63,7 +60,7 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
63 return undefined 60 return undefined
64 } 61 }
65 62
66 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ 63 const { video, created } = await getOrCreateAPVideo({
67 videoObject: videoObject.id, 64 videoObject: videoObject.id,
68 allowRefresh: false, 65 allowRefresh: false,
69 fetchType: 'all' 66 fetchType: 'all'
@@ -71,20 +68,8 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
71 // We did not have this video, it has been created so no need to update 68 // We did not have this video, it has been created so no need to update
72 if (created) return 69 if (created) return
73 70
74 // Load new channel 71 const updater = new APVideoUpdater(videoObject, video)
75 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 72 return updater.update(activity.to)
76
77 const account = actor.Account as MAccountIdActor
78 account.Actor = actor
79
80 const updateOptions = {
81 video,
82 videoObject,
83 account,
84 channel: channelActor.VideoChannel,
85 overrideTo: activity.to
86 }
87 return updateVideoFromAP(updateOptions)
88} 73}
89 74
90async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { 75async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) {
@@ -97,7 +82,7 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ
97 return undefined 82 return undefined
98 } 83 }
99 84
100 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) 85 const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object })
101 86
102 await sequelizeTypescript.transaction(async t => { 87 await sequelizeTypescript.transaction(async t => {
103 await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) 88 await createOrUpdateCacheFile(cacheFileObject, video, byActor, t)
@@ -111,56 +96,13 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ
111 } 96 }
112} 97}
113 98
114async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { 99async function processUpdateActor (actor: MActorFull, activity: ActivityUpdate) {
115 const actorAttributesToUpdate = activity.object as ActivityPubActor 100 const actorObject = activity.object as ActivityPubActor
116 101
117 logger.debug('Updating remote account "%s".', actorAttributesToUpdate.url) 102 logger.debug('Updating remote account "%s".', actorObject.url)
118 let accountOrChannelInstance: AccountModel | VideoChannelModel
119 let actorFieldsSave: object
120 let accountOrChannelFieldsSave: object
121 103
122 // Fetch icon? 104 const updater = new APActorUpdater(actorObject, actor)
123 const avatarInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.AVATAR) 105 return updater.update()
124 const bannerInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.BANNER)
125
126 try {
127 await sequelizeTypescript.transaction(async t => {
128 actorFieldsSave = actor.toJSON()
129
130 if (actorAttributesToUpdate.type === 'Group') accountOrChannelInstance = actor.VideoChannel
131 else accountOrChannelInstance = actor.Account
132
133 accountOrChannelFieldsSave = accountOrChannelInstance.toJSON()
134
135 await updateActorInstance(actor, actorAttributesToUpdate)
136
137 await updateActorImageInstance(actor, ActorImageType.AVATAR, avatarInfo, t)
138 await updateActorImageInstance(actor, ActorImageType.BANNER, bannerInfo, t)
139
140 await actor.save({ transaction: t })
141
142 accountOrChannelInstance.name = actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername
143 accountOrChannelInstance.description = actorAttributesToUpdate.summary
144
145 if (accountOrChannelInstance instanceof VideoChannelModel) accountOrChannelInstance.support = actorAttributesToUpdate.support
146
147 await accountOrChannelInstance.save({ transaction: t })
148 })
149
150 logger.info('Remote account %s updated', actorAttributesToUpdate.url)
151 } catch (err) {
152 if (actor !== undefined && actorFieldsSave !== undefined) {
153 resetSequelizeInstance(actor, actorFieldsSave)
154 }
155
156 if (accountOrChannelInstance !== undefined && accountOrChannelFieldsSave !== undefined) {
157 resetSequelizeInstance(accountOrChannelInstance, accountOrChannelFieldsSave)
158 }
159
160 // This is just a debug because we will retry the insert
161 logger.debug('Cannot update the remote account.', { err })
162 throw err
163 }
164} 106}
165 107
166async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) { 108async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) {
@@ -169,5 +111,5 @@ async function processUpdatePlaylist (byActor: MActorSignature, activity: Activi
169 111
170 if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) 112 if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
171 113
172 await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to) 114 await createOrUpdateVideoPlaylist(playlistObject, activity.to)
173} 115}
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index 84697673b..5593ee257 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -1,10 +1,10 @@
1import { getOrCreateVideoAndAccountAndChannel } from '../videos' 1import { getOrCreateAPVideo } from '../videos'
2import { forwardVideoRelatedActivity } from '../send/utils' 2import { forwardVideoRelatedActivity } from '../send/utils'
3import { Redis } from '../../redis' 3import { Redis } from '../../redis'
4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' 4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub'
5import { APProcessorOptions } from '../../../types/activitypub-processor.model' 5import { APProcessorOptions } from '../../../types/activitypub-processor.model'
6import { MActorSignature } from '../../../types/models' 6import { MActorSignature } from '../../../types/models'
7import { LiveManager } from '@server/lib/live-manager' 7import { LiveManager } from '@server/lib/live/live-manager'
8 8
9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) { 9async function processViewActivity (options: APProcessorOptions<ActivityCreate | ActivityView>) {
10 const { activity, byActor } = options 10 const { activity, byActor } = options
@@ -24,12 +24,11 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
24 ? activity.object 24 ? activity.object
25 : (activity.object as ViewObject).object 25 : (activity.object as ViewObject).object
26 26
27 const options = { 27 const { video } = await getOrCreateAPVideo({
28 videoObject, 28 videoObject,
29 fetchType: 'only-video' as 'only-video', 29 fetchType: 'only-video',
30 allowRefresh: false as false 30 allowRefresh: false
31 } 31 })
32 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
33 32
34 if (!video.isLive) { 33 if (!video.isLive) {
35 await Redis.Instance.addVideoView(video.id) 34 await Redis.Instance.addVideoView(video.id)
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index 5cef75665..02a23d098 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -1,22 +1,22 @@
1import { StatsManager } from '@server/lib/stat-manager'
1import { Activity, ActivityType } from '../../../../shared/models/activitypub' 2import { Activity, ActivityType } from '../../../../shared/models/activitypub'
2import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' 3import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub'
3import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { APProcessorOptions } from '../../../types/activitypub-processor.model'
6import { MActorDefault, MActorSignature } from '../../../types/models'
7import { getOrCreateAPActor } from '../actors'
4import { processAcceptActivity } from './process-accept' 8import { processAcceptActivity } from './process-accept'
5import { processAnnounceActivity } from './process-announce' 9import { processAnnounceActivity } from './process-announce'
6import { processCreateActivity } from './process-create' 10import { processCreateActivity } from './process-create'
7import { processDeleteActivity } from './process-delete' 11import { processDeleteActivity } from './process-delete'
12import { processDislikeActivity } from './process-dislike'
13import { processFlagActivity } from './process-flag'
8import { processFollowActivity } from './process-follow' 14import { processFollowActivity } from './process-follow'
9import { processLikeActivity } from './process-like' 15import { processLikeActivity } from './process-like'
10import { processRejectActivity } from './process-reject' 16import { processRejectActivity } from './process-reject'
11import { processUndoActivity } from './process-undo' 17import { processUndoActivity } from './process-undo'
12import { processUpdateActivity } from './process-update' 18import { processUpdateActivity } from './process-update'
13import { getOrCreateActorAndServerAndModel } from '../actor'
14import { processDislikeActivity } from './process-dislike'
15import { processFlagActivity } from './process-flag'
16import { processViewActivity } from './process-view' 19import { processViewActivity } from './process-view'
17import { APProcessorOptions } from '../../../types/activitypub-processor.model'
18import { MActorDefault, MActorSignature } from '../../../types/models'
19import { StatsManager } from '@server/lib/stat-manager'
20 20
21const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = { 21const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions<Activity>) => Promise<any> } = {
22 Create: processCreateActivity, 22 Create: processCreateActivity,
@@ -65,7 +65,7 @@ async function processActivities (
65 continue 65 continue
66 } 66 }
67 67
68 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) 68 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl)
69 actorsCache[actorUrl] = byActor 69 actorsCache[actorUrl] = byActor
70 70
71 const activityProcessor = processActivity[activity.type] 71 const activityProcessor = processActivity[activity.type]
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index e0acced18..d31f8c10b 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize'
2import { getServerActor } from '@server/models/application/application' 2import { getServerActor } from '@server/models/application/application'
3import { ActivityAudience, ActivityDelete } from '../../../../shared/models/activitypub' 3import { ActivityAudience, ActivityDelete } from '../../../../shared/models/activitypub'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/actor/actor'
6import { VideoCommentModel } from '../../../models/video/video-comment' 6import { VideoCommentModel } from '../../../models/video/video-comment'
7import { VideoShareModel } from '../../../models/video/video-share' 7import { VideoShareModel } from '../../../models/video/video-share'
8import { MActorUrl } from '../../../types/models' 8import { MActorUrl } from '../../../types/models'
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts
index 9254dc7c5..153e94295 100644
--- a/server/lib/activitypub/send/send-view.ts
+++ b/server/lib/activitypub/send/send-view.ts
@@ -2,7 +2,7 @@ import { Transaction } from 'sequelize'
2import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models' 2import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models'
3import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub' 3import { ActivityAudience, ActivityView } from '../../../../shared/models/activitypub'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/actor/actor'
6import { audiencify, getAudience } from '../audience' 6import { audiencify, getAudience } from '../audience'
7import { getLocalVideoViewActivityPubUrl } from '../url' 7import { getLocalVideoViewActivityPubUrl } from '../url'
8import { sendVideoRelatedActivity } from './utils' 8import { sendVideoRelatedActivity } from './utils'
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index 85a9f009d..7cd8030e1 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -1,14 +1,14 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { getServerActor } from '@server/models/application/application'
3import { ContextType } from '@shared/models/activitypub/context'
2import { Activity, ActivityAudience } from '../../../../shared/models/activitypub' 4import { Activity, ActivityAudience } from '../../../../shared/models/activitypub'
5import { afterCommitIfTransaction } from '../../../helpers/database-utils'
3import { logger } from '../../../helpers/logger' 6import { logger } from '../../../helpers/logger'
4import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/actor/actor'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 8import { ActorFollowModel } from '../../../models/actor/actor-follow'
9import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../types/models'
6import { JobQueue } from '../../job-queue' 10import { JobQueue } from '../../job-queue'
7import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' 11import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
8import { afterCommitIfTransaction } from '../../../helpers/database-utils'
9import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../types/models'
10import { getServerActor } from '@server/models/application/application'
11import { ContextType } from '@shared/models/activitypub/context'
12 12
13async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { 13async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
14 byActor: MActorLight 14 byActor: MActorLight
@@ -22,7 +22,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
22 22
23 // Send to origin 23 // Send to origin
24 if (video.isOwned() === false) { 24 if (video.isOwned() === false) {
25 const accountActor = (video as MVideoAccountLight).VideoChannel?.Account?.Actor || await ActorModel.loadAccountActorByVideoId(video.id) 25 let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor
26
27 if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction)
26 28
27 const audience = getRemoteVideoAudience(accountActor, actorsInvolvedInVideo) 29 const audience = getRemoteVideoAudience(accountActor, actorsInvolvedInVideo)
28 const activity = activityBuilder(audience) 30 const activity = activityBuilder(audience)
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index c22fa0893..1ff01a175 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -7,7 +7,7 @@ import { doJSONRequest } from '../../helpers/requests'
7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
8import { VideoShareModel } from '../../models/video/video-share' 8import { VideoShareModel } from '../../models/video/video-share'
9import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' 9import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video'
10import { getOrCreateActorAndServerAndModel } from './actor' 10import { getOrCreateAPActor } from './actors'
11import { sendUndoAnnounce, sendVideoAnnounce } from './send' 11import { sendUndoAnnounce, sendVideoAnnounce } from './send'
12import { getLocalVideoAnnounceActivityPubUrl } from './url' 12import { getLocalVideoAnnounceActivityPubUrl } from './url'
13 13
@@ -40,23 +40,7 @@ async function changeVideoChannelShare (
40async function addVideoShares (shareUrls: string[], video: MVideoId) { 40async function addVideoShares (shareUrls: string[], video: MVideoId) {
41 await Bluebird.map(shareUrls, async shareUrl => { 41 await Bluebird.map(shareUrls, async shareUrl => {
42 try { 42 try {
43 const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true }) 43 await addVideoShare(shareUrl, video)
44 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
45
46 const actorUrl = getAPId(body.actor)
47 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
48 throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
49 }
50
51 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
52
53 const entry = {
54 actorId: actor.id,
55 videoId: video.id,
56 url: shareUrl
57 }
58
59 await VideoShareModel.upsert(entry)
60 } catch (err) { 44 } catch (err) {
61 logger.warn('Cannot add share %s.', shareUrl, { err }) 45 logger.warn('Cannot add share %s.', shareUrl, { err })
62 } 46 }
@@ -71,6 +55,26 @@ export {
71 55
72// --------------------------------------------------------------------------- 56// ---------------------------------------------------------------------------
73 57
58async function addVideoShare (shareUrl: string, video: MVideoId) {
59 const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
60 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
61
62 const actorUrl = getAPId(body.actor)
63 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
64 throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
65 }
66
67 const actor = await getOrCreateAPActor(actorUrl)
68
69 const entry = {
70 actorId: actor.id,
71 videoId: video.id,
72 url: shareUrl
73 }
74
75 await VideoShareModel.upsert(entry)
76}
77
74async function shareByServer (video: MVideo, t: Transaction) { 78async function shareByServer (video: MVideo, t: Transaction) {
75 const serverActor = await getServerActor() 79 const serverActor = await getServerActor()
76 80
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index e23e0c0e7..6b7f9504f 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -6,8 +6,8 @@ import { doJSONRequest } from '../../helpers/requests'
6import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 6import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
7import { VideoCommentModel } from '../../models/video/video-comment' 7import { VideoCommentModel } from '../../models/video/video-comment'
8import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 8import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateAPActor } from './actors'
10import { getOrCreateVideoAndAccountAndChannel } from './videos' 10import { getOrCreateAPVideo } from './videos'
11 11
12type ResolveThreadParams = { 12type ResolveThreadParams = {
13 url: string 13 url: string
@@ -29,10 +29,11 @@ async function addVideoComments (commentUrls: string[]) {
29 29
30async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { 30async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
31 const { url, isVideo } = params 31 const { url, isVideo } = params
32
32 if (params.commentCreated === undefined) params.commentCreated = false 33 if (params.commentCreated === undefined) params.commentCreated = false
33 if (params.comments === undefined) params.comments = [] 34 if (params.comments === undefined) params.comments = []
34 35
35 // If it is not a video, or if we don't know if it's a video 36 // If it is not a video, or if we don't know if it's a video, try to get the thread from DB
36 if (isVideo === false || isVideo === undefined) { 37 if (isVideo === false || isVideo === undefined) {
37 const result = await resolveCommentFromDB(params) 38 const result = await resolveCommentFromDB(params)
38 if (result) return result 39 if (result) return result
@@ -42,7 +43,7 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult
42 // If it is a video, or if we don't know if it's a video 43 // If it is a video, or if we don't know if it's a video
43 if (isVideo === true || isVideo === undefined) { 44 if (isVideo === true || isVideo === undefined) {
44 // Keep await so we catch the exception 45 // Keep await so we catch the exception
45 return await tryResolveThreadFromVideo(params) 46 return await tryToResolveThreadFromVideo(params)
46 } 47 }
47 } catch (err) { 48 } catch (err) {
48 logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) 49 logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err })
@@ -62,34 +63,32 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
62 const { url, comments, commentCreated } = params 63 const { url, comments, commentCreated } = params
63 64
64 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) 65 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
65 if (commentFromDatabase) { 66 if (!commentFromDatabase) return undefined
66 let parentComments = comments.concat([ commentFromDatabase ])
67 67
68 // Speed up things and resolve directly the thread 68 let parentComments = comments.concat([ commentFromDatabase ])
69 if (commentFromDatabase.InReplyToVideoComment) {
70 const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
71 69
72 parentComments = parentComments.concat(data) 70 // Speed up things and resolve directly the thread
73 } 71 if (commentFromDatabase.InReplyToVideoComment) {
72 const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
74 73
75 return resolveThread({ 74 parentComments = parentComments.concat(data)
76 url: commentFromDatabase.Video.url,
77 comments: parentComments,
78 isVideo: true,
79 commentCreated
80 })
81 } 75 }
82 76
83 return undefined 77 return resolveThread({
78 url: commentFromDatabase.Video.url,
79 comments: parentComments,
80 isVideo: true,
81 commentCreated
82 })
84} 83}
85 84
86async function tryResolveThreadFromVideo (params: ResolveThreadParams) { 85async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
87 const { url, comments, commentCreated } = params 86 const { url, comments, commentCreated } = params
88 87
89 // Maybe it's a reply to a video? 88 // Maybe it's a reply to a video?
90 // If yes, it's done: we resolved all the thread 89 // If yes, it's done: we resolved all the thread
91 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } 90 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false }
92 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) 91 const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam })
93 92
94 if (video.isOwned() && !video.hasPrivacyForFederation()) { 93 if (video.isOwned() && !video.hasPrivacyForFederation()) {
95 throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') 94 throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation')
@@ -148,7 +147,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
148 } 147 }
149 148
150 const actor = actorUrl 149 const actor = actorUrl
151 ? await getOrCreateActorAndServerAndModel(actorUrl, 'all') 150 ? await getOrCreateAPActor(actorUrl, 'all')
152 : null 151 : null
153 152
154 const comment = new VideoCommentModel({ 153 const comment = new VideoCommentModel({
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index f40c07fea..9fb97ef84 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -3,44 +3,23 @@ import { Transaction } from 'sequelize'
3import { doJSONRequest } from '@server/helpers/requests' 3import { doJSONRequest } from '@server/helpers/requests'
4import { VideoRateType } from '../../../shared/models/videos' 4import { VideoRateType } from '../../../shared/models/videos'
5import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 5import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
6import { logger } from '../../helpers/logger' 6import { logger, loggerTagsFactory } from '../../helpers/logger'
7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 7import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
8import { AccountVideoRateModel } from '../../models/account/account-video-rate' 8import { AccountVideoRateModel } from '../../models/account/account-video-rate'
9import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' 9import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models'
10import { getOrCreateActorAndServerAndModel } from './actor' 10import { getOrCreateAPActor } from './actors'
11import { sendLike, sendUndoDislike, sendUndoLike } from './send' 11import { sendLike, sendUndoDislike, sendUndoLike } from './send'
12import { sendDislike } from './send/send-dislike' 12import { sendDislike } from './send/send-dislike'
13import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' 13import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url'
14 14
15const lTags = loggerTagsFactory('ap', 'video-rate', 'create')
16
15async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { 17async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
16 await Bluebird.map(ratesUrl, async rateUrl => { 18 await Bluebird.map(ratesUrl, async rateUrl => {
17 try { 19 try {
18 // Fetch url 20 await createRate(rateUrl, video, rate)
19 const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
20 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
21
22 const actorUrl = getAPId(body.actor)
23 if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
24 throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
25 }
26
27 if (checkUrlsSameHost(body.id, rateUrl) !== true) {
28 throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
29 }
30
31 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
32
33 const entry = {
34 videoId: video.id,
35 accountId: actor.Account.id,
36 type: rate,
37 url: body.id
38 }
39
40 // Video "likes"/"dislikes" will be updated by the caller
41 await AccountVideoRateModel.upsert(entry)
42 } catch (err) { 21 } catch (err) {
43 logger.warn('Cannot add rate %s.', rateUrl, { err }) 22 logger.info('Cannot add rate %s.', rateUrl, { err, ...lTags(rateUrl, video.uuid, video.url) })
44 } 23 }
45 }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) 24 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
46} 25}
@@ -73,8 +52,39 @@ function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVid
73 : getVideoDislikeActivityPubUrlByLocalActor(actor, video) 52 : getVideoDislikeActivityPubUrlByLocalActor(actor, video)
74} 53}
75 54
55// ---------------------------------------------------------------------------
56
76export { 57export {
77 getLocalRateUrl, 58 getLocalRateUrl,
78 createRates, 59 createRates,
79 sendVideoRateChange 60 sendVideoRateChange
80} 61}
62
63// ---------------------------------------------------------------------------
64
65async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) {
66 // Fetch url
67 const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
68 if (!body || !body.actor) throw new Error('Body or body actor is invalid')
69
70 const actorUrl = getAPId(body.actor)
71 if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
72 throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
73 }
74
75 if (checkUrlsSameHost(body.id, rateUrl) !== true) {
76 throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
77 }
78
79 const actor = await getOrCreateAPActor(actorUrl)
80
81 const entry = {
82 videoId: video.id,
83 accountId: actor.Account.id,
84 type: rate,
85 url: body.id
86 }
87
88 // Video "likes"/"dislikes" will be updated by the caller
89 await AccountVideoRateModel.upsert(entry)
90}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
deleted file mode 100644
index 127a0dd8a..000000000
--- a/server/lib/activitypub/videos.ts
+++ /dev/null
@@ -1,931 +0,0 @@
1import * as Bluebird from 'bluebird'
2import { maxBy, minBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri'
4import { basename } from 'path'
5import { Transaction } from 'sequelize/types'
6import { TrackerModel } from '@server/models/server/tracker'
7import { VideoLiveModel } from '@server/models/video/video-live'
8import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
9import {
10 ActivityHashTagObject,
11 ActivityMagnetUrlObject,
12 ActivityPlaylistSegmentHashesObject,
13 ActivityPlaylistUrlObject,
14 ActivitypubHttpFetcherPayload,
15 ActivityTagObject,
16 ActivityUrlObject,
17 ActivityVideoUrlObject
18} from '../../../shared/index'
19import { ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects'
20import { VideoPrivacy } from '../../../shared/models/videos'
21import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
22import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
23import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
24import {
25 isAPVideoFileUrlMetadataObject,
26 isAPVideoTrackerUrlObject,
27 sanitizeAndCheckVideoTorrentObject
28} from '../../helpers/custom-validators/activitypub/videos'
29import { isArray } from '../../helpers/custom-validators/misc'
30import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
31import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
32import { logger } from '../../helpers/logger'
33import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
34import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
35import {
36 ACTIVITY_PUB,
37 MIMETYPES,
38 P2P_MEDIA_LOADER_PEER_VERSION,
39 PREVIEWS_SIZE,
40 REMOTE_SCHEME,
41 THUMBNAILS_SIZE
42} from '../../initializers/constants'
43import { sequelizeTypescript } from '../../initializers/database'
44import { AccountVideoRateModel } from '../../models/account/account-video-rate'
45import { VideoModel } from '../../models/video/video'
46import { VideoCaptionModel } from '../../models/video/video-caption'
47import { VideoCommentModel } from '../../models/video/video-comment'
48import { VideoFileModel } from '../../models/video/video-file'
49import { VideoShareModel } from '../../models/video/video-share'
50import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
51import {
52 MAccountIdActor,
53 MChannelAccountLight,
54 MChannelDefault,
55 MChannelId,
56 MStreamingPlaylist,
57 MStreamingPlaylistFilesVideo,
58 MStreamingPlaylistVideo,
59 MVideo,
60 MVideoAccountLight,
61 MVideoAccountLightBlacklistAllFiles,
62 MVideoAP,
63 MVideoAPWithoutCaption,
64 MVideoCaption,
65 MVideoFile,
66 MVideoFullLight,
67 MVideoId,
68 MVideoImmutable,
69 MVideoThumbnail,
70 MVideoWithHost
71} from '../../types/models'
72import { MThumbnail } from '../../types/models/video/thumbnail'
73import { FilteredModelAttributes } from '../../types/sequelize'
74import { ActorFollowScoreCache } from '../files-cache'
75import { JobQueue } from '../job-queue'
76import { Notifier } from '../notifier'
77import { PeerTubeSocket } from '../peertube-socket'
78import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
79import { setVideoTags } from '../video'
80import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
81import { generateTorrentFileName } from '../video-paths'
82import { getOrCreateActorAndServerAndModel } from './actor'
83import { crawlCollectionPage } from './crawl'
84import { sendCreateVideo, sendUpdateVideo } from './send'
85import { addVideoShares, shareVideoByServerAndChannel } from './share'
86import { addVideoComments } from './video-comments'
87import { createRates } from './video-rates'
88
89async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
90 const video = videoArg as MVideoAP
91
92 if (
93 // Check this is not a blacklisted video, or unfederated blacklisted video
94 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
95 // Check the video is public/unlisted and published
96 video.hasPrivacyForFederation() && video.hasStateForFederation()
97 ) {
98 // Fetch more attributes that we will need to serialize in AP object
99 if (isArray(video.VideoCaptions) === false) {
100 video.VideoCaptions = await video.$get('VideoCaptions', {
101 attributes: [ 'filename', 'language' ],
102 transaction
103 })
104 }
105
106 if (isNewVideo) {
107 // Now we'll add the video's meta data to our followers
108 await sendCreateVideo(video, transaction)
109 await shareVideoByServerAndChannel(video, transaction)
110 } else {
111 await sendUpdateVideo(video, transaction)
112 }
113 }
114}
115
116async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
117 logger.info('Fetching remote video %s.', videoUrl)
118
119 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
120
121 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
122 logger.debug('Remote video JSON is not valid.', { body })
123 return { statusCode, videoObject: undefined }
124 }
125
126 return { statusCode, videoObject: body }
127}
128
129async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
130 const host = video.VideoChannel.Account.Actor.Server.host
131 const path = video.getDescriptionAPIPath()
132 const url = REMOTE_SCHEME.HTTP + '://' + host + path
133
134 const { body } = await doJSONRequest<any>(url)
135 return body.description || ''
136}
137
138function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) {
139 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
140 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
141
142 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
143 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
144 }
145
146 return getOrCreateActorAndServerAndModel(channel.id, 'all')
147}
148
149type SyncParam = {
150 likes: boolean
151 dislikes: boolean
152 shares: boolean
153 comments: boolean
154 thumbnail: boolean
155 refreshVideo?: boolean
156}
157async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) {
158 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
159
160 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
161
162 if (syncParam.likes === true) {
163 const handler = items => createRates(items, video, 'like')
164 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
165
166 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
167 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes }))
168 } else {
169 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
170 }
171
172 if (syncParam.dislikes === true) {
173 const handler = items => createRates(items, video, 'dislike')
174 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
175
176 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
177 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes }))
178 } else {
179 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
180 }
181
182 if (syncParam.shares === true) {
183 const handler = items => addVideoShares(items, video)
184 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
185
186 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
187 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
188 } else {
189 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
190 }
191
192 if (syncParam.comments === true) {
193 const handler = items => addVideoComments(items)
194 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
195
196 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
197 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
198 } else {
199 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
200 }
201
202 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }))
203}
204
205type GetVideoResult <T> = Promise<{
206 video: T
207 created: boolean
208 autoBlacklisted?: boolean
209}>
210
211type GetVideoParamAll = {
212 videoObject: { id: string } | string
213 syncParam?: SyncParam
214 fetchType?: 'all'
215 allowRefresh?: boolean
216}
217
218type GetVideoParamImmutable = {
219 videoObject: { id: string } | string
220 syncParam?: SyncParam
221 fetchType: 'only-immutable-attributes'
222 allowRefresh: false
223}
224
225type GetVideoParamOther = {
226 videoObject: { id: string } | string
227 syncParam?: SyncParam
228 fetchType?: 'all' | 'only-video'
229 allowRefresh?: boolean
230}
231
232function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
233function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
234function getOrCreateVideoAndAccountAndChannel (
235 options: GetVideoParamOther
236): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
237async function getOrCreateVideoAndAccountAndChannel (
238 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
239): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
240 // Default params
241 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
242 const fetchType = options.fetchType || 'all'
243 const allowRefresh = options.allowRefresh !== false
244
245 // Get video url
246 const videoUrl = getAPId(options.videoObject)
247 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
248
249 if (videoFromDatabase) {
250 // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
251 if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
252 const refreshOptions = {
253 video: videoFromDatabase as MVideoThumbnail,
254 fetchedType: fetchType,
255 syncParam
256 }
257
258 if (syncParam.refreshVideo === true) {
259 videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
260 } else {
261 await JobQueue.Instance.createJobWithPromise({
262 type: 'activitypub-refresher',
263 payload: { type: 'video', url: videoFromDatabase.url }
264 })
265 }
266 }
267
268 return { video: videoFromDatabase, created: false }
269 }
270
271 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
272 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
273
274 const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
275 const videoChannel = actor.VideoChannel
276
277 try {
278 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
279
280 await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
281
282 return { video: videoCreated, created: true, autoBlacklisted }
283 } catch (err) {
284 // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
285 if (err.name === 'SequelizeUniqueConstraintError') {
286 const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
287 if (fallbackVideo) return { video: fallbackVideo, created: false }
288 }
289
290 throw err
291 }
292}
293
294async function updateVideoFromAP (options: {
295 video: MVideoAccountLightBlacklistAllFiles
296 videoObject: VideoObject
297 account: MAccountIdActor
298 channel: MChannelDefault
299 overrideTo?: string[]
300}) {
301 const { video, videoObject, account, channel, overrideTo } = options
302
303 logger.debug('Updating remote video "%s".', options.videoObject.uuid, { videoObject: options.videoObject, account, channel })
304
305 let videoFieldsSave: any
306 const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
307 const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
308
309 try {
310 let thumbnailModel: MThumbnail
311
312 try {
313 thumbnailModel = await createVideoMiniatureFromUrl({
314 downloadUrl: getThumbnailFromIcons(videoObject).url,
315 video,
316 type: ThumbnailType.MINIATURE
317 })
318 } catch (err) {
319 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
320 }
321
322 const videoUpdated = await sequelizeTypescript.transaction(async t => {
323 const sequelizeOptions = { transaction: t }
324
325 videoFieldsSave = video.toJSON()
326
327 // Check we can update the channel: we trust the remote server
328 const oldVideoChannel = video.VideoChannel
329
330 if (!oldVideoChannel.Actor.serverId || !channel.Actor.serverId) {
331 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
332 }
333
334 if (oldVideoChannel.Actor.serverId !== channel.Actor.serverId) {
335 throw new Error('New channel ' + channel.Actor.url + ' is not on the same server than new channel ' + oldVideoChannel.Actor.url)
336 }
337
338 const to = overrideTo || videoObject.to
339 const videoData = videoActivityObjectToDBAttributes(channel, videoObject, to)
340 video.name = videoData.name
341 video.uuid = videoData.uuid
342 video.url = videoData.url
343 video.category = videoData.category
344 video.licence = videoData.licence
345 video.language = videoData.language
346 video.description = videoData.description
347 video.support = videoData.support
348 video.nsfw = videoData.nsfw
349 video.commentsEnabled = videoData.commentsEnabled
350 video.downloadEnabled = videoData.downloadEnabled
351 video.waitTranscoding = videoData.waitTranscoding
352 video.state = videoData.state
353 video.duration = videoData.duration
354 video.createdAt = videoData.createdAt
355 video.publishedAt = videoData.publishedAt
356 video.originallyPublishedAt = videoData.originallyPublishedAt
357 video.privacy = videoData.privacy
358 video.channelId = videoData.channelId
359 video.views = videoData.views
360 video.isLive = videoData.isLive
361
362 // Ensures we update the updated video attribute
363 video.changed('updatedAt', true)
364
365 const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
366
367 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
368
369 const previewIcon = getPreviewFromIcons(videoObject)
370 if (videoUpdated.getPreview() && previewIcon) {
371 const previewModel = createPlaceholderThumbnail({
372 fileUrl: previewIcon.url,
373 video,
374 type: ThumbnailType.PREVIEW,
375 size: previewIcon
376 })
377 await videoUpdated.addAndSaveThumbnail(previewModel, t)
378 }
379
380 {
381 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
382 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
383
384 // Remove video files that do not exist anymore
385 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
386 await Promise.all(destroyTasks)
387
388 // Update or add other one
389 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
390 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
391 }
392
393 {
394 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
395 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
396
397 // Remove video playlists that do not exist anymore
398 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
399 await Promise.all(destroyTasks)
400
401 let oldStreamingPlaylistFiles: MVideoFile[] = []
402 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
403 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
404 }
405
406 videoUpdated.VideoStreamingPlaylists = []
407
408 for (const playlistAttributes of streamingPlaylistAttributes) {
409 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
410 .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo)
411 streamingPlaylistModel.Video = videoUpdated
412
413 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
414 .map(a => new VideoFileModel(a))
415 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
416 await Promise.all(destroyTasks)
417
418 // Update or add other one
419 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
420 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
421
422 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
423 }
424 }
425
426 {
427 // Update Tags
428 const tags = videoObject.tag
429 .filter(isAPHashTagObject)
430 .map(tag => tag.name)
431 await setVideoTags({ video: videoUpdated, tags, transaction: t })
432 }
433
434 // Update trackers
435 {
436 const trackers = getTrackerUrls(videoObject, videoUpdated)
437 await setVideoTrackers({ video: videoUpdated, trackers, transaction: t })
438 }
439
440 {
441 // Update captions
442 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
443
444 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
445 const caption = new VideoCaptionModel({
446 videoId: videoUpdated.id,
447 filename: VideoCaptionModel.generateCaptionName(c.identifier),
448 language: c.identifier,
449 fileUrl: c.url
450 }) as MVideoCaption
451
452 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
453 })
454 await Promise.all(videoCaptionsPromises)
455 }
456
457 {
458 // Create or update existing live
459 if (video.isLive) {
460 const [ videoLive ] = await VideoLiveModel.upsert({
461 saveReplay: videoObject.liveSaveReplay,
462 permanentLive: videoObject.permanentLive,
463 videoId: video.id
464 }, { transaction: t, returning: true })
465
466 videoUpdated.VideoLive = videoLive
467 } else { // Delete existing live if it exists
468 await VideoLiveModel.destroy({
469 where: {
470 videoId: video.id
471 },
472 transaction: t
473 })
474
475 videoUpdated.VideoLive = null
476 }
477 }
478
479 return videoUpdated
480 })
481
482 await autoBlacklistVideoIfNeeded({
483 video: videoUpdated,
484 user: undefined,
485 isRemote: true,
486 isNew: false,
487 transaction: undefined
488 })
489
490 // Notify our users?
491 if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
492
493 if (videoUpdated.isLive) {
494 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
495 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
496 }
497
498 logger.info('Remote video with uuid %s updated', videoObject.uuid)
499
500 return videoUpdated
501 } catch (err) {
502 if (video !== undefined && videoFieldsSave !== undefined) {
503 resetSequelizeInstance(video, videoFieldsSave)
504 }
505
506 // This is just a debug because we will retry the insert
507 logger.debug('Cannot update the remote video.', { err })
508 throw err
509 }
510}
511
512async function refreshVideoIfNeeded (options: {
513 video: MVideoThumbnail
514 fetchedType: VideoFetchByUrlType
515 syncParam: SyncParam
516}): Promise<MVideoThumbnail> {
517 if (!options.video.isOutdated()) return options.video
518
519 // We need more attributes if the argument video was fetched with not enough joints
520 const video = options.fetchedType === 'all'
521 ? options.video as MVideoAccountLightBlacklistAllFiles
522 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
523
524 try {
525 const { videoObject } = await fetchRemoteVideo(video.url)
526
527 if (videoObject === undefined) {
528 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
529
530 await video.setAsRefreshed()
531 return video
532 }
533
534 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
535
536 const updateOptions = {
537 video,
538 videoObject,
539 account: channelActor.VideoChannel.Account,
540 channel: channelActor.VideoChannel
541 }
542 await updateVideoFromAP(updateOptions)
543 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
544
545 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
546
547 return video
548 } catch (err) {
549 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
550 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
551
552 // Video does not exist anymore
553 await video.destroy()
554 return undefined
555 }
556
557 logger.warn('Cannot refresh video %s.', options.video.url, { err })
558
559 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
560
561 // Don't refresh in loop
562 await video.setAsRefreshed()
563 return video
564 }
565}
566
567export {
568 updateVideoFromAP,
569 refreshVideoIfNeeded,
570 federateVideoIfNeeded,
571 fetchRemoteVideo,
572 getOrCreateVideoAndAccountAndChannel,
573 fetchRemoteVideoDescription,
574 getOrCreateVideoChannelFromVideoObject
575}
576
577// ---------------------------------------------------------------------------
578
579function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
580 const urlMediaType = url.mediaType
581
582 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
583}
584
585function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
586 return url && url.mediaType === 'application/x-mpegURL'
587}
588
589function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
590 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
591}
592
593function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
594 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
595}
596
597function isAPHashTagObject (url: any): url is ActivityHashTagObject {
598 return url && url.type === 'Hashtag'
599}
600
601async function createVideo (videoObject: VideoObject, channel: MChannelAccountLight, waitThumbnail = false) {
602 logger.debug('Adding remote video %s.', videoObject.id)
603
604 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
605 const video = VideoModel.build(videoData) as MVideoThumbnail
606
607 const promiseThumbnail = createVideoMiniatureFromUrl({
608 downloadUrl: getThumbnailFromIcons(videoObject).url,
609 video,
610 type: ThumbnailType.MINIATURE
611 }).catch(err => {
612 logger.error('Cannot create miniature from url.', { err })
613 return undefined
614 })
615
616 let thumbnailModel: MThumbnail
617 if (waitThumbnail === true) {
618 thumbnailModel = await promiseThumbnail
619 }
620
621 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
622 try {
623 const sequelizeOptions = { transaction: t }
624
625 const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
626 videoCreated.VideoChannel = channel
627
628 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
629
630 const previewIcon = getPreviewFromIcons(videoObject)
631 if (previewIcon) {
632 const previewModel = createPlaceholderThumbnail({
633 fileUrl: previewIcon.url,
634 video: videoCreated,
635 type: ThumbnailType.PREVIEW,
636 size: previewIcon
637 })
638
639 await videoCreated.addAndSaveThumbnail(previewModel, t)
640 }
641
642 // Process files
643 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
644
645 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
646 const videoFiles = await Promise.all(videoFilePromises)
647
648 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
649 videoCreated.VideoStreamingPlaylists = []
650
651 for (const playlistAttributes of streamingPlaylistsAttributes) {
652 const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo
653 playlist.Video = videoCreated
654
655 const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject)
656 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
657 playlist.VideoFiles = await Promise.all(videoFilePromises)
658
659 videoCreated.VideoStreamingPlaylists.push(playlist)
660 }
661
662 // Process tags
663 const tags = videoObject.tag
664 .filter(isAPHashTagObject)
665 .map(t => t.name)
666 await setVideoTags({ video: videoCreated, tags, transaction: t })
667
668 // Process captions
669 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
670 const caption = new VideoCaptionModel({
671 videoId: videoCreated.id,
672 filename: VideoCaptionModel.generateCaptionName(c.identifier),
673 language: c.identifier,
674 fileUrl: c.url
675 }) as MVideoCaption
676
677 return VideoCaptionModel.insertOrReplaceLanguage(caption, t)
678 })
679 await Promise.all(videoCaptionsPromises)
680
681 // Process trackers
682 {
683 const trackers = getTrackerUrls(videoObject, videoCreated)
684 await setVideoTrackers({ video: videoCreated, trackers, transaction: t })
685 }
686
687 videoCreated.VideoFiles = videoFiles
688
689 if (videoCreated.isLive) {
690 const videoLive = new VideoLiveModel({
691 streamKey: null,
692 saveReplay: videoObject.liveSaveReplay,
693 permanentLive: videoObject.permanentLive,
694 videoId: videoCreated.id
695 })
696
697 videoCreated.VideoLive = await videoLive.save({ transaction: t })
698 }
699
700 // We added a video in this channel, set it as updated
701 await channel.setAsUpdated(t)
702
703 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
704 video: videoCreated,
705 user: undefined,
706 isRemote: true,
707 isNew: true,
708 transaction: t
709 })
710
711 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
712
713 return { autoBlacklisted, videoCreated }
714 } catch (err) {
715 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
716 // Remove thumbnail
717 if (thumbnailModel) await thumbnailModel.removeThumbnail()
718
719 throw err
720 }
721 })
722
723 if (waitThumbnail === false) {
724 // Error is already caught above
725 // eslint-disable-next-line @typescript-eslint/no-floating-promises
726 promiseThumbnail.then(thumbnailModel => {
727 if (!thumbnailModel) return
728
729 thumbnailModel = videoCreated.id
730
731 return thumbnailModel.save()
732 })
733 }
734
735 return { autoBlacklisted, videoCreated }
736}
737
738function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
739 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
740 ? VideoPrivacy.PUBLIC
741 : VideoPrivacy.UNLISTED
742
743 const duration = videoObject.duration.replace(/[^\d]+/, '')
744 const language = videoObject.language?.identifier
745
746 const category = videoObject.category
747 ? parseInt(videoObject.category.identifier, 10)
748 : undefined
749
750 const licence = videoObject.licence
751 ? parseInt(videoObject.licence.identifier, 10)
752 : undefined
753
754 const description = videoObject.content || null
755 const support = videoObject.support || null
756
757 return {
758 name: videoObject.name,
759 uuid: videoObject.uuid,
760 url: videoObject.id,
761 category,
762 licence,
763 language,
764 description,
765 support,
766 nsfw: videoObject.sensitive,
767 commentsEnabled: videoObject.commentsEnabled,
768 downloadEnabled: videoObject.downloadEnabled,
769 waitTranscoding: videoObject.waitTranscoding,
770 isLive: videoObject.isLiveBroadcast,
771 state: videoObject.state,
772 channelId: videoChannel.id,
773 duration: parseInt(duration, 10),
774 createdAt: new Date(videoObject.published),
775 publishedAt: new Date(videoObject.published),
776
777 originallyPublishedAt: videoObject.originallyPublishedAt
778 ? new Date(videoObject.originallyPublishedAt)
779 : null,
780
781 updatedAt: new Date(videoObject.updated),
782 views: videoObject.views,
783 likes: 0,
784 dislikes: 0,
785 remote: true,
786 privacy
787 }
788}
789
790function videoFileActivityUrlToDBAttributes (
791 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
792 urls: (ActivityTagObject | ActivityUrlObject)[]
793) {
794 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
795
796 if (fileUrls.length === 0) return []
797
798 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
799 for (const fileUrl of fileUrls) {
800 // Fetch associated magnet uri
801 const magnet = urls.filter(isAPMagnetUrlObject)
802 .find(u => u.height === fileUrl.height)
803
804 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
805
806 const parsed = magnetUtil.decode(magnet.href)
807 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
808 throw new Error('Cannot parse magnet URI ' + magnet.href)
809 }
810
811 const torrentUrl = Array.isArray(parsed.xs)
812 ? parsed.xs[0]
813 : parsed.xs
814
815 // Fetch associated metadata url, if any
816 const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
817 .find(u => {
818 return u.height === fileUrl.height &&
819 u.fps === fileUrl.fps &&
820 u.rel.includes(fileUrl.mediaType)
821 })
822
823 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
824 const resolution = fileUrl.height
825 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
826 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
827
828 const attribute = {
829 extname,
830 infoHash: parsed.infoHash,
831 resolution,
832 size: fileUrl.size,
833 fps: fileUrl.fps || -1,
834 metadataUrl: metadata?.href,
835
836 // Use the name of the remote file because we don't proxify video file requests
837 filename: basename(fileUrl.href),
838 fileUrl: fileUrl.href,
839
840 torrentUrl,
841 // Use our own torrent name since we proxify torrent requests
842 torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
843
844 // This is a video file owned by a video or by a streaming playlist
845 videoId,
846 videoStreamingPlaylistId
847 }
848
849 attributes.push(attribute)
850 }
851
852 return attributes
853}
854
855function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
856 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
857 if (playlistUrls.length === 0) return []
858
859 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
860 for (const playlistUrlObject of playlistUrls) {
861 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
862
863 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
864
865 // FIXME: backward compatibility introduced in v2.1.0
866 if (files.length === 0) files = videoFiles
867
868 if (!segmentsSha256UrlObject) {
869 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
870 continue
871 }
872
873 const attribute = {
874 type: VideoStreamingPlaylistType.HLS,
875 playlistUrl: playlistUrlObject.href,
876 segmentsSha256Url: segmentsSha256UrlObject.href,
877 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
878 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
879 videoId: video.id,
880 tagAPObject: playlistUrlObject.tag
881 }
882
883 attributes.push(attribute)
884 }
885
886 return attributes
887}
888
889function getThumbnailFromIcons (videoObject: VideoObject) {
890 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
891 // Fallback if there are not valid icons
892 if (validIcons.length === 0) validIcons = videoObject.icon
893
894 return minBy(validIcons, 'width')
895}
896
897function getPreviewFromIcons (videoObject: VideoObject) {
898 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
899
900 return maxBy(validIcons, 'width')
901}
902
903function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
904 let wsFound = false
905
906 const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
907 .map((u: ActivityTrackerUrlObject) => {
908 if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
909
910 return u.href
911 })
912
913 if (wsFound) return trackers
914
915 return [
916 buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
917 buildRemoteVideoBaseUrl(video, '/tracker/announce')
918 ]
919}
920
921async function setVideoTrackers (options: {
922 video: MVideo
923 trackers: string[]
924 transaction?: Transaction
925}) {
926 const { video, trackers, transaction } = options
927
928 const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
929
930 await video.$set('Trackers', trackerInstances, { transaction })
931}
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts
new file mode 100644
index 000000000..bd0c54b0c
--- /dev/null
+++ b/server/lib/activitypub/videos/federate.ts
@@ -0,0 +1,36 @@
1import { Transaction } from 'sequelize/types'
2import { isArray } from '@server/helpers/custom-validators/misc'
3import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models'
4import { sendCreateVideo, sendUpdateVideo } from '../send'
5import { shareVideoByServerAndChannel } from '../share'
6
7async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) {
8 const video = videoArg as MVideoAP
9
10 if (
11 // Check this is not a blacklisted video, or unfederated blacklisted video
12 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
13 // Check the video is public/unlisted and published
14 video.hasPrivacyForFederation() && video.hasStateForFederation()
15 ) {
16 // Fetch more attributes that we will need to serialize in AP object
17 if (isArray(video.VideoCaptions) === false) {
18 video.VideoCaptions = await video.$get('VideoCaptions', {
19 attributes: [ 'filename', 'language' ],
20 transaction
21 })
22 }
23
24 if (isNewVideo) {
25 // Now we'll add the video's meta data to our followers
26 await sendCreateVideo(video, transaction)
27 await shareVideoByServerAndChannel(video, transaction)
28 } else {
29 await sendUpdateVideo(video, transaction)
30 }
31 }
32}
33
34export {
35 federateVideoIfNeeded
36}
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts
new file mode 100644
index 000000000..f3e2f0625
--- /dev/null
+++ b/server/lib/activitypub/videos/get.ts
@@ -0,0 +1,113 @@
1import { getAPId } from '@server/helpers/activitypub'
2import { retryTransactionWrapper } from '@server/helpers/database-utils'
3import { JobQueue } from '@server/lib/job-queue'
4import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders'
5import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
6import { APObject } from '@shared/models'
7import { refreshVideoIfNeeded } from './refresh'
8import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
9
10type GetVideoResult <T> = Promise<{
11 video: T
12 created: boolean
13 autoBlacklisted?: boolean
14}>
15
16type GetVideoParamAll = {
17 videoObject: APObject
18 syncParam?: SyncParam
19 fetchType?: 'all'
20 allowRefresh?: boolean
21}
22
23type GetVideoParamImmutable = {
24 videoObject: APObject
25 syncParam?: SyncParam
26 fetchType: 'only-immutable-attributes'
27 allowRefresh: false
28}
29
30type GetVideoParamOther = {
31 videoObject: APObject
32 syncParam?: SyncParam
33 fetchType?: 'all' | 'only-video'
34 allowRefresh?: boolean
35}
36
37function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
38function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
39function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
40
41async function getOrCreateAPVideo (
42 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
43): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
44 // Default params
45 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
46 const fetchType = options.fetchType || 'all'
47 const allowRefresh = options.allowRefresh !== false
48
49 // Get video url
50 const videoUrl = getAPId(options.videoObject)
51 let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
52
53 if (videoFromDatabase) {
54 if (allowRefresh === true) {
55 // Typings ensure allowRefresh === false in only-immutable-attributes fetch type
56 videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam)
57 }
58
59 return { video: videoFromDatabase, created: false }
60 }
61
62 const { videoObject } = await fetchRemoteVideo(videoUrl)
63 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
64
65 // videoUrl is just an alias/rediraction, so process object id instead
66 if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject })
67
68 try {
69 const creator = new APVideoCreator(videoObject)
70 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
71
72 await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
73
74 return { video: videoCreated, created: true, autoBlacklisted }
75 } catch (err) {
76 // Maybe a concurrent getOrCreateAPVideo call created this video
77 if (err.name === 'SequelizeUniqueConstraintError') {
78 const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType)
79 if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false }
80 }
81
82 throw err
83 }
84}
85
86// ---------------------------------------------------------------------------
87
88export {
89 getOrCreateAPVideo
90}
91
92// ---------------------------------------------------------------------------
93
94async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) {
95 if (!video.isOutdated()) return video
96
97 const refreshOptions = {
98 video,
99 fetchedType: fetchType,
100 syncParam
101 }
102
103 if (syncParam.refreshVideo === true) {
104 return refreshVideoIfNeeded(refreshOptions)
105 }
106
107 await JobQueue.Instance.createJobWithPromise({
108 type: 'activitypub-refresher',
109 payload: { type: 'video', url: video.url }
110 })
111
112 return video
113}
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts
new file mode 100644
index 000000000..b22062598
--- /dev/null
+++ b/server/lib/activitypub/videos/index.ts
@@ -0,0 +1,4 @@
1export * from './federate'
2export * from './get'
3export * from './refresh'
4export * from './updater'
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts
new file mode 100644
index 000000000..a7b82f286
--- /dev/null
+++ b/server/lib/activitypub/videos/refresh.ts
@@ -0,0 +1,68 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { PeerTubeRequestError } from '@server/helpers/requests'
3import { ActorFollowScoreCache } from '@server/lib/files-cache'
4import { VideoLoadByUrlType } from '@server/lib/model-loaders'
5import { VideoModel } from '@server/models/video/video'
6import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models'
7import { HttpStatusCode } from '@shared/core-utils'
8import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
9import { APVideoUpdater } from './updater'
10
11async function refreshVideoIfNeeded (options: {
12 video: MVideoThumbnail
13 fetchedType: VideoLoadByUrlType
14 syncParam: SyncParam
15}): Promise<MVideoThumbnail> {
16 if (!options.video.isOutdated()) return options.video
17
18 // We need more attributes if the argument video was fetched with not enough joints
19 const video = options.fetchedType === 'all'
20 ? options.video as MVideoAccountLightBlacklistAllFiles
21 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
22
23 const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url)
24
25 logger.info('Refreshing video %s.', video.url, lTags())
26
27 try {
28 const { videoObject } = await fetchRemoteVideo(video.url)
29
30 if (videoObject === undefined) {
31 logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags())
32
33 await video.setAsRefreshed()
34 return video
35 }
36
37 const videoUpdater = new APVideoUpdater(videoObject, video)
38 await videoUpdater.update()
39
40 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
41
42 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
43
44 return video
45 } catch (err) {
46 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
47 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags())
48
49 // Video does not exist anymore
50 await video.destroy()
51 return undefined
52 }
53
54 logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() })
55
56 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
57
58 // Don't refresh in loop
59 await video.setAsRefreshed()
60 return video
61 }
62}
63
64// ---------------------------------------------------------------------------
65
66export {
67 refreshVideoIfNeeded
68}
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
new file mode 100644
index 000000000..e89c94bcd
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -0,0 +1,173 @@
1import { Transaction } from 'sequelize/types'
2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { deleteNonExistingModels } from '@server/helpers/database-utils'
4import { logger, LoggerTagsFn } from '@server/helpers/logger'
5import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
6import { setVideoTags } from '@server/lib/video'
7import { VideoCaptionModel } from '@server/models/video/video-caption'
8import { VideoFileModel } from '@server/models/video/video-file'
9import { VideoLiveModel } from '@server/models/video/video-live'
10import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
11import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models'
12import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models'
13import { getOrCreateAPActor } from '../../actors'
14import {
15 getCaptionAttributesFromObject,
16 getFileAttributesFromUrl,
17 getLiveAttributesFromObject,
18 getPreviewFromIcons,
19 getStreamingPlaylistAttributesFromObject,
20 getTagsFromObject,
21 getThumbnailFromIcons
22} from './object-to-model-attributes'
23import { getTrackerUrls, setVideoTrackers } from './trackers'
24
25export abstract class APVideoAbstractBuilder {
26 protected abstract videoObject: VideoObject
27 protected abstract lTags: LoggerTagsFn
28
29 protected async getOrCreateVideoChannelFromVideoObject () {
30 const channel = this.videoObject.attributedTo.find(a => a.type === 'Group')
31 if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url)
32
33 if (checkUrlsSameHost(channel.id, this.videoObject.id) !== true) {
34 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${this.videoObject.id}`)
35 }
36
37 return getOrCreateAPActor(channel.id, 'all')
38 }
39
40 protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise<MThumbnail> {
41 return updateVideoMiniatureFromUrl({
42 downloadUrl: getThumbnailFromIcons(this.videoObject).url,
43 video,
44 type: ThumbnailType.MINIATURE
45 }).catch(err => {
46 logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags() })
47
48 return undefined
49 })
50 }
51
52 protected async setPreview (video: MVideoFullLight, t?: Transaction) {
53 // Don't fetch the preview that could be big, create a placeholder instead
54 const previewIcon = getPreviewFromIcons(this.videoObject)
55 if (!previewIcon) return
56
57 const previewModel = updatePlaceholderThumbnail({
58 fileUrl: previewIcon.url,
59 video,
60 type: ThumbnailType.PREVIEW,
61 size: previewIcon
62 })
63
64 await video.addAndSaveThumbnail(previewModel, t)
65 }
66
67 protected async setTags (video: MVideoFullLight, t: Transaction) {
68 const tags = getTagsFromObject(this.videoObject)
69 await setVideoTags({ video, tags, transaction: t })
70 }
71
72 protected async setTrackers (video: MVideoFullLight, t: Transaction) {
73 const trackers = getTrackerUrls(this.videoObject, video)
74 await setVideoTrackers({ video, trackers, transaction: t })
75 }
76
77 protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) {
78 const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t)
79
80 let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject)
81 .map(a => new VideoCaptionModel(a) as MVideoCaption)
82
83 for (const existingCaption of existingCaptions) {
84 // Only keep captions that do not already exist
85 const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption))
86
87 // This caption already exists, we don't need to destroy and create it
88 if (filtered.length !== captionsToCreate.length) {
89 captionsToCreate = filtered
90 continue
91 }
92
93 // Destroy this caption that does not exist anymore
94 await existingCaption.destroy({ transaction: t })
95 }
96
97 for (const captionToCreate of captionsToCreate) {
98 await captionToCreate.save({ transaction: t })
99 }
100 }
101
102 protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) {
103 const attributes = getLiveAttributesFromObject(video, this.videoObject)
104 const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true })
105
106 video.VideoLive = videoLive
107 }
108
109 protected async setWebTorrentFiles (video: MVideoFullLight, t: Transaction) {
110 const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url)
111 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
112
113 // Remove video files that do not exist anymore
114 const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t)
115 await Promise.all(destroyTasks)
116
117 // Update or add other one
118 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
119 video.VideoFiles = await Promise.all(upsertTasks)
120 }
121
122 protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) {
123 const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject, video.VideoFiles || [])
124 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
125
126 // Remove video playlists that do not exist anymore
127 const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t)
128 await Promise.all(destroyTasks)
129
130 video.VideoStreamingPlaylists = []
131
132 for (const playlistAttributes of streamingPlaylistAttributes) {
133
134 const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
135 streamingPlaylistModel.Video = video
136
137 await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t)
138
139 video.VideoStreamingPlaylists.push(streamingPlaylistModel)
140 }
141 }
142
143 private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) {
144 const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t })
145
146 return streamingPlaylist as MStreamingPlaylistFilesVideo
147 }
148
149 private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) {
150 const playlist = video.VideoStreamingPlaylists.find(s => s.type === type)
151 if (!playlist) return []
152
153 return playlist.VideoFiles
154 }
155
156 private async setStreamingPlaylistFiles (
157 video: MVideoFullLight,
158 playlistModel: MStreamingPlaylistFilesVideo,
159 tagObjects: ActivityTagObject[],
160 t: Transaction
161 ) {
162 const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type)
163
164 const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
165
166 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
167 await Promise.all(destroyTasks)
168
169 // Update or add other one
170 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
171 playlistModel.VideoFiles = await Promise.all(upsertTasks)
172 }
173}
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
new file mode 100644
index 000000000..ad3b88936
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -0,0 +1,88 @@
1
2import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
3import { sequelizeTypescript } from '@server/initializers/database'
4import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
5import { VideoModel } from '@server/models/video/video'
6import { MThumbnail, MVideoFullLight, MVideoThumbnail } from '@server/types/models'
7import { VideoObject } from '@shared/models'
8import { APVideoAbstractBuilder } from './abstract-builder'
9import { getVideoAttributesFromObject } from './object-to-model-attributes'
10
11export class APVideoCreator extends APVideoAbstractBuilder {
12 protected lTags: LoggerTagsFn
13
14 constructor (protected readonly videoObject: VideoObject) {
15 super()
16
17 this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id)
18 }
19
20 async create (waitThumbnail = false) {
21 logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags())
22
23 const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
24 const channel = channelActor.VideoChannel
25
26 const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to)
27 const video = VideoModel.build(videoData) as MVideoThumbnail
28
29 const promiseThumbnail = this.tryToGenerateThumbnail(video)
30
31 let thumbnailModel: MThumbnail
32 if (waitThumbnail === true) {
33 thumbnailModel = await promiseThumbnail
34 }
35
36 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
37 try {
38 const videoCreated = await video.save({ transaction: t }) as MVideoFullLight
39 videoCreated.VideoChannel = channel
40
41 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
42
43 await this.setPreview(videoCreated, t)
44 await this.setWebTorrentFiles(videoCreated, t)
45 await this.setStreamingPlaylists(videoCreated, t)
46 await this.setTags(videoCreated, t)
47 await this.setTrackers(videoCreated, t)
48 await this.insertOrReplaceCaptions(videoCreated, t)
49 await this.insertOrReplaceLive(videoCreated, t)
50
51 // We added a video in this channel, set it as updated
52 await channel.setAsUpdated(t)
53
54 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
55 video: videoCreated,
56 user: undefined,
57 isRemote: true,
58 isNew: true,
59 transaction: t
60 })
61
62 logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags())
63
64 return { autoBlacklisted, videoCreated }
65 } catch (err) {
66 // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released
67 // Remove thumbnail
68 if (thumbnailModel) await thumbnailModel.removeThumbnail()
69
70 throw err
71 }
72 })
73
74 if (waitThumbnail === false) {
75 // Error is already caught above
76 // eslint-disable-next-line @typescript-eslint/no-floating-promises
77 promiseThumbnail.then(thumbnailModel => {
78 if (!thumbnailModel) return
79
80 thumbnailModel = videoCreated.id
81
82 return thumbnailModel.save()
83 })
84 }
85
86 return { autoBlacklisted, videoCreated }
87 }
88}
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts
new file mode 100644
index 000000000..951403493
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/index.ts
@@ -0,0 +1,6 @@
1export * from './abstract-builder'
2export * from './creator'
3export * from './object-to-model-attributes'
4export * from './trackers'
5export * from './url-to-object'
6export * from './video-sync-attributes'
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..85548428c
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -0,0 +1,256 @@
1import { maxBy, minBy } from 'lodash'
2import * as magnetUtil from 'magnet-uri'
3import { basename } from 'path'
4import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos'
5import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos'
6import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths'
10import { VideoFileModel } from '@server/models/video/video-file'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { FilteredModelAttributes } from '@server/types'
13import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
14import {
15 ActivityHashTagObject,
16 ActivityMagnetUrlObject,
17 ActivityPlaylistSegmentHashesObject,
18 ActivityPlaylistUrlObject,
19 ActivityTagObject,
20 ActivityUrlObject,
21 ActivityVideoUrlObject,
22 VideoObject,
23 VideoPrivacy,
24 VideoStreamingPlaylistType
25} from '@shared/models'
26import { VideoCaptionModel } from '@server/models/video/video-caption'
27
28function getThumbnailFromIcons (videoObject: VideoObject) {
29 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
30 // Fallback if there are not valid icons
31 if (validIcons.length === 0) validIcons = videoObject.icon
32
33 return minBy(validIcons, 'width')
34}
35
36function getPreviewFromIcons (videoObject: VideoObject) {
37 const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth)
38
39 return maxBy(validIcons, 'width')
40}
41
42function getTagsFromObject (videoObject: VideoObject) {
43 return videoObject.tag
44 .filter(isAPHashTagObject)
45 .map(t => t.name)
46}
47
48function getFileAttributesFromUrl (
49 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
50 urls: (ActivityTagObject | ActivityUrlObject)[]
51) {
52 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
53
54 if (fileUrls.length === 0) return []
55
56 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
57 for (const fileUrl of fileUrls) {
58 // Fetch associated magnet uri
59 const magnet = urls.filter(isAPMagnetUrlObject)
60 .find(u => u.height === fileUrl.height)
61
62 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
63
64 const parsed = magnetUtil.decode(magnet.href)
65 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
66 throw new Error('Cannot parse magnet URI ' + magnet.href)
67 }
68
69 const torrentUrl = Array.isArray(parsed.xs)
70 ? parsed.xs[0]
71 : parsed.xs
72
73 // Fetch associated metadata url, if any
74 const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
75 .find(u => {
76 return u.height === fileUrl.height &&
77 u.fps === fileUrl.fps &&
78 u.rel.includes(fileUrl.mediaType)
79 })
80
81 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
82 const resolution = fileUrl.height
83 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
84 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
85
86 const attribute = {
87 extname,
88 infoHash: parsed.infoHash,
89 resolution,
90 size: fileUrl.size,
91 fps: fileUrl.fps || -1,
92 metadataUrl: metadata?.href,
93
94 // Use the name of the remote file because we don't proxify video file requests
95 filename: basename(fileUrl.href),
96 fileUrl: fileUrl.href,
97
98 torrentUrl,
99 // Use our own torrent name since we proxify torrent requests
100 torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution),
101
102 // This is a video file owned by a video or by a streaming playlist
103 videoId,
104 videoStreamingPlaylistId
105 }
106
107 attributes.push(attribute)
108 }
109
110 return attributes
111}
112
113function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject, videoFiles: MVideoFile[]) {
114 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
115 if (playlistUrls.length === 0) return []
116
117 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
118 for (const playlistUrlObject of playlistUrls) {
119 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
120
121 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
122
123 // FIXME: backward compatibility introduced in v2.1.0
124 if (files.length === 0) files = videoFiles
125
126 if (!segmentsSha256UrlObject) {
127 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
128 continue
129 }
130
131 const attribute = {
132 type: VideoStreamingPlaylistType.HLS,
133 playlistUrl: playlistUrlObject.href,
134 segmentsSha256Url: segmentsSha256UrlObject.href,
135 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
136 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
137 videoId: video.id,
138
139 tagAPObject: playlistUrlObject.tag
140 }
141
142 attributes.push(attribute)
143 }
144
145 return attributes
146}
147
148function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
149 return {
150 saveReplay: videoObject.liveSaveReplay,
151 permanentLive: videoObject.permanentLive,
152 videoId: video.id
153 }
154}
155
156function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
157 return videoObject.subtitleLanguage.map(c => ({
158 videoId: video.id,
159 filename: VideoCaptionModel.generateCaptionName(c.identifier),
160 language: c.identifier,
161 fileUrl: c.url
162 }))
163}
164
165function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) {
166 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
167 ? VideoPrivacy.PUBLIC
168 : VideoPrivacy.UNLISTED
169
170 const duration = videoObject.duration.replace(/[^\d]+/, '')
171 const language = videoObject.language?.identifier
172
173 const category = videoObject.category
174 ? parseInt(videoObject.category.identifier, 10)
175 : undefined
176
177 const licence = videoObject.licence
178 ? parseInt(videoObject.licence.identifier, 10)
179 : undefined
180
181 const description = videoObject.content || null
182 const support = videoObject.support || null
183
184 return {
185 name: videoObject.name,
186 uuid: videoObject.uuid,
187 url: videoObject.id,
188 category,
189 licence,
190 language,
191 description,
192 support,
193 nsfw: videoObject.sensitive,
194 commentsEnabled: videoObject.commentsEnabled,
195 downloadEnabled: videoObject.downloadEnabled,
196 waitTranscoding: videoObject.waitTranscoding,
197 isLive: videoObject.isLiveBroadcast,
198 state: videoObject.state,
199 channelId: videoChannel.id,
200 duration: parseInt(duration, 10),
201 createdAt: new Date(videoObject.published),
202 publishedAt: new Date(videoObject.published),
203
204 originallyPublishedAt: videoObject.originallyPublishedAt
205 ? new Date(videoObject.originallyPublishedAt)
206 : null,
207
208 updatedAt: new Date(videoObject.updated),
209 views: videoObject.views,
210 likes: 0,
211 dislikes: 0,
212 remote: true,
213 privacy
214 }
215}
216
217// ---------------------------------------------------------------------------
218
219export {
220 getThumbnailFromIcons,
221 getPreviewFromIcons,
222
223 getTagsFromObject,
224
225 getFileAttributesFromUrl,
226 getStreamingPlaylistAttributesFromObject,
227
228 getLiveAttributesFromObject,
229 getCaptionAttributesFromObject,
230
231 getVideoAttributesFromObject
232}
233
234// ---------------------------------------------------------------------------
235
236function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
237 const urlMediaType = url.mediaType
238
239 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
240}
241
242function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject {
243 return url && url.mediaType === 'application/x-mpegURL'
244}
245
246function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
247 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
248}
249
250function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
251 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
252}
253
254function isAPHashTagObject (url: any): url is ActivityHashTagObject {
255 return url && url.type === 'Hashtag'
256}
diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts
new file mode 100644
index 000000000..1c5fc4f84
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/trackers.ts
@@ -0,0 +1,43 @@
1import { Transaction } from 'sequelize/types'
2import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
3import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos'
4import { isArray } from '@server/helpers/custom-validators/misc'
5import { REMOTE_SCHEME } from '@server/initializers/constants'
6import { TrackerModel } from '@server/models/server/tracker'
7import { MVideo, MVideoWithHost } from '@server/types/models'
8import { ActivityTrackerUrlObject, VideoObject } from '@shared/models'
9
10function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
11 let wsFound = false
12
13 const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u))
14 .map((u: ActivityTrackerUrlObject) => {
15 if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true
16
17 return u.href
18 })
19
20 if (wsFound) return trackers
21
22 return [
23 buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS),
24 buildRemoteVideoBaseUrl(video, '/tracker/announce')
25 ]
26}
27
28async function setVideoTrackers (options: {
29 video: MVideo
30 trackers: string[]
31 transaction: Transaction
32}) {
33 const { video, trackers, transaction } = options
34
35 const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction)
36
37 await video.$set('Trackers', trackerInstances, { transaction })
38}
39
40export {
41 getTrackerUrls,
42 setVideoTrackers
43}
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts
new file mode 100644
index 000000000..dba3e9480
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/url-to-object.ts
@@ -0,0 +1,25 @@
1import { checkUrlsSameHost } from '@server/helpers/activitypub'
2import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
3import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { doJSONRequest } from '@server/helpers/requests'
5import { VideoObject } from '@shared/models'
6
7const lTags = loggerTagsFactory('ap', 'video')
8
9async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
10 logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl))
11
12 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
13
14 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
15 logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) })
16
17 return { statusCode, videoObject: undefined }
18 }
19
20 return { statusCode, videoObject: body }
21}
22
23export {
24 fetchRemoteVideo
25}
diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts
new file mode 100644
index 000000000..c4e101005
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/video-sync-attributes.ts
@@ -0,0 +1,94 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { JobQueue } from '@server/lib/job-queue'
3import { AccountVideoRateModel } from '@server/models/account/account-video-rate'
4import { VideoCommentModel } from '@server/models/video/video-comment'
5import { VideoShareModel } from '@server/models/video/video-share'
6import { MVideo } from '@server/types/models'
7import { ActivitypubHttpFetcherPayload, VideoObject } from '@shared/models'
8import { crawlCollectionPage } from '../../crawl'
9import { addVideoShares } from '../../share'
10import { addVideoComments } from '../../video-comments'
11import { createRates } from '../../video-rates'
12
13const lTags = loggerTagsFactory('ap', 'video')
14
15type SyncParam = {
16 likes: boolean
17 dislikes: boolean
18 shares: boolean
19 comments: boolean
20 thumbnail: boolean
21 refreshVideo?: boolean
22}
23
24async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoObject, syncParam: SyncParam) {
25 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
26
27 await syncRates('like', video, fetchedVideo, syncParam.likes)
28 await syncRates('dislike', video, fetchedVideo, syncParam.dislikes)
29
30 await syncShares(video, fetchedVideo, syncParam.shares)
31
32 await syncComments(video, fetchedVideo, syncParam.comments)
33}
34
35// ---------------------------------------------------------------------------
36
37export {
38 SyncParam,
39 syncVideoExternalAttributes
40}
41
42// ---------------------------------------------------------------------------
43
44function createJob (payload: ActivitypubHttpFetcherPayload) {
45 return JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })
46}
47
48function syncRates (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
49 const uri = type === 'like'
50 ? fetchedVideo.likes
51 : fetchedVideo.dislikes
52
53 if (!isSync) {
54 const jobType = type === 'like'
55 ? 'video-likes'
56 : 'video-dislikes'
57
58 return createJob({ uri, videoId: video.id, type: jobType })
59 }
60
61 const handler = items => createRates(items, video, type)
62 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, type, crawlStartDate)
63
64 return crawlCollectionPage<string>(uri, handler, cleaner)
65 .catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) }))
66}
67
68function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
69 const uri = fetchedVideo.shares
70
71 if (!isSync) {
72 return createJob({ uri, videoId: video.id, type: 'video-shares' })
73 }
74
75 const handler = items => addVideoShares(items, video)
76 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
77
78 return crawlCollectionPage<string>(uri, handler, cleaner)
79 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) }))
80}
81
82function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
83 const uri = fetchedVideo.comments
84
85 if (!isSync) {
86 return createJob({ uri, videoId: video.id, type: 'video-comments' })
87 }
88
89 const handler = items => addVideoComments(items)
90 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
91
92 return crawlCollectionPage<string>(uri, handler, cleaner)
93 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) }))
94}
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
new file mode 100644
index 000000000..157569414
--- /dev/null
+++ b/server/lib/activitypub/videos/updater.ts
@@ -0,0 +1,166 @@
1import { Transaction } from 'sequelize/types'
2import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
3import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
4import { Notifier } from '@server/lib/notifier'
5import { PeerTubeSocket } from '@server/lib/peertube-socket'
6import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
7import { VideoLiveModel } from '@server/models/video/video-live'
8import { MActor, MChannelAccountLight, MChannelId, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models'
9import { VideoObject, VideoPrivacy } from '@shared/models'
10import { APVideoAbstractBuilder, getVideoAttributesFromObject } from './shared'
11
12export class APVideoUpdater extends APVideoAbstractBuilder {
13 private readonly wasPrivateVideo: boolean
14 private readonly wasUnlistedVideo: boolean
15
16 private readonly videoFieldsSave: any
17
18 private readonly oldVideoChannel: MChannelAccountLight
19
20 protected lTags: LoggerTagsFn
21
22 constructor (
23 protected readonly videoObject: VideoObject,
24 private readonly video: MVideoAccountLightBlacklistAllFiles
25 ) {
26 super()
27
28 this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE
29 this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED
30
31 this.oldVideoChannel = this.video.VideoChannel
32
33 this.videoFieldsSave = this.video.toJSON()
34
35 this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url)
36 }
37
38 async update (overrideTo?: string[]) {
39 logger.debug(
40 'Updating remote video "%s".', this.videoObject.uuid,
41 { videoObject: this.videoObject, ...this.lTags() }
42 )
43
44 try {
45 const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
46
47 const thumbnailModel = await this.tryToGenerateThumbnail(this.video)
48
49 this.checkChannelUpdateOrThrow(channelActor)
50
51 const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo)
52
53 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel)
54
55 await runInReadCommittedTransaction(async t => {
56 await this.setWebTorrentFiles(videoUpdated, t)
57 await this.setStreamingPlaylists(videoUpdated, t)
58 })
59
60 await Promise.all([
61 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
62 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
63 this.setOrDeleteLive(videoUpdated),
64 this.setPreview(videoUpdated)
65 ])
66
67 await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t))
68
69 await autoBlacklistVideoIfNeeded({
70 video: videoUpdated,
71 user: undefined,
72 isRemote: true,
73 isNew: false,
74 transaction: undefined
75 })
76
77 // Notify our users?
78 if (this.wasPrivateVideo || this.wasUnlistedVideo) {
79 Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated)
80 }
81
82 if (videoUpdated.isLive) {
83 PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated)
84 PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
85 }
86
87 logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags())
88
89 return videoUpdated
90 } catch (err) {
91 this.catchUpdateError(err)
92 }
93 }
94
95 // Check we can update the channel: we trust the remote server
96 private checkChannelUpdateOrThrow (newChannelActor: MActor) {
97 if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) {
98 throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
99 }
100
101 if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) {
102 throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
103 }
104 }
105
106 private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) {
107 const to = overrideTo || this.videoObject.to
108 const videoData = getVideoAttributesFromObject(channel, this.videoObject, to)
109 this.video.name = videoData.name
110 this.video.uuid = videoData.uuid
111 this.video.url = videoData.url
112 this.video.category = videoData.category
113 this.video.licence = videoData.licence
114 this.video.language = videoData.language
115 this.video.description = videoData.description
116 this.video.support = videoData.support
117 this.video.nsfw = videoData.nsfw
118 this.video.commentsEnabled = videoData.commentsEnabled
119 this.video.downloadEnabled = videoData.downloadEnabled
120 this.video.waitTranscoding = videoData.waitTranscoding
121 this.video.state = videoData.state
122 this.video.duration = videoData.duration
123 this.video.createdAt = videoData.createdAt
124 this.video.publishedAt = videoData.publishedAt
125 this.video.originallyPublishedAt = videoData.originallyPublishedAt
126 this.video.privacy = videoData.privacy
127 this.video.channelId = videoData.channelId
128 this.video.views = videoData.views
129 this.video.isLive = videoData.isLive
130
131 // Ensures we update the updatedAt attribute, even if main attributes did not change
132 this.video.changed('updatedAt', true)
133
134 return this.video.save({ transaction }) as Promise<MVideoFullLight>
135 }
136
137 private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) {
138 await this.insertOrReplaceCaptions(videoUpdated, t)
139 }
140
141 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
142 if (!this.video.isLive) return
143
144 if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction)
145
146 // Delete existing live if it exists
147 await VideoLiveModel.destroy({
148 where: {
149 videoId: this.video.id
150 },
151 transaction
152 })
153
154 videoUpdated.VideoLive = null
155 }
156
157 private catchUpdateError (err: Error) {
158 if (this.video !== undefined && this.videoFieldsSave !== undefined) {
159 resetSequelizeInstance(this.video, this.videoFieldsSave)
160 }
161
162 // This is just a debug because we will retry the insert
163 logger.debug('Cannot update the remote video.', { err, ...this.lTags() })
164 throw err
165 }
166}
diff --git a/server/lib/auth/oauth-model.ts b/server/lib/auth/oauth-model.ts
index b9c69eb2d..ae728d080 100644
--- a/server/lib/auth/oauth-model.ts
+++ b/server/lib/auth/oauth-model.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import { AccessDeniedError } from 'oauth2-server' 2import { AccessDeniedError } from 'oauth2-server'
3import { PluginManager } from '@server/lib/plugins/plugin-manager' 3import { PluginManager } from '@server/lib/plugins/plugin-manager'
4import { ActorModel } from '@server/models/activitypub/actor' 4import { ActorModel } from '@server/models/actor/actor'
5import { MOAuthClient } from '@server/types/models' 5import { MOAuthClient } from '@server/types/models'
6import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' 6import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token'
7import { MUser } from '@server/types/models/user/user' 7import { MUser } from '@server/types/models/user/user'
@@ -9,7 +9,7 @@ import { UserAdminFlag } from '@shared/models/users/user-flag.model'
9import { UserRole } from '@shared/models/users/user-role' 9import { UserRole } from '@shared/models/users/user-role'
10import { logger } from '../../helpers/logger' 10import { logger } from '../../helpers/logger'
11import { CONFIG } from '../../initializers/config' 11import { CONFIG } from '../../initializers/config'
12import { UserModel } from '../../models/account/user' 12import { UserModel } from '../../models/user/user'
13import { OAuthClientModel } from '../../models/oauth/oauth-client' 13import { OAuthClientModel } from '../../models/oauth/oauth-client'
14import { OAuthTokenModel } from '../../models/oauth/oauth-token' 14import { OAuthTokenModel } from '../../models/oauth/oauth-token'
15import { createUserAccountAndChannelAndPlaylist } from '../user' 15import { createUserAccountAndChannelAndPlaylist } from '../user'
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index 203bd3893..72194416d 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -2,12 +2,14 @@ import * as express from 'express'
2import { readFile } from 'fs-extra' 2import { readFile } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import validator from 'validator' 4import validator from 'validator'
5import { escapeHTML } from '@shared/core-utils/renderer'
6import { HTMLServerConfig } from '@shared/models'
5import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' 7import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n'
6import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' 8import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
7import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' 9import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos'
8import { isTestInstance, sha256 } from '../helpers/core-utils' 10import { isTestInstance, sha256 } from '../helpers/core-utils'
9import { escapeHTML } from '@shared/core-utils/renderer'
10import { logger } from '../helpers/logger' 11import { logger } from '../helpers/logger'
12import { mdToPlainText } from '../helpers/markdown'
11import { CONFIG } from '../initializers/config' 13import { CONFIG } from '../initializers/config'
12import { 14import {
13 ACCEPT_HEADERS, 15 ACCEPT_HEADERS,
@@ -19,12 +21,13 @@ import {
19 WEBSERVER 21 WEBSERVER
20} from '../initializers/constants' 22} from '../initializers/constants'
21import { AccountModel } from '../models/account/account' 23import { AccountModel } from '../models/account/account'
24import { getActivityStreamDuration } from '../models/video/formatter/video-format-utils'
22import { VideoModel } from '../models/video/video' 25import { VideoModel } from '../models/video/video'
23import { VideoChannelModel } from '../models/video/video-channel' 26import { VideoChannelModel } from '../models/video/video-channel'
24import { getActivityStreamDuration } from '../models/video/video-format-utils'
25import { VideoPlaylistModel } from '../models/video/video-playlist' 27import { VideoPlaylistModel } from '../models/video/video-playlist'
26import { MAccountActor, MChannelActor } from '../types/models' 28import { MAccountActor, MChannelActor } from '../types/models'
27import { mdToPlainText } from '../helpers/markdown' 29import { ServerConfigManager } from './server-config-manager'
30import { toCompleteUUID } from '@server/helpers/custom-validators/misc'
28 31
29type Tags = { 32type Tags = {
30 ogType: string 33 ogType: string
@@ -76,7 +79,9 @@ class ClientHtml {
76 return customHtml 79 return customHtml
77 } 80 }
78 81
79 static async getWatchHTMLPage (videoId: string, req: express.Request, res: express.Response) { 82 static async getWatchHTMLPage (videoIdArg: string, req: express.Request, res: express.Response) {
83 const videoId = toCompleteUUID(videoIdArg)
84
80 // Let Angular application handle errors 85 // Let Angular application handle errors
81 if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { 86 if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) {
82 res.status(HttpStatusCode.NOT_FOUND_404) 87 res.status(HttpStatusCode.NOT_FOUND_404)
@@ -134,7 +139,9 @@ class ClientHtml {
134 return customHtml 139 return customHtml
135 } 140 }
136 141
137 static async getWatchPlaylistHTMLPage (videoPlaylistId: string, req: express.Request, res: express.Response) { 142 static async getWatchPlaylistHTMLPage (videoPlaylistIdArg: string, req: express.Request, res: express.Response) {
143 const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg)
144
138 // Let Angular application handle errors 145 // Let Angular application handle errors
139 if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) { 146 if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) {
140 res.status(HttpStatusCode.NOT_FOUND_404) 147 res.status(HttpStatusCode.NOT_FOUND_404)
@@ -196,11 +203,22 @@ class ClientHtml {
196 } 203 }
197 204
198 static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { 205 static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) {
199 return this.getAccountOrChannelHTMLPage(() => AccountModel.loadByNameWithHost(nameWithHost), req, res) 206 const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost)
207 return this.getAccountOrChannelHTMLPage(() => accountModelPromise, req, res)
200 } 208 }
201 209
202 static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { 210 static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) {
203 return this.getAccountOrChannelHTMLPage(() => VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost), req, res) 211 const videoChannelModelPromise = VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost)
212 return this.getAccountOrChannelHTMLPage(() => videoChannelModelPromise, req, res)
213 }
214
215 static async getActorHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) {
216 const [ account, channel ] = await Promise.all([
217 AccountModel.loadByNameWithHost(nameWithHost),
218 VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost)
219 ])
220
221 return this.getAccountOrChannelHTMLPage(() => Promise.resolve(account || channel), req, res)
204 } 222 }
205 223
206 static async getEmbedHTML () { 224 static async getEmbedHTML () {
@@ -209,11 +227,14 @@ class ClientHtml {
209 if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] 227 if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path]
210 228
211 const buffer = await readFile(path) 229 const buffer = await readFile(path)
230 const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig()
212 231
213 let html = buffer.toString() 232 let html = buffer.toString()
214 html = await ClientHtml.addAsyncPluginCSS(html) 233 html = await ClientHtml.addAsyncPluginCSS(html)
215 html = ClientHtml.addCustomCSS(html) 234 html = ClientHtml.addCustomCSS(html)
216 html = ClientHtml.addTitleTag(html) 235 html = ClientHtml.addTitleTag(html)
236 html = ClientHtml.addDescriptionTag(html)
237 html = ClientHtml.addServerConfig(html, serverConfig)
217 238
218 ClientHtml.htmlCache[path] = html 239 ClientHtml.htmlCache[path] = html
219 240
@@ -275,6 +296,7 @@ class ClientHtml {
275 if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] 296 if (!isTestInstance() && ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path]
276 297
277 const buffer = await readFile(path) 298 const buffer = await readFile(path)
299 const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig()
278 300
279 let html = buffer.toString() 301 let html = buffer.toString()
280 302
@@ -283,6 +305,7 @@ class ClientHtml {
283 html = ClientHtml.addFaviconContentHash(html) 305 html = ClientHtml.addFaviconContentHash(html)
284 html = ClientHtml.addLogoContentHash(html) 306 html = ClientHtml.addLogoContentHash(html)
285 html = ClientHtml.addCustomCSS(html) 307 html = ClientHtml.addCustomCSS(html)
308 html = ClientHtml.addServerConfig(html, serverConfig)
286 html = await ClientHtml.addAsyncPluginCSS(html) 309 html = await ClientHtml.addAsyncPluginCSS(html)
287 310
288 ClientHtml.htmlCache[path] = html 311 ClientHtml.htmlCache[path] = html
@@ -355,6 +378,13 @@ class ClientHtml {
355 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) 378 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag)
356 } 379 }
357 380
381 private static addServerConfig (htmlStringPage: string, serverConfig: HTMLServerConfig) {
382 const serverConfigString = JSON.stringify(serverConfig)
383 const configScriptTag = `<script type="application/javascript">window.PeerTubeServerConfig = '${serverConfigString}'</script>`
384
385 return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.SERVER_CONFIG, configScriptTag)
386 }
387
358 private static async addAsyncPluginCSS (htmlStringPage: string) { 388 private static async addAsyncPluginCSS (htmlStringPage: string) {
359 const globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) 389 const globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH)
360 if (globalCSSContent.byteLength === 0) return htmlStringPage 390 if (globalCSSContent.byteLength === 0) return htmlStringPage
@@ -524,11 +554,11 @@ async function serveIndexHTML (req: express.Request, res: express.Response) {
524 return 554 return
525 } catch (err) { 555 } catch (err) {
526 logger.error('Cannot generate HTML page.', err) 556 logger.error('Cannot generate HTML page.', err)
527 return res.sendStatus(HttpStatusCode.INTERNAL_SERVER_ERROR_500) 557 return res.status(HttpStatusCode.INTERNAL_SERVER_ERROR_500).end()
528 } 558 }
529 } 559 }
530 560
531 return res.sendStatus(HttpStatusCode.NOT_ACCEPTABLE_406) 561 return res.status(HttpStatusCode.NOT_ACCEPTABLE_406).end()
532} 562}
533 563
534// --------------------------------------------------------------------------- 564// ---------------------------------------------------------------------------
diff --git a/server/lib/config.ts b/server/lib/config.ts
deleted file mode 100644
index b4c4c9299..000000000
--- a/server/lib/config.ts
+++ /dev/null
@@ -1,255 +0,0 @@
1import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/helpers/signup'
2import { getServerCommit } from '@server/helpers/utils'
3import { CONFIG, isEmailEnabled } from '@server/initializers/config'
4import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants'
5import { RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models'
6import { Hooks } from './plugins/hooks'
7import { PluginManager } from './plugins/plugin-manager'
8import { getThemeOrDefault } from './plugins/theme-utils'
9import { getEnabledResolutions } from './video-transcoding'
10import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
11
12let serverCommit: string
13
14async function getServerConfig (ip?: string): Promise<ServerConfig> {
15 if (serverCommit === undefined) serverCommit = await getServerCommit()
16
17 const { allowed } = await Hooks.wrapPromiseFun(
18 isSignupAllowed,
19 {
20 ip
21 },
22 'filter:api.user.signup.allowed.result'
23 )
24
25 const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip)
26 const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME)
27
28 return {
29 instance: {
30 name: CONFIG.INSTANCE.NAME,
31 shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION,
32 isNSFW: CONFIG.INSTANCE.IS_NSFW,
33 defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
34 defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE,
35 customizations: {
36 javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT,
37 css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS
38 }
39 },
40 search: {
41 remoteUri: {
42 users: CONFIG.SEARCH.REMOTE_URI.USERS,
43 anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS
44 },
45 searchIndex: {
46 enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED,
47 url: CONFIG.SEARCH.SEARCH_INDEX.URL,
48 disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH,
49 isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH
50 }
51 },
52 plugin: {
53 registered: getRegisteredPlugins(),
54 registeredExternalAuths: getExternalAuthsPlugins(),
55 registeredIdAndPassAuths: getIdAndPassAuthPlugins()
56 },
57 theme: {
58 registered: getRegisteredThemes(),
59 default: defaultTheme
60 },
61 email: {
62 enabled: isEmailEnabled()
63 },
64 contactForm: {
65 enabled: CONFIG.CONTACT_FORM.ENABLED
66 },
67 serverVersion: PEERTUBE_VERSION,
68 serverCommit,
69 signup: {
70 allowed,
71 allowedForCurrentIP,
72 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION
73 },
74 transcoding: {
75 hls: {
76 enabled: CONFIG.TRANSCODING.HLS.ENABLED
77 },
78 webtorrent: {
79 enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED
80 },
81 enabledResolutions: getEnabledResolutions('vod'),
82 profile: CONFIG.TRANSCODING.PROFILE,
83 availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod')
84 },
85 live: {
86 enabled: CONFIG.LIVE.ENABLED,
87
88 allowReplay: CONFIG.LIVE.ALLOW_REPLAY,
89 maxDuration: CONFIG.LIVE.MAX_DURATION,
90 maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES,
91 maxUserLives: CONFIG.LIVE.MAX_USER_LIVES,
92
93 transcoding: {
94 enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
95 enabledResolutions: getEnabledResolutions('live'),
96 profile: CONFIG.LIVE.TRANSCODING.PROFILE,
97 availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live')
98 },
99
100 rtmp: {
101 port: CONFIG.LIVE.RTMP.PORT
102 }
103 },
104 import: {
105 videos: {
106 http: {
107 enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
108 },
109 torrent: {
110 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
111 }
112 }
113 },
114 autoBlacklist: {
115 videos: {
116 ofUsers: {
117 enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED
118 }
119 }
120 },
121 avatar: {
122 file: {
123 size: {
124 max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
125 },
126 extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
127 }
128 },
129 banner: {
130 file: {
131 size: {
132 max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
133 },
134 extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
135 }
136 },
137 video: {
138 image: {
139 extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME,
140 size: {
141 max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max
142 }
143 },
144 file: {
145 extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME
146 }
147 },
148 videoCaption: {
149 file: {
150 size: {
151 max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max
152 },
153 extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME
154 }
155 },
156 user: {
157 videoQuota: CONFIG.USER.VIDEO_QUOTA,
158 videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY
159 },
160 trending: {
161 videos: {
162 intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS,
163 algorithms: {
164 enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED,
165 default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT
166 }
167 }
168 },
169 tracker: {
170 enabled: CONFIG.TRACKER.ENABLED
171 },
172
173 followings: {
174 instance: {
175 autoFollowIndex: {
176 indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL
177 }
178 }
179 },
180
181 broadcastMessage: {
182 enabled: CONFIG.BROADCAST_MESSAGE.ENABLED,
183 message: CONFIG.BROADCAST_MESSAGE.MESSAGE,
184 level: CONFIG.BROADCAST_MESSAGE.LEVEL,
185 dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE
186 }
187 }
188}
189
190function getRegisteredThemes () {
191 return PluginManager.Instance.getRegisteredThemes()
192 .map(t => ({
193 name: t.name,
194 version: t.version,
195 description: t.description,
196 css: t.css,
197 clientScripts: t.clientScripts
198 }))
199}
200
201function getRegisteredPlugins () {
202 return PluginManager.Instance.getRegisteredPlugins()
203 .map(p => ({
204 name: p.name,
205 version: p.version,
206 description: p.description,
207 clientScripts: p.clientScripts
208 }))
209}
210
211// ---------------------------------------------------------------------------
212
213export {
214 getServerConfig,
215 getRegisteredThemes,
216 getRegisteredPlugins
217}
218
219// ---------------------------------------------------------------------------
220
221function getIdAndPassAuthPlugins () {
222 const result: RegisteredIdAndPassAuthConfig[] = []
223
224 for (const p of PluginManager.Instance.getIdAndPassAuths()) {
225 for (const auth of p.idAndPassAuths) {
226 result.push({
227 npmName: p.npmName,
228 name: p.name,
229 version: p.version,
230 authName: auth.authName,
231 weight: auth.getWeight()
232 })
233 }
234 }
235
236 return result
237}
238
239function getExternalAuthsPlugins () {
240 const result: RegisteredExternalAuthConfig[] = []
241
242 for (const p of PluginManager.Instance.getExternalAuths()) {
243 for (const auth of p.externalAuths) {
244 result.push({
245 npmName: p.npmName,
246 name: p.name,
247 version: p.version,
248 authName: auth.authName,
249 authDisplayName: auth.authDisplayName()
250 })
251 }
252 }
253
254 return result
255}
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts
index 82c95be80..f896d7af4 100644
--- a/server/lib/job-queue/handlers/activitypub-follow.ts
+++ b/server/lib/job-queue/handlers/activitypub-follow.ts
@@ -1,18 +1,17 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url'
3import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' 3import { ActivitypubFollowPayload } from '@shared/models'
4import { sendFollow } from '../../activitypub/send'
5import { sanitizeHost } from '../../../helpers/core-utils' 4import { sanitizeHost } from '../../../helpers/core-utils'
6import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger'
7import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor'
8import { retryTransactionWrapper } from '../../../helpers/database-utils' 5import { retryTransactionWrapper } from '../../../helpers/database-utils'
9import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 6import { logger } from '../../../helpers/logger'
10import { ActorModel } from '../../../models/activitypub/actor' 7import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants'
11import { Notifier } from '../../notifier'
12import { sequelizeTypescript } from '../../../initializers/database' 8import { sequelizeTypescript } from '../../../initializers/database'
9import { ActorModel } from '../../../models/actor/actor'
10import { ActorFollowModel } from '../../../models/actor/actor-follow'
13import { MActor, MActorFollowActors, MActorFull } from '../../../types/models' 11import { MActor, MActorFollowActors, MActorFull } from '../../../types/models'
14import { ActivitypubFollowPayload } from '@shared/models' 12import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors'
15import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url' 13import { sendFollow } from '../../activitypub/send'
14import { Notifier } from '../../notifier'
16 15
17async function processActivityPubFollow (job: Bull.Job) { 16async function processActivityPubFollow (job: Bull.Job) {
18 const payload = job.data as ActivitypubFollowPayload 17 const payload = job.data as ActivitypubFollowPayload
@@ -26,7 +25,7 @@ async function processActivityPubFollow (job: Bull.Job) {
26 } else { 25 } else {
27 const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) 26 const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP)
28 const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) 27 const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost)
29 targetActor = await getOrCreateActorAndServerAndModel(actorUrl, 'all') 28 targetActor = await getOrCreateAPActor(actorUrl, 'all')
30 } 29 }
31 30
32 if (payload.assertIsChannel && !targetActor.VideoChannel) { 31 if (payload.assertIsChannel && !targetActor.VideoChannel) {
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index c69ff9e83..d4b328635 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -3,7 +3,7 @@ import * as Bull from 'bull'
3import { ActivitypubHttpBroadcastPayload } from '@shared/models' 3import { ActivitypubHttpBroadcastPayload } from '@shared/models'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { doRequest } from '../../../helpers/requests' 5import { doRequest } from '../../../helpers/requests'
6import { BROADCAST_CONCURRENCY, REQUEST_TIMEOUT } from '../../../initializers/constants' 6import { BROADCAST_CONCURRENCY } from '../../../initializers/constants'
7import { ActorFollowScoreCache } from '../../files-cache' 7import { ActorFollowScoreCache } from '../../files-cache'
8import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 8import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
9 9
@@ -19,7 +19,6 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) {
19 method: 'POST' as 'POST', 19 method: 'POST' as 'POST',
20 json: body, 20 json: body,
21 httpSignature: httpSignatureOptions, 21 httpSignature: httpSignatureOptions,
22 timeout: REQUEST_TIMEOUT,
23 headers: buildGlobalHeaders(body) 22 headers: buildGlobalHeaders(body)
24 } 23 }
25 24
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index e210ac3ef..ab9675cae 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,14 +1,13 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' 2import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { AccountModel } from '../../../models/account/account'
5import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 4import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
6import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
7import { VideoCommentModel } from '../../../models/video/video-comment' 6import { VideoCommentModel } from '../../../models/video/video-comment'
8import { VideoShareModel } from '../../../models/video/video-share' 7import { VideoShareModel } from '../../../models/video/video-share'
9import { MAccountDefault, MVideoFullLight } from '../../../types/models' 8import { MVideoFullLight } from '../../../types/models'
10import { crawlCollectionPage } from '../../activitypub/crawl' 9import { crawlCollectionPage } from '../../activitypub/crawl'
11import { createAccountPlaylists } from '../../activitypub/playlist' 10import { createAccountPlaylists } from '../../activitypub/playlists'
12import { processActivities } from '../../activitypub/process' 11import { processActivities } from '../../activitypub/process'
13import { addVideoShares } from '../../activitypub/share' 12import { addVideoShares } from '../../activitypub/share'
14import { addVideoComments } from '../../activitypub/video-comments' 13import { addVideoComments } from '../../activitypub/video-comments'
@@ -22,16 +21,13 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
22 let video: MVideoFullLight 21 let video: MVideoFullLight
23 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId) 22 if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
24 23
25 let account: MAccountDefault
26 if (payload.accountId) account = await AccountModel.load(payload.accountId)
27
28 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = { 24 const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
29 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), 25 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
30 'video-likes': items => createRates(items, video, 'like'), 26 'video-likes': items => createRates(items, video, 'like'),
31 'video-dislikes': items => createRates(items, video, 'dislike'), 27 'video-dislikes': items => createRates(items, video, 'dislike'),
32 'video-shares': items => addVideoShares(items, video), 28 'video-shares': items => addVideoShares(items, video),
33 'video-comments': items => addVideoComments(items), 29 'video-comments': items => addVideoComments(items),
34 'account-playlists': items => createAccountPlaylists(items, account) 30 'account-playlists': items => createAccountPlaylists(items)
35 } 31 }
36 32
37 const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = { 33 const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = {
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index 585dad671..9e561c6b7 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -2,7 +2,6 @@ import * as Bull from 'bull'
2import { ActivitypubHttpUnicastPayload } from '@shared/models' 2import { ActivitypubHttpUnicastPayload } from '@shared/models'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { doRequest } from '../../../helpers/requests' 4import { doRequest } from '../../../helpers/requests'
5import { REQUEST_TIMEOUT } from '../../../initializers/constants'
6import { ActorFollowScoreCache } from '../../files-cache' 5import { ActorFollowScoreCache } from '../../files-cache'
7import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 6import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
8 7
@@ -19,7 +18,6 @@ async function processActivityPubHttpUnicast (job: Bull.Job) {
19 method: 'POST' as 'POST', 18 method: 'POST' as 'POST',
20 json: body, 19 json: body,
21 httpSignature: httpSignatureOptions, 20 httpSignature: httpSignatureOptions,
22 timeout: REQUEST_TIMEOUT,
23 headers: buildGlobalHeaders(body) 21 headers: buildGlobalHeaders(body)
24 } 22 }
25 23
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
index 666e56868..d97e50ebc 100644
--- a/server/lib/job-queue/handlers/activitypub-refresher.ts
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -1,12 +1,12 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists'
3import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos'
4import { loadVideoByUrl } from '@server/lib/model-loaders'
5import { RefreshPayload } from '@shared/models'
2import { logger } from '../../../helpers/logger' 6import { logger } from '../../../helpers/logger'
3import { fetchVideoByUrl } from '../../../helpers/video' 7import { ActorModel } from '../../../models/actor/actor'
4import { refreshActorIfNeeded } from '../../activitypub/actor'
5import { refreshVideoIfNeeded } from '../../activitypub/videos'
6import { ActorModel } from '../../../models/activitypub/actor'
7import { VideoPlaylistModel } from '../../../models/video/video-playlist' 8import { VideoPlaylistModel } from '../../../models/video/video-playlist'
8import { RefreshPayload } from '@shared/models' 9import { refreshActorIfNeeded } from '../../activitypub/actors'
9import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist'
10 10
11async function refreshAPObject (job: Bull.Job) { 11async function refreshAPObject (job: Bull.Job) {
12 const payload = job.data as RefreshPayload 12 const payload = job.data as RefreshPayload
@@ -30,7 +30,7 @@ async function refreshVideo (videoUrl: string) {
30 const fetchType = 'all' as 'all' 30 const fetchType = 'all' as 'all'
31 const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } 31 const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true }
32 32
33 const videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) 33 const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
34 if (videoFromDatabase) { 34 if (videoFromDatabase) {
35 const refreshOptions = { 35 const refreshOptions = {
36 video: videoFromDatabase, 36 video: videoFromDatabase,
@@ -47,7 +47,7 @@ async function refreshActor (actorUrl: string) {
47 const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) 47 const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl)
48 48
49 if (actor) { 49 if (actor) {
50 await refreshActorIfNeeded(actor, fetchType) 50 await refreshActorIfNeeded({ actor, fetchedType: fetchType })
51 } 51 }
52} 52}
53 53
diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts
index 125307843..60ac61afd 100644
--- a/server/lib/job-queue/handlers/actor-keys.ts
+++ b/server/lib/job-queue/handlers/actor-keys.ts
@@ -1,6 +1,6 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { generateAndSaveActorKeys } from '@server/lib/activitypub/actor' 2import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors'
3import { ActorModel } from '@server/models/activitypub/actor' 3import { ActorModel } from '@server/models/actor/actor'
4import { ActorKeysPayload } from '@shared/models' 4import { ActorKeysPayload } from '@shared/models'
5import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
6 6
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
index e8a91450d..37e7c1fad 100644
--- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
+++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
@@ -1,10 +1,10 @@
1import { buildDigest } from '@server/helpers/peertube-crypto'
2import { getServerActor } from '@server/models/application/application'
3import { ContextType } from '@shared/models/activitypub/context'
1import { buildSignedActivity } from '../../../../helpers/activitypub' 4import { buildSignedActivity } from '../../../../helpers/activitypub'
2import { ActorModel } from '../../../../models/activitypub/actor'
3import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants' 5import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants'
6import { ActorModel } from '../../../../models/actor/actor'
4import { MActor } from '../../../../types/models' 7import { MActor } from '../../../../types/models'
5import { getServerActor } from '@server/models/application/application'
6import { buildDigest } from '@server/helpers/peertube-crypto'
7import { ContextType } from '@shared/models/activitypub/context'
8 8
9type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number } 9type Payload <T> = { body: T, contextType?: ContextType, signatureActorId?: number }
10 10
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 71f2cafcd..187cb652e 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -1,9 +1,9 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { copy, stat } from 'fs-extra' 2import { copy, stat } from 'fs-extra'
3import { extname } from 'path' 3import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
5import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 5import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
6import { UserModel } from '@server/models/account/user' 6import { UserModel } from '@server/models/user/user'
7import { MVideoFullLight } from '@server/types/models' 7import { MVideoFullLight } from '@server/types/models'
8import { VideoFileImportPayload } from '@shared/models' 8import { VideoFileImportPayload } from '@shared/models'
9import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' 9import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
@@ -55,7 +55,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
55 const { size } = await stat(inputFilePath) 55 const { size } = await stat(inputFilePath)
56 const fps = await getVideoFileFPS(inputFilePath) 56 const fps = await getVideoFileFPS(inputFilePath)
57 57
58 const fileExt = extname(inputFilePath) 58 const fileExt = getLowercaseExtension(inputFilePath)
59 59
60 const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === videoFileResolution) 60 const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === videoFileResolution)
61 61
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index ed2c5eac0..55498003d 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -1,9 +1,11 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { move, remove, stat } from 'fs-extra' 2import { move, remove, stat } from 'fs-extra'
3import { extname } from 'path' 3import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { retryTransactionWrapper } from '@server/helpers/database-utils' 4import { retryTransactionWrapper } from '@server/helpers/database-utils'
5import { YoutubeDL } from '@server/helpers/youtube-dl'
5import { isPostImportVideoAccepted } from '@server/lib/moderation' 6import { isPostImportVideoAccepted } from '@server/lib/moderation'
6import { Hooks } from '@server/lib/plugins/hooks' 7import { Hooks } from '@server/lib/plugins/hooks'
8import { ServerConfigManager } from '@server/lib/server-config-manager'
7import { isAbleToUploadVideo } from '@server/lib/user' 9import { isAbleToUploadVideo } from '@server/lib/user'
8import { addOptimizeOrMergeAudioJob } from '@server/lib/video' 10import { addOptimizeOrMergeAudioJob } from '@server/lib/video'
9import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 11import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
@@ -23,7 +25,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
23import { logger } from '../../../helpers/logger' 25import { logger } from '../../../helpers/logger'
24import { getSecureTorrentName } from '../../../helpers/utils' 26import { getSecureTorrentName } from '../../../helpers/utils'
25import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 27import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
26import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
27import { CONFIG } from '../../../initializers/config' 28import { CONFIG } from '../../../initializers/config'
28import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 29import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
29import { sequelizeTypescript } from '../../../initializers/database' 30import { sequelizeTypescript } from '../../../initializers/database'
@@ -75,8 +76,10 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
75 videoImportId: videoImport.id 76 videoImportId: videoImport.id
76 } 77 }
77 78
79 const youtubeDL = new YoutubeDL(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
80
78 return processFile( 81 return processFile(
79 () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), 82 () => youtubeDL.downloadYoutubeDLVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT),
80 videoImport, 83 videoImport,
81 options 84 options
82 ) 85 )
@@ -116,7 +119,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
116 const duration = await getDurationFromVideoFile(tempVideoPath) 119 const duration = await getDurationFromVideoFile(tempVideoPath)
117 120
118 // Prepare video file object for creation in database 121 // Prepare video file object for creation in database
119 const fileExt = extname(tempVideoPath) 122 const fileExt = getLowercaseExtension(tempVideoPath)
120 const videoFileData = { 123 const videoFileData = {
121 extname: fileExt, 124 extname: fileExt,
122 resolution: videoFileResolution, 125 resolution: videoFileResolution,
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index d57202ca5..9eba41bf8 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -3,16 +3,16 @@ import { pathExists, readdir, remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { VIDEO_LIVE } from '@server/initializers/constants' 5import { VIDEO_LIVE } from '@server/initializers/constants'
6import { LiveManager } from '@server/lib/live-manager' 6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
7import { generateVideoMiniature } from '@server/lib/thumbnail' 7import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
8import { publishAndFederateIfNeeded } from '@server/lib/video' 9import { publishAndFederateIfNeeded } from '@server/lib/video'
9import { getHLSDirectory } from '@server/lib/video-paths' 10import { getHLSDirectory } from '@server/lib/video-paths'
10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/video-transcoding'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 13import { VideoLiveModel } from '@server/models/video/video-live'
14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
15import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' 15import { MVideo, MVideoLive } from '@server/types/models'
16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
17import { logger } from '../../../helpers/logger' 17import { logger } from '../../../helpers/logger'
18 18
@@ -37,7 +37,7 @@ async function processVideoLiveEnding (job: Bull.Job) {
37 return 37 return
38 } 38 }
39 39
40 LiveManager.Instance.cleanupShaSegments(video.uuid) 40 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid)
41 41
42 if (live.saveReplay !== true) { 42 if (live.saveReplay !== true) {
43 return cleanupLive(video, streamingPlaylist) 43 return cleanupLive(video, streamingPlaylist)
@@ -46,19 +46,10 @@ async function processVideoLiveEnding (job: Bull.Job) {
46 return saveLive(video, live) 46 return saveLive(video, live)
47} 47}
48 48
49async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
50 const hlsDirectory = getHLSDirectory(video)
51
52 await remove(hlsDirectory)
53
54 await streamingPlaylist.destroy()
55}
56
57// --------------------------------------------------------------------------- 49// ---------------------------------------------------------------------------
58 50
59export { 51export {
60 processVideoLiveEnding, 52 processVideoLiveEnding
61 cleanupLive
62} 53}
63 54
64// --------------------------------------------------------------------------- 55// ---------------------------------------------------------------------------
@@ -94,7 +85,7 @@ async function saveLive (video: MVideo, live: MVideoLive) {
94 let durationDone = false 85 let durationDone = false
95 86
96 for (const playlistFile of playlistFiles) { 87 for (const playlistFile of playlistFiles) {
97 const concatenatedTsFile = LiveManager.Instance.buildConcatenatedName(playlistFile) 88 const concatenatedTsFile = buildConcatenatedName(playlistFile)
98 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) 89 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
99 90
100 const probe = await ffprobePromise(concatenatedTsFilePath) 91 const probe = await ffprobePromise(concatenatedTsFilePath)
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 010b95b05..f5ba6f435 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -2,7 +2,7 @@ import * as Bull from 'bull'
2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' 2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils'
3import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' 3import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video'
4import { getVideoFilePath } from '@server/lib/video-paths' 4import { getVideoFilePath } from '@server/lib/video-paths'
5import { UserModel } from '@server/models/account/user' 5import { UserModel } from '@server/models/user/user'
6import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' 6import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models'
7import { 7import {
8 HLSTranscodingPayload, 8 HLSTranscodingPayload,
@@ -15,7 +15,6 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
15import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' 15import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
16import { logger } from '../../../helpers/logger' 16import { logger } from '../../../helpers/logger'
17import { CONFIG } from '../../../initializers/config' 17import { CONFIG } from '../../../initializers/config'
18import { sequelizeTypescript } from '../../../initializers/database'
19import { VideoModel } from '../../../models/video/video' 18import { VideoModel } from '../../../models/video/video'
20import { federateVideoIfNeeded } from '../../activitypub/videos' 19import { federateVideoIfNeeded } from '../../activitypub/videos'
21import { Notifier } from '../../notifier' 20import { Notifier } from '../../notifier'
@@ -24,7 +23,7 @@ import {
24 mergeAudioVideofile, 23 mergeAudioVideofile,
25 optimizeOriginalVideofile, 24 optimizeOriginalVideofile,
26 transcodeNewWebTorrentResolution 25 transcodeNewWebTorrentResolution
27} from '../../video-transcoding' 26} from '../../transcoding/video-transcoding'
28import { JobQueue } from '../job-queue' 27import { JobQueue } from '../job-queue'
29 28
30type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> 29type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any>
@@ -151,35 +150,31 @@ async function onVideoFileOptimizer (
151 // Outside the transaction (IO on disk) 150 // Outside the transaction (IO on disk)
152 const { videoFileResolution, isPortraitMode } = await videoArg.getMaxQualityResolution() 151 const { videoFileResolution, isPortraitMode } = await videoArg.getMaxQualityResolution()
153 152
154 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { 153 // Maybe the video changed in database, refresh it
155 // Maybe the video changed in database, refresh it 154 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid)
156 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t) 155 // Video does not exist anymore
157 // Video does not exist anymore 156 if (!videoDatabase) return undefined
158 if (!videoDatabase) return undefined
159
160 let videoPublished = false
161
162 // Generate HLS version of the original file
163 const originalFileHLSPayload = Object.assign({}, payload, {
164 isPortraitMode,
165 resolution: videoDatabase.getMaxQualityFile().resolution,
166 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
167 copyCodecs: transcodeType !== 'quick-transcode',
168 isMaxQuality: true
169 })
170 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
171
172 const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, videoFileResolution, isPortraitMode, 'webtorrent')
173
174 if (!hasHls && !hasNewResolutions) {
175 // No transcoding to do, it's now published
176 videoPublished = await videoDatabase.publishIfNeededAndSave(t)
177 }
178 157
179 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t) 158 let videoPublished = false
180 159
181 return { videoDatabase, videoPublished } 160 // Generate HLS version of the original file
161 const originalFileHLSPayload = Object.assign({}, payload, {
162 isPortraitMode,
163 resolution: videoDatabase.getMaxQualityFile().resolution,
164 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
165 copyCodecs: transcodeType !== 'quick-transcode',
166 isMaxQuality: true
182 }) 167 })
168 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
169
170 const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, videoFileResolution, isPortraitMode, 'webtorrent')
171
172 if (!hasHls && !hasNewResolutions) {
173 // No transcoding to do, it's now published
174 videoPublished = await videoDatabase.publishIfNeededAndSave(undefined)
175 }
176
177 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo)
183 178
184 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) 179 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
185 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) 180 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index 897235ec0..86d0a271f 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -36,8 +36,8 @@ async function processVideosViews () {
36 } 36 }
37 37
38 await VideoViewModel.create({ 38 await VideoViewModel.create({
39 startDate, 39 startDate: new Date(startDate),
40 endDate, 40 endDate: new Date(endDate),
41 views, 41 views,
42 videoId 42 videoId
43 }) 43 })
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
deleted file mode 100644
index 66b5d119b..000000000
--- a/server/lib/live-manager.ts
+++ /dev/null
@@ -1,621 +0,0 @@
1
2import * as Bluebird from 'bluebird'
3import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import { createServer, Server } from 'net'
7import { basename, join } from 'path'
8import { isTestInstance } from '@server/helpers/core-utils'
9import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
10import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
11import { logger } from '@server/helpers/logger'
12import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
13import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
14import { UserModel } from '@server/models/account/user'
15import { VideoModel } from '@server/models/video/video'
16import { VideoFileModel } from '@server/models/video/video-file'
17import { VideoLiveModel } from '@server/models/video/video-live'
18import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
19import { MStreamingPlaylist, MStreamingPlaylistVideo, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models'
20import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
21import { federateVideoIfNeeded } from './activitypub/videos'
22import { buildSha256Segment } from './hls'
23import { JobQueue } from './job-queue'
24import { cleanupLive } from './job-queue/handlers/video-live-ending'
25import { PeerTubeSocket } from './peertube-socket'
26import { isAbleToUploadVideo } from './user'
27import { getHLSDirectory } from './video-paths'
28import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
29
30import memoizee = require('memoizee')
31const NodeRtmpSession = require('node-media-server/node_rtmp_session')
32const context = require('node-media-server/node_core_ctx')
33const nodeMediaServerLogger = require('node-media-server/node_core_logger')
34
35// Disable node media server logs
36nodeMediaServerLogger.setLogType(0)
37
38const config = {
39 rtmp: {
40 port: CONFIG.LIVE.RTMP.PORT,
41 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
42 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
43 ping: VIDEO_LIVE.RTMP.PING,
44 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
45 },
46 transcoding: {
47 ffmpeg: 'ffmpeg'
48 }
49}
50
51class LiveManager {
52
53 private static instance: LiveManager
54
55 private readonly transSessions = new Map<string, FfmpegCommand>()
56 private readonly videoSessions = new Map<number, string>()
57 // Values are Date().getTime()
58 private readonly watchersPerVideo = new Map<number, number[]>()
59 private readonly segmentsSha256 = new Map<string, Map<string, string>>()
60 private readonly livesPerUser = new Map<number, { liveId: number, videoId: number, size: number }[]>()
61
62 private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
63 return isAbleToUploadVideo(userId, 1000)
64 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
65
66 private readonly hasClientSocketsInBadHealthWithCache = memoizee((sessionId: string) => {
67 return this.hasClientSocketsInBadHealth(sessionId)
68 }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH })
69
70 private rtmpServer: Server
71
72 private constructor () {
73 }
74
75 init () {
76 const events = this.getContext().nodeEvent
77 events.on('postPublish', (sessionId: string, streamPath: string) => {
78 logger.debug('RTMP received stream', { id: sessionId, streamPath })
79
80 const splittedPath = streamPath.split('/')
81 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
82 logger.warn('Live path is incorrect.', { streamPath })
83 return this.abortSession(sessionId)
84 }
85
86 this.handleSession(sessionId, streamPath, splittedPath[2])
87 .catch(err => logger.error('Cannot handle sessions.', { err }))
88 })
89
90 events.on('donePublish', sessionId => {
91 logger.info('Live session ended.', { sessionId })
92 })
93
94 registerConfigChangedHandler(() => {
95 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
96 this.run()
97 return
98 }
99
100 if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) {
101 this.stop()
102 }
103 })
104
105 // Cleanup broken lives, that were terminated by a server restart for example
106 this.handleBrokenLives()
107 .catch(err => logger.error('Cannot handle broken lives.', { err }))
108
109 setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
110 }
111
112 run () {
113 logger.info('Running RTMP server on port %d', config.rtmp.port)
114
115 this.rtmpServer = createServer(socket => {
116 const session = new NodeRtmpSession(config, socket)
117
118 session.run()
119 })
120
121 this.rtmpServer.on('error', err => {
122 logger.error('Cannot run RTMP server.', { err })
123 })
124
125 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT)
126 }
127
128 stop () {
129 logger.info('Stopping RTMP server.')
130
131 this.rtmpServer.close()
132 this.rtmpServer = undefined
133
134 // Sessions is an object
135 this.getContext().sessions.forEach((session: any) => {
136 if (session instanceof NodeRtmpSession) {
137 session.stop()
138 }
139 })
140 }
141
142 isRunning () {
143 return !!this.rtmpServer
144 }
145
146 getSegmentsSha256 (videoUUID: string) {
147 return this.segmentsSha256.get(videoUUID)
148 }
149
150 stopSessionOf (videoId: number) {
151 const sessionId = this.videoSessions.get(videoId)
152 if (!sessionId) return
153
154 this.videoSessions.delete(videoId)
155 this.abortSession(sessionId)
156 }
157
158 getLiveQuotaUsedByUser (userId: number) {
159 const currentLives = this.livesPerUser.get(userId)
160 if (!currentLives) return 0
161
162 return currentLives.reduce((sum, obj) => sum + obj.size, 0)
163 }
164
165 addViewTo (videoId: number) {
166 if (this.videoSessions.has(videoId) === false) return
167
168 let watchers = this.watchersPerVideo.get(videoId)
169
170 if (!watchers) {
171 watchers = []
172 this.watchersPerVideo.set(videoId, watchers)
173 }
174
175 watchers.push(new Date().getTime())
176 }
177
178 cleanupShaSegments (videoUUID: string) {
179 this.segmentsSha256.delete(videoUUID)
180 }
181
182 addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
183 const segmentName = basename(segmentPath)
184 const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, this.buildConcatenatedName(segmentName))
185
186 return readFile(segmentPath)
187 .then(data => appendFile(dest, data))
188 .catch(err => logger.error('Cannot copy segment %s to repay directory.', segmentPath, { err }))
189 }
190
191 buildConcatenatedName (segmentOrPlaylistPath: string) {
192 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
193
194 return 'concat-' + num[1] + '.ts'
195 }
196
197 private processSegments (hlsVideoPath: string, videoUUID: string, videoLive: MVideoLive, segmentPaths: string[]) {
198 Bluebird.mapSeries(segmentPaths, async previousSegment => {
199 // Add sha hash of previous segments, because ffmpeg should have finished generating them
200 await this.addSegmentSha(videoUUID, previousSegment)
201
202 if (videoLive.saveReplay) {
203 await this.addSegmentToReplay(hlsVideoPath, previousSegment)
204 }
205 }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err }))
206 }
207
208 private getContext () {
209 return context
210 }
211
212 private abortSession (id: string) {
213 const session = this.getContext().sessions.get(id)
214 if (session) {
215 session.stop()
216 this.getContext().sessions.delete(id)
217 }
218
219 const transSession = this.transSessions.get(id)
220 if (transSession) {
221 transSession.kill('SIGINT')
222 this.transSessions.delete(id)
223 }
224 }
225
226 private async handleSession (sessionId: string, streamPath: string, streamKey: string) {
227 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
228 if (!videoLive) {
229 logger.warn('Unknown live video with stream key %s.', streamKey)
230 return this.abortSession(sessionId)
231 }
232
233 const video = videoLive.Video
234 if (video.isBlacklisted()) {
235 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey)
236 return this.abortSession(sessionId)
237 }
238
239 // Cleanup old potential live files (could happen with a permanent live)
240 this.cleanupShaSegments(video.uuid)
241
242 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
243 if (oldStreamingPlaylist) {
244 await cleanupLive(video, oldStreamingPlaylist)
245 }
246
247 this.videoSessions.set(video.id, sessionId)
248
249 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
250
251 const session = this.getContext().sessions.get(sessionId)
252 const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
253
254 const [ resolutionResult, fps ] = await Promise.all([
255 getVideoFileResolution(rtmpUrl),
256 getVideoFileFPS(rtmpUrl)
257 ])
258
259 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
260 ? computeResolutionsToTranscode(resolutionResult.videoFileResolution, 'live')
261 : []
262
263 const allResolutions = resolutionsEnabled.concat([ session.videoHeight ])
264
265 logger.info('Will mux/transcode live video of original resolution %d.', session.videoHeight, { allResolutions })
266
267 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
268 videoId: video.id,
269 playlistUrl,
270 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
271 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
272 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
273
274 type: VideoStreamingPlaylistType.HLS
275 }, { returning: true }) as [ MStreamingPlaylist, boolean ]
276
277 return this.runMuxing({
278 sessionId,
279 videoLive,
280 playlist: Object.assign(videoStreamingPlaylist, { Video: video }),
281 rtmpUrl,
282 fps,
283 allResolutions
284 })
285 }
286
287 private async runMuxing (options: {
288 sessionId: string
289 videoLive: MVideoLiveVideo
290 playlist: MStreamingPlaylistVideo
291 rtmpUrl: string
292 fps: number
293 allResolutions: number[]
294 }) {
295 const { sessionId, videoLive, playlist, allResolutions, fps, rtmpUrl } = options
296 const startStreamDateTime = new Date().getTime()
297
298 const user = await UserModel.loadByLiveId(videoLive.id)
299 if (!this.livesPerUser.has(user.id)) {
300 this.livesPerUser.set(user.id, [])
301 }
302
303 const currentUserLive = { liveId: videoLive.id, videoId: videoLive.videoId, size: 0 }
304 const livesOfUser = this.livesPerUser.get(user.id)
305 livesOfUser.push(currentUserLive)
306
307 for (let i = 0; i < allResolutions.length; i++) {
308 const resolution = allResolutions[i]
309
310 const file = new VideoFileModel({
311 resolution,
312 size: -1,
313 extname: '.ts',
314 infoHash: null,
315 fps,
316 videoStreamingPlaylistId: playlist.id
317 })
318
319 VideoFileModel.customUpsert(file, 'streaming-playlist', null)
320 .catch(err => logger.error('Cannot create file for live streaming.', { err }))
321 }
322
323 const outPath = getHLSDirectory(videoLive.Video)
324 await ensureDir(outPath)
325
326 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
327
328 if (videoLive.saveReplay === true) {
329 await ensureDir(replayDirectory)
330 }
331
332 const videoUUID = videoLive.Video.uuid
333
334 const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED
335 ? await getLiveTranscodingCommand({
336 rtmpUrl,
337 outPath,
338 resolutions: allResolutions,
339 fps,
340 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
341 profile: CONFIG.LIVE.TRANSCODING.PROFILE
342 })
343 : getLiveMuxingCommand(rtmpUrl, outPath)
344
345 logger.info('Running live muxing/transcoding for %s.', videoUUID)
346 this.transSessions.set(sessionId, ffmpegExec)
347
348 const tsWatcher = chokidar.watch(outPath + '/*.ts')
349
350 const segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
351 const playlistIdMatcher = /^([\d+])-/
352
353 const addHandler = segmentPath => {
354 logger.debug('Live add handler of %s.', segmentPath)
355
356 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
357
358 const segmentsToProcess = segmentsToProcessPerPlaylist[playlistId] || []
359 this.processSegments(outPath, videoUUID, videoLive, segmentsToProcess)
360
361 segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
362
363 if (this.hasClientSocketsInBadHealthWithCache(sessionId)) {
364 logger.error(
365 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
366 ' Stopping session of video %s.', videoUUID)
367
368 this.stopSessionOf(videoLive.videoId)
369 return
370 }
371
372 // Duration constraint check
373 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
374 logger.info('Stopping session of %s: max duration exceeded.', videoUUID)
375
376 this.stopSessionOf(videoLive.videoId)
377 return
378 }
379
380 // Check user quota if the user enabled replay saving
381 if (videoLive.saveReplay === true) {
382 stat(segmentPath)
383 .then(segmentStat => {
384 currentUserLive.size += segmentStat.size
385 })
386 .then(() => this.isQuotaConstraintValid(user, videoLive))
387 .then(quotaValid => {
388 if (quotaValid !== true) {
389 logger.info('Stopping session of %s: user quota exceeded.', videoUUID)
390
391 this.stopSessionOf(videoLive.videoId)
392 }
393 })
394 .catch(err => logger.error('Cannot stat %s or check quota of %d.', segmentPath, user.id, { err }))
395 }
396 }
397
398 const deleteHandler = segmentPath => this.removeSegmentSha(videoUUID, segmentPath)
399
400 tsWatcher.on('add', p => addHandler(p))
401 tsWatcher.on('unlink', p => deleteHandler(p))
402
403 const masterWatcher = chokidar.watch(outPath + '/master.m3u8')
404 masterWatcher.on('add', async () => {
405 try {
406 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoLive.videoId)
407
408 video.state = VideoState.PUBLISHED
409 await video.save()
410 videoLive.Video = video
411
412 setTimeout(() => {
413 federateVideoIfNeeded(video, false)
414 .catch(err => logger.error('Cannot federate live video %s.', video.url, { err }))
415
416 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
417 }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
418
419 } catch (err) {
420 logger.error('Cannot save/federate live video %d.', videoLive.videoId, { err })
421 } finally {
422 masterWatcher.close()
423 .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err }))
424 }
425 })
426
427 const onFFmpegEnded = () => {
428 logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', rtmpUrl)
429
430 this.transSessions.delete(sessionId)
431
432 this.watchersPerVideo.delete(videoLive.videoId)
433 this.videoSessions.delete(videoLive.videoId)
434
435 const newLivesPerUser = this.livesPerUser.get(user.id)
436 .filter(o => o.liveId !== videoLive.id)
437 this.livesPerUser.set(user.id, newLivesPerUser)
438
439 setTimeout(() => {
440 // Wait latest segments generation, and close watchers
441
442 Promise.all([ tsWatcher.close(), masterWatcher.close() ])
443 .then(() => {
444 // Process remaining segments hash
445 for (const key of Object.keys(segmentsToProcessPerPlaylist)) {
446 this.processSegments(outPath, videoUUID, videoLive, segmentsToProcessPerPlaylist[key])
447 }
448 })
449 .catch(err => logger.error('Cannot close watchers of %s or process remaining hash segments.', outPath, { err }))
450
451 this.onEndTransmuxing(videoLive.Video.id)
452 .catch(err => logger.error('Error in closed transmuxing.', { err }))
453 }, 1000)
454 }
455
456 ffmpegExec.on('error', (err, stdout, stderr) => {
457 onFFmpegEnded()
458
459 // Don't care that we killed the ffmpeg process
460 if (err?.message?.includes('Exiting normally')) return
461
462 logger.error('Live transcoding error.', { err, stdout, stderr })
463
464 this.abortSession(sessionId)
465 })
466
467 ffmpegExec.on('end', () => onFFmpegEnded())
468
469 ffmpegExec.run()
470 }
471
472 private async onEndTransmuxing (videoId: number, cleanupNow = false) {
473 try {
474 const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
475 if (!fullVideo) return
476
477 const live = await VideoLiveModel.loadByVideoId(videoId)
478
479 if (!live.permanentLive) {
480 JobQueue.Instance.createJob({
481 type: 'video-live-ending',
482 payload: {
483 videoId: fullVideo.id
484 }
485 }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
486
487 fullVideo.state = VideoState.LIVE_ENDED
488 } else {
489 fullVideo.state = VideoState.WAITING_FOR_LIVE
490 }
491
492 await fullVideo.save()
493
494 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
495
496 await federateVideoIfNeeded(fullVideo, false)
497 } catch (err) {
498 logger.error('Cannot save/federate new video state of live streaming of video id %d.', videoId, { err })
499 }
500 }
501
502 private async addSegmentSha (videoUUID: string, segmentPath: string) {
503 const segmentName = basename(segmentPath)
504 logger.debug('Adding live sha segment %s.', segmentPath)
505
506 const shaResult = await buildSha256Segment(segmentPath)
507
508 if (!this.segmentsSha256.has(videoUUID)) {
509 this.segmentsSha256.set(videoUUID, new Map())
510 }
511
512 const filesMap = this.segmentsSha256.get(videoUUID)
513 filesMap.set(segmentName, shaResult)
514 }
515
516 private removeSegmentSha (videoUUID: string, segmentPath: string) {
517 const segmentName = basename(segmentPath)
518
519 logger.debug('Removing live sha segment %s.', segmentPath)
520
521 const filesMap = this.segmentsSha256.get(videoUUID)
522 if (!filesMap) {
523 logger.warn('Unknown files map to remove sha for %s.', videoUUID)
524 return
525 }
526
527 if (!filesMap.has(segmentName)) {
528 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath)
529 return
530 }
531
532 filesMap.delete(segmentName)
533 }
534
535 private isDurationConstraintValid (streamingStartTime: number) {
536 const maxDuration = CONFIG.LIVE.MAX_DURATION
537 // No limit
538 if (maxDuration < 0) return true
539
540 const now = new Date().getTime()
541 const max = streamingStartTime + maxDuration
542
543 return now <= max
544 }
545
546 private hasClientSocketsInBadHealth (sessionId: string) {
547 const rtmpSession = this.getContext().sessions.get(sessionId)
548
549 if (!rtmpSession) {
550 logger.warn('Cannot get session %s to check players socket health.', sessionId)
551 return
552 }
553
554 for (const playerSessionId of rtmpSession.players) {
555 const playerSession = this.getContext().sessions.get(playerSessionId)
556
557 if (!playerSession) {
558 logger.error('Cannot get player session %s to check socket health.', playerSession)
559 continue
560 }
561
562 if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) {
563 return true
564 }
565 }
566
567 return false
568 }
569
570 private async isQuotaConstraintValid (user: MUserId, live: MVideoLive) {
571 if (live.saveReplay !== true) return true
572
573 return this.isAbleToUploadVideoWithCache(user.id)
574 }
575
576 private async updateLiveViews () {
577 if (!this.isRunning()) return
578
579 if (!isTestInstance()) logger.info('Updating live video views.')
580
581 for (const videoId of this.watchersPerVideo.keys()) {
582 const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE
583
584 const watchers = this.watchersPerVideo.get(videoId)
585
586 const numWatchers = watchers.length
587
588 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
589 video.views = numWatchers
590 await video.save()
591
592 await federateVideoIfNeeded(video, false)
593
594 PeerTubeSocket.Instance.sendVideoViewsUpdate(video)
595
596 // Only keep not expired watchers
597 const newWatchers = watchers.filter(w => w > notBefore)
598 this.watchersPerVideo.set(videoId, newWatchers)
599
600 logger.debug('New live video views for %s is %d.', video.url, numWatchers)
601 }
602 }
603
604 private async handleBrokenLives () {
605 const videoIds = await VideoModel.listPublishedLiveIds()
606
607 for (const id of videoIds) {
608 await this.onEndTransmuxing(id, true)
609 }
610 }
611
612 static get Instance () {
613 return this.instance || (this.instance = new this())
614 }
615}
616
617// ---------------------------------------------------------------------------
618
619export {
620 LiveManager
621}
diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts
new file mode 100644
index 000000000..8b46800da
--- /dev/null
+++ b/server/lib/live/index.ts
@@ -0,0 +1,4 @@
1export * from './live-manager'
2export * from './live-quota-store'
3export * from './live-segment-sha-store'
4export * from './live-utils'
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
new file mode 100644
index 000000000..014cd3fcf
--- /dev/null
+++ b/server/lib/live/live-manager.ts
@@ -0,0 +1,419 @@
1
2import { createServer, Server } from 'net'
3import { isTestInstance } from '@server/helpers/core-utils'
4import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { logger, loggerTagsFactory } from '@server/helpers/logger'
6import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
8import { UserModel } from '@server/models/user/user'
9import { VideoModel } from '@server/models/video/video'
10import { VideoLiveModel } from '@server/models/video/video-live'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
13import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
14import { federateVideoIfNeeded } from '../activitypub/videos'
15import { JobQueue } from '../job-queue'
16import { PeerTubeSocket } from '../peertube-socket'
17import { LiveQuotaStore } from './live-quota-store'
18import { LiveSegmentShaStore } from './live-segment-sha-store'
19import { cleanupLive } from './live-utils'
20import { MuxingSession } from './shared'
21
22const NodeRtmpSession = require('node-media-server/node_rtmp_session')
23const context = require('node-media-server/node_core_ctx')
24const nodeMediaServerLogger = require('node-media-server/node_core_logger')
25
26// Disable node media server logs
27nodeMediaServerLogger.setLogType(0)
28
29const config = {
30 rtmp: {
31 port: CONFIG.LIVE.RTMP.PORT,
32 chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE,
33 gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE,
34 ping: VIDEO_LIVE.RTMP.PING,
35 ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT
36 },
37 transcoding: {
38 ffmpeg: 'ffmpeg'
39 }
40}
41
42const lTags = loggerTagsFactory('live')
43
44class LiveManager {
45
46 private static instance: LiveManager
47
48 private readonly muxingSessions = new Map<string, MuxingSession>()
49 private readonly videoSessions = new Map<number, string>()
50 // Values are Date().getTime()
51 private readonly watchersPerVideo = new Map<number, number[]>()
52
53 private rtmpServer: Server
54
55 private constructor () {
56 }
57
58 init () {
59 const events = this.getContext().nodeEvent
60 events.on('postPublish', (sessionId: string, streamPath: string) => {
61 logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) })
62
63 const splittedPath = streamPath.split('/')
64 if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) {
65 logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) })
66 return this.abortSession(sessionId)
67 }
68
69 this.handleSession(sessionId, streamPath, splittedPath[2])
70 .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) }))
71 })
72
73 events.on('donePublish', sessionId => {
74 logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
75 })
76
77 registerConfigChangedHandler(() => {
78 if (!this.rtmpServer && CONFIG.LIVE.ENABLED === true) {
79 this.run()
80 return
81 }
82
83 if (this.rtmpServer && CONFIG.LIVE.ENABLED === false) {
84 this.stop()
85 }
86 })
87
88 // Cleanup broken lives, that were terminated by a server restart for example
89 this.handleBrokenLives()
90 .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() }))
91
92 setInterval(() => this.updateLiveViews(), VIEW_LIFETIME.LIVE)
93 }
94
95 run () {
96 logger.info('Running RTMP server on port %d', config.rtmp.port, lTags())
97
98 this.rtmpServer = createServer(socket => {
99 const session = new NodeRtmpSession(config, socket)
100
101 session.run()
102 })
103
104 this.rtmpServer.on('error', err => {
105 logger.error('Cannot run RTMP server.', { err, ...lTags() })
106 })
107
108 this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT)
109 }
110
111 stop () {
112 logger.info('Stopping RTMP server.', lTags())
113
114 this.rtmpServer.close()
115 this.rtmpServer = undefined
116
117 // Sessions is an object
118 this.getContext().sessions.forEach((session: any) => {
119 if (session instanceof NodeRtmpSession) {
120 session.stop()
121 }
122 })
123 }
124
125 isRunning () {
126 return !!this.rtmpServer
127 }
128
129 stopSessionOf (videoId: number) {
130 const sessionId = this.videoSessions.get(videoId)
131 if (!sessionId) return
132
133 this.videoSessions.delete(videoId)
134 this.abortSession(sessionId)
135 }
136
137 addViewTo (videoId: number) {
138 if (this.videoSessions.has(videoId) === false) return
139
140 let watchers = this.watchersPerVideo.get(videoId)
141
142 if (!watchers) {
143 watchers = []
144 this.watchersPerVideo.set(videoId, watchers)
145 }
146
147 watchers.push(new Date().getTime())
148 }
149
150 private getContext () {
151 return context
152 }
153
154 private abortSession (sessionId: string) {
155 const session = this.getContext().sessions.get(sessionId)
156 if (session) {
157 session.stop()
158 this.getContext().sessions.delete(sessionId)
159 }
160
161 const muxingSession = this.muxingSessions.get(sessionId)
162 if (muxingSession) {
163 // Muxing session will fire and event so we correctly cleanup the session
164 muxingSession.abort()
165
166 this.muxingSessions.delete(sessionId)
167 }
168 }
169
170 private async handleSession (sessionId: string, streamPath: string, streamKey: string) {
171 const videoLive = await VideoLiveModel.loadByStreamKey(streamKey)
172 if (!videoLive) {
173 logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId))
174 return this.abortSession(sessionId)
175 }
176
177 const video = videoLive.Video
178 if (video.isBlacklisted()) {
179 logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid))
180 return this.abortSession(sessionId)
181 }
182
183 // Cleanup old potential live files (could happen with a permanent live)
184 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid)
185
186 const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
187 if (oldStreamingPlaylist) {
188 await cleanupLive(video, oldStreamingPlaylist)
189 }
190
191 this.videoSessions.set(video.id, sessionId)
192
193 const rtmpUrl = 'rtmp://127.0.0.1:' + config.rtmp.port + streamPath
194
195 const [ { videoFileResolution }, fps ] = await Promise.all([
196 getVideoFileResolution(rtmpUrl),
197 getVideoFileFPS(rtmpUrl)
198 ])
199
200 const allResolutions = this.buildAllResolutionsToTranscode(videoFileResolution)
201
202 logger.info(
203 'Will mux/transcode live video of original resolution %d.', videoFileResolution,
204 { allResolutions, ...lTags(sessionId, video.uuid) }
205 )
206
207 const streamingPlaylist = await this.createLivePlaylist(video, allResolutions)
208
209 return this.runMuxingSession({
210 sessionId,
211 videoLive,
212 streamingPlaylist,
213 rtmpUrl,
214 fps,
215 allResolutions
216 })
217 }
218
219 private async runMuxingSession (options: {
220 sessionId: string
221 videoLive: MVideoLiveVideo
222 streamingPlaylist: MStreamingPlaylistVideo
223 rtmpUrl: string
224 fps: number
225 allResolutions: number[]
226 }) {
227 const { sessionId, videoLive, streamingPlaylist, allResolutions, fps, rtmpUrl } = options
228 const videoUUID = videoLive.Video.uuid
229 const localLTags = lTags(sessionId, videoUUID)
230
231 const user = await UserModel.loadByLiveId(videoLive.id)
232 LiveQuotaStore.Instance.addNewLive(user.id, videoLive.id)
233
234 const muxingSession = new MuxingSession({
235 context: this.getContext(),
236 user,
237 sessionId,
238 videoLive,
239 streamingPlaylist,
240 rtmpUrl,
241 fps,
242 allResolutions
243 })
244
245 muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags))
246
247 muxingSession.on('bad-socket-health', ({ videoId }) => {
248 logger.error(
249 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' +
250 ' Stopping session of video %s.', videoUUID,
251 localLTags
252 )
253
254 this.stopSessionOf(videoId)
255 })
256
257 muxingSession.on('duration-exceeded', ({ videoId }) => {
258 logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags)
259
260 this.stopSessionOf(videoId)
261 })
262
263 muxingSession.on('quota-exceeded', ({ videoId }) => {
264 logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags)
265
266 this.stopSessionOf(videoId)
267 })
268
269 muxingSession.on('ffmpeg-error', ({ sessionId }) => this.abortSession(sessionId))
270 muxingSession.on('ffmpeg-end', ({ videoId }) => {
271 this.onMuxingFFmpegEnd(videoId)
272 })
273
274 muxingSession.on('after-cleanup', ({ videoId }) => {
275 this.muxingSessions.delete(sessionId)
276
277 muxingSession.destroy()
278
279 return this.onAfterMuxingCleanup(videoId)
280 .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags }))
281 })
282
283 this.muxingSessions.set(sessionId, muxingSession)
284
285 muxingSession.runMuxing()
286 .catch(err => {
287 logger.error('Cannot run muxing.', { err, ...localLTags })
288 this.abortSession(sessionId)
289 })
290 }
291
292 private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) {
293 const videoId = live.videoId
294
295 try {
296 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
297
298 logger.info('Will publish and federate live %s.', video.url, localLTags)
299
300 video.state = VideoState.PUBLISHED
301 await video.save()
302
303 live.Video = video
304
305 setTimeout(() => {
306 federateVideoIfNeeded(video, false)
307 .catch(err => logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }))
308
309 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
310 }, VIDEO_LIVE.SEGMENT_TIME_SECONDS * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION)
311 } catch (err) {
312 logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags })
313 }
314 }
315
316 private onMuxingFFmpegEnd (videoId: number) {
317 this.watchersPerVideo.delete(videoId)
318 this.videoSessions.delete(videoId)
319 }
320
321 private async onAfterMuxingCleanup (videoUUID: string, cleanupNow = false) {
322 try {
323 const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoUUID)
324 if (!fullVideo) return
325
326 const live = await VideoLiveModel.loadByVideoId(fullVideo.id)
327
328 if (!live.permanentLive) {
329 JobQueue.Instance.createJob({
330 type: 'video-live-ending',
331 payload: {
332 videoId: fullVideo.id
333 }
334 }, { delay: cleanupNow ? 0 : VIDEO_LIVE.CLEANUP_DELAY })
335
336 fullVideo.state = VideoState.LIVE_ENDED
337 } else {
338 fullVideo.state = VideoState.WAITING_FOR_LIVE
339 }
340
341 await fullVideo.save()
342
343 PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo)
344
345 await federateVideoIfNeeded(fullVideo, false)
346 } catch (err) {
347 logger.error('Cannot save/federate new video state of live streaming of video %d.', videoUUID, { err, ...lTags(videoUUID) })
348 }
349 }
350
351 private async updateLiveViews () {
352 if (!this.isRunning()) return
353
354 if (!isTestInstance()) logger.info('Updating live video views.', lTags())
355
356 for (const videoId of this.watchersPerVideo.keys()) {
357 const notBefore = new Date().getTime() - VIEW_LIFETIME.LIVE
358
359 const watchers = this.watchersPerVideo.get(videoId)
360
361 const numWatchers = watchers.length
362
363 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
364 video.views = numWatchers
365 await video.save()
366
367 await federateVideoIfNeeded(video, false)
368
369 PeerTubeSocket.Instance.sendVideoViewsUpdate(video)
370
371 // Only keep not expired watchers
372 const newWatchers = watchers.filter(w => w > notBefore)
373 this.watchersPerVideo.set(videoId, newWatchers)
374
375 logger.debug('New live video views for %s is %d.', video.url, numWatchers, lTags())
376 }
377 }
378
379 private async handleBrokenLives () {
380 const videoUUIDs = await VideoModel.listPublishedLiveUUIDs()
381
382 for (const uuid of videoUUIDs) {
383 await this.onAfterMuxingCleanup(uuid, true)
384 }
385 }
386
387 private buildAllResolutionsToTranscode (originResolution: number) {
388 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
389 ? computeResolutionsToTranscode(originResolution, 'live')
390 : []
391
392 return resolutionsEnabled.concat([ originResolution ])
393 }
394
395 private async createLivePlaylist (video: MVideo, allResolutions: number[]) {
396 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
397 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
398 videoId: video.id,
399 playlistUrl,
400 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
401 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
402 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
403
404 type: VideoStreamingPlaylistType.HLS
405 }, { returning: true }) as [ MStreamingPlaylist, boolean ]
406
407 return Object.assign(videoStreamingPlaylist, { Video: video })
408 }
409
410 static get Instance () {
411 return this.instance || (this.instance = new this())
412 }
413}
414
415// ---------------------------------------------------------------------------
416
417export {
418 LiveManager
419}
diff --git a/server/lib/live/live-quota-store.ts b/server/lib/live/live-quota-store.ts
new file mode 100644
index 000000000..8ceccde98
--- /dev/null
+++ b/server/lib/live/live-quota-store.ts
@@ -0,0 +1,48 @@
1class LiveQuotaStore {
2
3 private static instance: LiveQuotaStore
4
5 private readonly livesPerUser = new Map<number, { liveId: number, size: number }[]>()
6
7 private constructor () {
8 }
9
10 addNewLive (userId: number, liveId: number) {
11 if (!this.livesPerUser.has(userId)) {
12 this.livesPerUser.set(userId, [])
13 }
14
15 const currentUserLive = { liveId, size: 0 }
16 const livesOfUser = this.livesPerUser.get(userId)
17 livesOfUser.push(currentUserLive)
18 }
19
20 removeLive (userId: number, liveId: number) {
21 const newLivesPerUser = this.livesPerUser.get(userId)
22 .filter(o => o.liveId !== liveId)
23
24 this.livesPerUser.set(userId, newLivesPerUser)
25 }
26
27 addQuotaTo (userId: number, liveId: number, size: number) {
28 const lives = this.livesPerUser.get(userId)
29 const live = lives.find(l => l.liveId === liveId)
30
31 live.size += size
32 }
33
34 getLiveQuotaOf (userId: number) {
35 const currentLives = this.livesPerUser.get(userId)
36 if (!currentLives) return 0
37
38 return currentLives.reduce((sum, obj) => sum + obj.size, 0)
39 }
40
41 static get Instance () {
42 return this.instance || (this.instance = new this())
43 }
44}
45
46export {
47 LiveQuotaStore
48}
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
new file mode 100644
index 000000000..4af6f3ebf
--- /dev/null
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -0,0 +1,64 @@
1import { basename } from 'path'
2import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { buildSha256Segment } from '../hls'
4
5const lTags = loggerTagsFactory('live')
6
7class LiveSegmentShaStore {
8
9 private static instance: LiveSegmentShaStore
10
11 private readonly segmentsSha256 = new Map<string, Map<string, string>>()
12
13 private constructor () {
14 }
15
16 getSegmentsSha256 (videoUUID: string) {
17 return this.segmentsSha256.get(videoUUID)
18 }
19
20 async addSegmentSha (videoUUID: string, segmentPath: string) {
21 const segmentName = basename(segmentPath)
22 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
23
24 const shaResult = await buildSha256Segment(segmentPath)
25
26 if (!this.segmentsSha256.has(videoUUID)) {
27 this.segmentsSha256.set(videoUUID, new Map())
28 }
29
30 const filesMap = this.segmentsSha256.get(videoUUID)
31 filesMap.set(segmentName, shaResult)
32 }
33
34 removeSegmentSha (videoUUID: string, segmentPath: string) {
35 const segmentName = basename(segmentPath)
36
37 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID))
38
39 const filesMap = this.segmentsSha256.get(videoUUID)
40 if (!filesMap) {
41 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
42 return
43 }
44
45 if (!filesMap.has(segmentName)) {
46 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
47 return
48 }
49
50 filesMap.delete(segmentName)
51 }
52
53 cleanupShaSegments (videoUUID: string) {
54 this.segmentsSha256.delete(videoUUID)
55 }
56
57 static get Instance () {
58 return this.instance || (this.instance = new this())
59 }
60}
61
62export {
63 LiveSegmentShaStore
64}
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
new file mode 100644
index 000000000..e4526c7a5
--- /dev/null
+++ b/server/lib/live/live-utils.ts
@@ -0,0 +1,23 @@
1import { remove } from 'fs-extra'
2import { basename } from 'path'
3import { MStreamingPlaylist, MVideo } from '@server/types/models'
4import { getHLSDirectory } from '../video-paths'
5
6function buildConcatenatedName (segmentOrPlaylistPath: string) {
7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
8
9 return 'concat-' + num[1] + '.ts'
10}
11
12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
13 const hlsDirectory = getHLSDirectory(video)
14
15 await remove(hlsDirectory)
16
17 await streamingPlaylist.destroy()
18}
19
20export {
21 cleanupLive,
22 buildConcatenatedName
23}
diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts
new file mode 100644
index 000000000..c4d1b59ec
--- /dev/null
+++ b/server/lib/live/shared/index.ts
@@ -0,0 +1 @@
export * from './muxing-session'
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
new file mode 100644
index 000000000..26467f060
--- /dev/null
+++ b/server/lib/live/shared/muxing-session.ts
@@ -0,0 +1,346 @@
1
2import * as Bluebird from 'bluebird'
3import * as chokidar from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import { basename, join } from 'path'
7import { EventEmitter } from 'stream'
8import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
10import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
14import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles'
15import { isAbleToUploadVideo } from '../../user'
16import { getHLSDirectory } from '../../video-paths'
17import { LiveQuotaStore } from '../live-quota-store'
18import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils'
20
21import memoizee = require('memoizee')
22
23interface MuxingSessionEvents {
24 'master-playlist-created': ({ videoId: number }) => void
25
26 'bad-socket-health': ({ videoId: number }) => void
27 'duration-exceeded': ({ videoId: number }) => void
28 'quota-exceeded': ({ videoId: number }) => void
29
30 'ffmpeg-end': ({ videoId: number }) => void
31 'ffmpeg-error': ({ sessionId: string }) => void
32
33 'after-cleanup': ({ videoId: number }) => void
34}
35
36declare interface MuxingSession {
37 on<U extends keyof MuxingSessionEvents>(
38 event: U, listener: MuxingSessionEvents[U]
39 ): this
40
41 emit<U extends keyof MuxingSessionEvents>(
42 event: U, ...args: Parameters<MuxingSessionEvents[U]>
43 ): boolean
44}
45
46class MuxingSession extends EventEmitter {
47
48 private ffmpegCommand: FfmpegCommand
49
50 private readonly context: any
51 private readonly user: MUserId
52 private readonly sessionId: string
53 private readonly videoLive: MVideoLiveVideo
54 private readonly streamingPlaylist: MStreamingPlaylistVideo
55 private readonly rtmpUrl: string
56 private readonly fps: number
57 private readonly allResolutions: number[]
58
59 private readonly videoId: number
60 private readonly videoUUID: string
61 private readonly saveReplay: boolean
62
63 private readonly lTags: LoggerTagsFn
64
65 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
66
67 private tsWatcher: chokidar.FSWatcher
68 private masterWatcher: chokidar.FSWatcher
69
70 private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
71 return isAbleToUploadVideo(userId, 1000)
72 }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
73
74 private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => {
75 return this.hasClientSocketInBadHealth(sessionId)
76 }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH })
77
78 constructor (options: {
79 context: any
80 user: MUserId
81 sessionId: string
82 videoLive: MVideoLiveVideo
83 streamingPlaylist: MStreamingPlaylistVideo
84 rtmpUrl: string
85 fps: number
86 allResolutions: number[]
87 }) {
88 super()
89
90 this.context = options.context
91 this.user = options.user
92 this.sessionId = options.sessionId
93 this.videoLive = options.videoLive
94 this.streamingPlaylist = options.streamingPlaylist
95 this.rtmpUrl = options.rtmpUrl
96 this.fps = options.fps
97 this.allResolutions = options.allResolutions
98
99 this.videoId = this.videoLive.Video.id
100 this.videoUUID = this.videoLive.Video.uuid
101
102 this.saveReplay = this.videoLive.saveReplay
103
104 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID)
105 }
106
107 async runMuxing () {
108 this.createFiles()
109
110 const outPath = await this.prepareDirectories()
111
112 this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
113 ? await getLiveTranscodingCommand({
114 rtmpUrl: this.rtmpUrl,
115 outPath,
116 resolutions: this.allResolutions,
117 fps: this.fps,
118 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
119 profile: CONFIG.LIVE.TRANSCODING.PROFILE
120 })
121 : getLiveMuxingCommand(this.rtmpUrl, outPath)
122
123 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
124
125 this.watchTSFiles(outPath)
126 this.watchMasterFile(outPath)
127
128 this.ffmpegCommand.on('error', (err, stdout, stderr) => {
129 this.onFFmpegError(err, stdout, stderr, outPath)
130 })
131
132 this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath))
133
134 this.ffmpegCommand.run()
135 }
136
137 abort () {
138 if (!this.ffmpegCommand) return
139
140 this.ffmpegCommand.kill('SIGINT')
141 }
142
143 destroy () {
144 this.removeAllListeners()
145 this.isAbleToUploadVideoWithCache.clear()
146 this.hasClientSocketInBadHealthWithCache.clear()
147 }
148
149 private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) {
150 this.onFFmpegEnded(outPath)
151
152 // Don't care that we killed the ffmpeg process
153 if (err?.message?.includes('Exiting normally')) return
154
155 logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags })
156
157 this.emit('ffmpeg-error', ({ sessionId: this.sessionId }))
158 }
159
160 private onFFmpegEnded (outPath: string) {
161 logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.rtmpUrl, this.lTags)
162
163 setTimeout(() => {
164 // Wait latest segments generation, and close watchers
165
166 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ])
167 .then(() => {
168 // Process remaining segments hash
169 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
170 this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key])
171 }
172 })
173 .catch(err => {
174 logger.error(
175 'Cannot close watchers of %s or process remaining hash segments.', outPath,
176 { err, ...this.lTags }
177 )
178 })
179
180 this.emit('after-cleanup', { videoId: this.videoId })
181 }, 1000)
182 }
183
184 private watchMasterFile (outPath: string) {
185 this.masterWatcher = chokidar.watch(outPath + '/master.m3u8')
186
187 this.masterWatcher.on('add', async () => {
188 this.emit('master-playlist-created', { videoId: this.videoId })
189
190 this.masterWatcher.close()
191 .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags }))
192 })
193 }
194
195 private watchTSFiles (outPath: string) {
196 const startStreamDateTime = new Date().getTime()
197
198 this.tsWatcher = chokidar.watch(outPath + '/*.ts')
199
200 const playlistIdMatcher = /^([\d+])-/
201
202 const addHandler = async segmentPath => {
203 logger.debug('Live add handler of %s.', segmentPath, this.lTags)
204
205 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
206
207 const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || []
208 this.processSegments(outPath, segmentsToProcess)
209
210 this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
211
212 if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) {
213 this.emit('bad-socket-health', { videoId: this.videoId })
214 return
215 }
216
217 // Duration constraint check
218 if (this.isDurationConstraintValid(startStreamDateTime) !== true) {
219 this.emit('duration-exceeded', { videoId: this.videoId })
220 return
221 }
222
223 // Check user quota if the user enabled replay saving
224 if (await this.isQuotaExceeded(segmentPath) === true) {
225 this.emit('quota-exceeded', { videoId: this.videoId })
226 }
227 }
228
229 const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath)
230
231 this.tsWatcher.on('add', p => addHandler(p))
232 this.tsWatcher.on('unlink', p => deleteHandler(p))
233 }
234
235 private async isQuotaExceeded (segmentPath: string) {
236 if (this.saveReplay !== true) return false
237
238 try {
239 const segmentStat = await stat(segmentPath)
240
241 LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.videoLive.id, segmentStat.size)
242
243 const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id)
244
245 return canUpload !== true
246 } catch (err) {
247 logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags })
248 }
249 }
250
251 private createFiles () {
252 for (let i = 0; i < this.allResolutions.length; i++) {
253 const resolution = this.allResolutions[i]
254
255 const file = new VideoFileModel({
256 resolution,
257 size: -1,
258 extname: '.ts',
259 infoHash: null,
260 fps: this.fps,
261 videoStreamingPlaylistId: this.streamingPlaylist.id
262 })
263
264 VideoFileModel.customUpsert(file, 'streaming-playlist', null)
265 .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags }))
266 }
267 }
268
269 private async prepareDirectories () {
270 const outPath = getHLSDirectory(this.videoLive.Video)
271 await ensureDir(outPath)
272
273 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
274
275 if (this.videoLive.saveReplay === true) {
276 await ensureDir(replayDirectory)
277 }
278
279 return outPath
280 }
281
282 private isDurationConstraintValid (streamingStartTime: number) {
283 const maxDuration = CONFIG.LIVE.MAX_DURATION
284 // No limit
285 if (maxDuration < 0) return true
286
287 const now = new Date().getTime()
288 const max = streamingStartTime + maxDuration
289
290 return now <= max
291 }
292
293 private processSegments (hlsVideoPath: string, segmentPaths: string[]) {
294 Bluebird.mapSeries(segmentPaths, async previousSegment => {
295 // Add sha hash of previous segments, because ffmpeg should have finished generating them
296 await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment)
297
298 if (this.saveReplay) {
299 await this.addSegmentToReplay(hlsVideoPath, previousSegment)
300 }
301 }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags }))
302 }
303
304 private hasClientSocketInBadHealth (sessionId: string) {
305 const rtmpSession = this.context.sessions.get(sessionId)
306
307 if (!rtmpSession) {
308 logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags)
309 return
310 }
311
312 for (const playerSessionId of rtmpSession.players) {
313 const playerSession = this.context.sessions.get(playerSessionId)
314
315 if (!playerSession) {
316 logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags)
317 continue
318 }
319
320 if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) {
321 return true
322 }
323 }
324
325 return false
326 }
327
328 private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
329 const segmentName = basename(segmentPath)
330 const dest = join(hlsVideoPath, VIDEO_LIVE.REPLAY_DIRECTORY, buildConcatenatedName(segmentName))
331
332 try {
333 const data = await readFile(segmentPath)
334
335 await appendFile(dest, data)
336 } catch (err) {
337 logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags })
338 }
339 }
340}
341
342// ---------------------------------------------------------------------------
343
344export {
345 MuxingSession
346}
diff --git a/server/lib/actor-image.ts b/server/lib/local-actor.ts
index f271f0b5b..77667f6b0 100644
--- a/server/lib/actor-image.ts
+++ b/server/lib/local-actor.ts
@@ -1,19 +1,38 @@
1import 'multer' 1import 'multer'
2import { queue } from 'async' 2import { queue } from 'async'
3import * as LRUCache from 'lru-cache' 3import * as LRUCache from 'lru-cache'
4import { extname, join } from 'path' 4import { join } from 'path'
5import { v4 as uuidv4 } from 'uuid' 5import { getLowercaseExtension } from '@server/helpers/core-utils'
6import { ActorImageType } from '@shared/models' 6import { buildUUID } from '@server/helpers/uuid'
7import { ActorModel } from '@server/models/actor/actor'
8import { ActivityPubActorType, ActorImageType } from '@shared/models'
7import { retryTransactionWrapper } from '../helpers/database-utils' 9import { retryTransactionWrapper } from '../helpers/database-utils'
8import { processImage } from '../helpers/image-utils' 10import { processImage } from '../helpers/image-utils'
9import { downloadImage } from '../helpers/requests' 11import { downloadImage } from '../helpers/requests'
10import { CONFIG } from '../initializers/config' 12import { CONFIG } from '../initializers/config'
11import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants' 13import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY, WEBSERVER } from '../initializers/constants'
12import { sequelizeTypescript } from '../initializers/database' 14import { sequelizeTypescript } from '../initializers/database'
13import { MAccountDefault, MChannelDefault } from '../types/models' 15import { MAccountDefault, MActor, MChannelDefault } from '../types/models'
14import { deleteActorImageInstance, updateActorImageInstance } from './activitypub/actor' 16import { deleteActorImageInstance, updateActorImageInstance } from './activitypub/actors'
15import { sendUpdateActor } from './activitypub/send' 17import { sendUpdateActor } from './activitypub/send'
16 18
19function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) {
20 return new ActorModel({
21 type,
22 url,
23 preferredUsername,
24 publicKey: null,
25 privateKey: null,
26 followersCount: 0,
27 followingCount: 0,
28 inboxUrl: url + '/inbox',
29 outboxUrl: url + '/outbox',
30 sharedInboxUrl: WEBSERVER.URL + '/inbox',
31 followersUrl: url + '/followers',
32 followingUrl: url + '/following'
33 }) as MActor
34}
35
17async function updateLocalActorImageFile ( 36async function updateLocalActorImageFile (
18 accountOrChannel: MAccountDefault | MChannelDefault, 37 accountOrChannel: MAccountDefault | MChannelDefault,
19 imagePhysicalFile: Express.Multer.File, 38 imagePhysicalFile: Express.Multer.File,
@@ -23,9 +42,9 @@ async function updateLocalActorImageFile (
23 ? ACTOR_IMAGES_SIZE.AVATARS 42 ? ACTOR_IMAGES_SIZE.AVATARS
24 : ACTOR_IMAGES_SIZE.BANNERS 43 : ACTOR_IMAGES_SIZE.BANNERS
25 44
26 const extension = extname(imagePhysicalFile.filename) 45 const extension = getLowercaseExtension(imagePhysicalFile.filename)
27 46
28 const imageName = uuidv4() + extension 47 const imageName = buildUUID() + extension
29 const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) 48 const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName)
30 await processImage(imagePhysicalFile.path, destination, imageSize) 49 await processImage(imagePhysicalFile.path, destination, imageSize)
31 50
@@ -93,5 +112,6 @@ export {
93 actorImagePathUnsafeCache, 112 actorImagePathUnsafeCache,
94 updateLocalActorImageFile, 113 updateLocalActorImageFile,
95 deleteLocalActorImageFile, 114 deleteLocalActorImageFile,
96 pushActorImageProcessInQueue 115 pushActorImageProcessInQueue,
116 buildActorInstance
97} 117}
diff --git a/server/lib/model-loaders/actor.ts b/server/lib/model-loaders/actor.ts
new file mode 100644
index 000000000..1355d8ee2
--- /dev/null
+++ b/server/lib/model-loaders/actor.ts
@@ -0,0 +1,17 @@
1
2import { ActorModel } from '../../models/actor/actor'
3import { MActorAccountChannelId, MActorFull } from '../../types/models'
4
5type ActorLoadByUrlType = 'all' | 'association-ids'
6
7function loadActorByUrl (url: string, fetchType: ActorLoadByUrlType): Promise<MActorFull | MActorAccountChannelId> {
8 if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url)
9
10 if (fetchType === 'association-ids') return ActorModel.loadByUrl(url)
11}
12
13export {
14 ActorLoadByUrlType,
15
16 loadActorByUrl
17}
diff --git a/server/lib/model-loaders/index.ts b/server/lib/model-loaders/index.ts
new file mode 100644
index 000000000..9e5152cb2
--- /dev/null
+++ b/server/lib/model-loaders/index.ts
@@ -0,0 +1,2 @@
1export * from './actor'
2export * from './video'
diff --git a/server/lib/model-loaders/video.ts b/server/lib/model-loaders/video.ts
new file mode 100644
index 000000000..0a3c15ad8
--- /dev/null
+++ b/server/lib/model-loaders/video.ts
@@ -0,0 +1,73 @@
1import { VideoModel } from '@server/models/video/video'
2import {
3 MVideoAccountLightBlacklistAllFiles,
4 MVideoFormattableDetails,
5 MVideoFullLight,
6 MVideoId,
7 MVideoImmutable,
8 MVideoThumbnail
9} from '@server/types/models'
10import { Hooks } from '../plugins/hooks'
11
12type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes'
13
14function loadVideo (id: number | string, fetchType: 'for-api', userId?: number): Promise<MVideoFormattableDetails>
15function loadVideo (id: number | string, fetchType: 'all', userId?: number): Promise<MVideoFullLight>
16function loadVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable>
17function loadVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise<MVideoThumbnail>
18function loadVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise<MVideoId>
19function loadVideo (
20 id: number | string,
21 fetchType: VideoLoadType,
22 userId?: number
23): Promise<MVideoFullLight | MVideoThumbnail | MVideoId | MVideoImmutable>
24function loadVideo (
25 id: number | string,
26 fetchType: VideoLoadType,
27 userId?: number
28): Promise<MVideoFullLight | MVideoThumbnail | MVideoId | MVideoImmutable> {
29
30 if (fetchType === 'for-api') {
31 return Hooks.wrapPromiseFun(
32 VideoModel.loadForGetAPI,
33 { id, userId },
34 'filter:api.video.get.result'
35 )
36 }
37
38 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
39
40 if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id)
41
42 if (fetchType === 'only-video') return VideoModel.load(id)
43
44 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
45}
46
47type VideoLoadByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
48
49function loadVideoByUrl (url: string, fetchType: 'all'): Promise<MVideoAccountLightBlacklistAllFiles>
50function loadVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable>
51function loadVideoByUrl (url: string, fetchType: 'only-video'): Promise<MVideoThumbnail>
52function loadVideoByUrl (
53 url: string,
54 fetchType: VideoLoadByUrlType
55): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
56function loadVideoByUrl (
57 url: string,
58 fetchType: VideoLoadByUrlType
59): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
60 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
61
62 if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
63
64 if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
65}
66
67export {
68 VideoLoadType,
69 VideoLoadByUrlType,
70
71 loadVideo,
72 loadVideoByUrl
73}
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index 925d64902..14e00518e 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -23,9 +23,9 @@ import { ActivityCreate } from '../../shared/models/activitypub'
23import { VideoObject } from '../../shared/models/activitypub/objects' 23import { VideoObject } from '../../shared/models/activitypub/objects'
24import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' 24import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
25import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' 25import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos'
26import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model' 26import { VideoCommentCreate } from '../../shared/models/videos/comment/video-comment.model'
27import { UserModel } from '../models/account/user' 27import { ActorModel } from '../models/actor/actor'
28import { ActorModel } from '../models/activitypub/actor' 28import { UserModel } from '../models/user/user'
29import { VideoModel } from '../models/video/video' 29import { VideoModel } from '../models/video/video'
30import { VideoCommentModel } from '../models/video/video-comment' 30import { VideoCommentModel } from '../models/video/video-comment'
31import { sendAbuse } from './activitypub/send/send-flag' 31import { sendAbuse } from './activitypub/send/send-flag'
@@ -221,7 +221,7 @@ async function createAbuse (options: {
221 const { isOwned } = await associateFun(abuseInstance) 221 const { isOwned } = await associateFun(abuseInstance)
222 222
223 if (isOwned === false) { 223 if (isOwned === false) {
224 await sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) 224 sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction)
225 } 225 }
226 226
227 const abuseJSON = abuseInstance.toFormattedAdminJSON() 227 const abuseJSON = abuseInstance.toFormattedAdminJSON()
diff --git a/server/lib/notifier.ts b/server/lib/notifier.ts
index da7f7cc05..1f9ff16df 100644
--- a/server/lib/notifier.ts
+++ b/server/lib/notifier.ts
@@ -17,8 +17,8 @@ import { VideoPrivacy, VideoState } from '../../shared/models/videos'
17import { logger } from '../helpers/logger' 17import { logger } from '../helpers/logger'
18import { CONFIG } from '../initializers/config' 18import { CONFIG } from '../initializers/config'
19import { AccountBlocklistModel } from '../models/account/account-blocklist' 19import { AccountBlocklistModel } from '../models/account/account-blocklist'
20import { UserModel } from '../models/account/user' 20import { UserModel } from '../models/user/user'
21import { UserNotificationModel } from '../models/account/user-notification' 21import { UserNotificationModel } from '../models/user/user-notification'
22import { MAbuseFull, MAbuseMessage, MAccountServer, MActorFollowFull, MApplication, MPlugin } from '../types/models' 22import { MAbuseFull, MAbuseMessage, MAccountServer, MActorFollowFull, MApplication, MPlugin } from '../types/models'
23import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../types/models/video' 23import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../types/models/video'
24import { isBlockedByServerOrAccount } from './blocklist' 24import { isBlockedByServerOrAccount } from './blocklist'
diff --git a/server/lib/plugins/hooks.ts b/server/lib/plugins/hooks.ts
index aa92f03cc..5e97b52a0 100644
--- a/server/lib/plugins/hooks.ts
+++ b/server/lib/plugins/hooks.ts
@@ -1,7 +1,7 @@
1import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models/plugins/server-hook.model'
2import { PluginManager } from './plugin-manager'
3import { logger } from '../../helpers/logger'
4import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models'
3import { logger } from '../../helpers/logger'
4import { PluginManager } from './plugin-manager'
5 5
6type PromiseFunction <U, T> = (params: U) => Promise<T> | Bluebird<T> 6type PromiseFunction <U, T> = (params: U) => Promise<T> | Bluebird<T>
7type RawFunction <U, T> = (params: U) => T 7type RawFunction <U, T> = (params: U) => T
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts
index f1bc24d8b..8487672ba 100644
--- a/server/lib/plugins/plugin-helpers-builder.ts
+++ b/server/lib/plugins/plugin-helpers-builder.ts
@@ -15,9 +15,9 @@ import { MPlugin } from '@server/types/models'
15import { PeerTubeHelpers } from '@server/types/plugins' 15import { PeerTubeHelpers } from '@server/types/plugins'
16import { VideoBlacklistCreate } from '@shared/models' 16import { VideoBlacklistCreate } from '@shared/models'
17import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' 17import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
18import { getServerConfig } from '../config' 18import { ServerConfigManager } from '../server-config-manager'
19import { blacklistVideo, unblacklistVideo } from '../video-blacklist' 19import { blacklistVideo, unblacklistVideo } from '../video-blacklist'
20import { UserModel } from '@server/models/account/user' 20import { UserModel } from '@server/models/user/user'
21 21
22function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { 22function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers {
23 const logger = buildPluginLogger(npmName) 23 const logger = buildPluginLogger(npmName)
@@ -147,7 +147,7 @@ function buildConfigHelpers () {
147 }, 147 },
148 148
149 getServerConfig () { 149 getServerConfig () {
150 return getServerConfig() 150 return ServerConfigManager.Instance.getServerConfig()
151 } 151 }
152 } 152 }
153} 153}
diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts
index 165bc91b3..119cee8e0 100644
--- a/server/lib/plugins/plugin-index.ts
+++ b/server/lib/plugins/plugin-index.ts
@@ -1,16 +1,16 @@
1import { sanitizeUrl } from '@server/helpers/core-utils' 1import { sanitizeUrl } from '@server/helpers/core-utils'
2import { ResultList } from '../../../shared/models' 2import { logger } from '@server/helpers/logger'
3import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' 3import { doJSONRequest } from '@server/helpers/requests'
4import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' 4import { CONFIG } from '@server/initializers/config'
5import { PEERTUBE_VERSION } from '@server/initializers/constants'
6import { PluginModel } from '@server/models/server/plugin'
5import { 7import {
8 PeerTubePluginIndex,
9 PeertubePluginIndexList,
6 PeertubePluginLatestVersionRequest, 10 PeertubePluginLatestVersionRequest,
7 PeertubePluginLatestVersionResponse 11 PeertubePluginLatestVersionResponse,
8} from '../../../shared/models/plugins/peertube-plugin-latest-version.model' 12 ResultList
9import { logger } from '../../helpers/logger' 13} from '@shared/models'
10import { doJSONRequest } from '../../helpers/requests'
11import { CONFIG } from '../../initializers/config'
12import { PEERTUBE_VERSION } from '../../initializers/constants'
13import { PluginModel } from '../../models/server/plugin'
14import { PluginManager } from './plugin-manager' 14import { PluginManager } from './plugin-manager'
15 15
16async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { 16async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) {
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index ba9814383..6599bccca 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -4,16 +4,11 @@ import { createReadStream, createWriteStream } from 'fs'
4import { ensureDir, outputFile, readJSON } from 'fs-extra' 4import { ensureDir, outputFile, readJSON } from 'fs-extra'
5import { basename, join } from 'path' 5import { basename, join } from 'path'
6import { MOAuthTokenUser, MUser } from '@server/types/models' 6import { MOAuthTokenUser, MUser } from '@server/types/models'
7import { RegisterServerHookOptions } from '@shared/models/plugins/register-server-hook.model' 7import { getCompleteLocale } from '@shared/core-utils'
8import { ClientScript, PluginPackageJson, PluginTranslation, PluginTranslationPaths, RegisterServerHookOptions } from '@shared/models'
8import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' 9import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks'
9import {
10 ClientScript,
11 PluginPackageJson,
12 PluginTranslationPaths as PackagePluginTranslations
13} from '../../../shared/models/plugins/plugin-package-json.model'
14import { PluginTranslation } from '../../../shared/models/plugins/plugin-translation.model'
15import { PluginType } from '../../../shared/models/plugins/plugin.type' 10import { PluginType } from '../../../shared/models/plugins/plugin.type'
16import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server-hook.model' 11import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model'
17import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins' 12import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins'
18import { logger } from '../../helpers/logger' 13import { logger } from '../../helpers/logger'
19import { CONFIG } from '../../initializers/config' 14import { CONFIG } from '../../initializers/config'
@@ -23,7 +18,6 @@ import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPas
23import { ClientHtml } from '../client-html' 18import { ClientHtml } from '../client-html'
24import { RegisterHelpers } from './register-helpers' 19import { RegisterHelpers } from './register-helpers'
25import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn' 20import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn'
26import { getCompleteLocale } from '@shared/core-utils'
27 21
28export interface RegisteredPlugin { 22export interface RegisteredPlugin {
29 npmName: string 23 npmName: string
@@ -310,22 +304,28 @@ export class PluginManager implements ServerHook {
310 uninstalled: false, 304 uninstalled: false,
311 peertubeEngine: packageJSON.engine.peertube 305 peertubeEngine: packageJSON.engine.peertube
312 }, { returning: true }) 306 }, { returning: true })
313 } catch (err) { 307
314 logger.error('Cannot install plugin %s, removing it...', toInstall, { err }) 308 logger.info('Successful installation of plugin %s.', toInstall)
309
310 await this.registerPluginOrTheme(plugin)
311 } catch (rootErr) {
312 logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr })
315 313
316 try { 314 try {
317 await removeNpmPlugin(npmName) 315 await this.uninstall(npmName)
318 } catch (err) { 316 } catch (err) {
319 logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) 317 logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err })
318
319 try {
320 await removeNpmPlugin(npmName)
321 } catch (err) {
322 logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err })
323 }
320 } 324 }
321 325
322 throw err 326 throw rootErr
323 } 327 }
324 328
325 logger.info('Successful installation of plugin %s.', toInstall)
326
327 await this.registerPluginOrTheme(plugin)
328
329 return plugin 329 return plugin
330 } 330 }
331 331
@@ -431,8 +431,7 @@ export class PluginManager implements ServerHook {
431 431
432 await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) 432 await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath())
433 433
434 library.register(registerOptions) 434 await library.register(registerOptions)
435 .catch(err => logger.error('Cannot register plugin %s.', npmName, { err }))
436 435
437 logger.info('Add plugin %s CSS to global file.', npmName) 436 logger.info('Add plugin %s CSS to global file.', npmName)
438 437
@@ -443,7 +442,7 @@ export class PluginManager implements ServerHook {
443 442
444 // ###################### Translations ###################### 443 // ###################### Translations ######################
445 444
446 private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PackagePluginTranslations) { 445 private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPaths) {
447 for (const locale of Object.keys(translationPaths)) { 446 for (const locale of Object.keys(translationPaths)) {
448 const path = translationPaths[locale] 447 const path = translationPaths[locale]
449 const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) 448 const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path))
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts
index aa69ca2a2..09275f9ba 100644
--- a/server/lib/plugins/register-helpers.ts
+++ b/server/lib/plugins/register-helpers.ts
@@ -26,10 +26,10 @@ import {
26 PluginVideoLicenceManager, 26 PluginVideoLicenceManager,
27 PluginVideoPrivacyManager, 27 PluginVideoPrivacyManager,
28 RegisterServerHookOptions, 28 RegisterServerHookOptions,
29 RegisterServerSettingOptions 29 RegisterServerSettingOptions,
30 serverHookObject
30} from '@shared/models' 31} from '@shared/models'
31import { serverHookObject } from '@shared/models/plugins/server-hook.model' 32import { VideoTranscodingProfilesManager } from '../transcoding/video-transcoding-profiles'
32import { VideoTranscodingProfilesManager } from '../video-transcoding-profiles'
33import { buildPluginHelpers } from './plugin-helpers-builder' 33import { buildPluginHelpers } from './plugin-helpers-builder'
34 34
35type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' 35type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy'
@@ -37,18 +37,20 @@ type VideoConstant = { [key in number | string]: string }
37 37
38type UpdatedVideoConstant = { 38type UpdatedVideoConstant = {
39 [name in AlterableVideoConstant]: { 39 [name in AlterableVideoConstant]: {
40 added: { key: number | string, label: string }[] 40 [ npmName: string]: {
41 deleted: { key: number | string, label: string }[] 41 added: { key: number | string, label: string }[]
42 deleted: { key: number | string, label: string }[]
43 }
42 } 44 }
43} 45}
44 46
45export class RegisterHelpers { 47export class RegisterHelpers {
46 private readonly updatedVideoConstants: UpdatedVideoConstant = { 48 private readonly updatedVideoConstants: UpdatedVideoConstant = {
47 playlistPrivacy: { added: [], deleted: [] }, 49 playlistPrivacy: { },
48 privacy: { added: [], deleted: [] }, 50 privacy: { },
49 language: { added: [], deleted: [] }, 51 language: { },
50 licence: { added: [], deleted: [] }, 52 licence: { },
51 category: { added: [], deleted: [] } 53 category: { }
52 } 54 }
53 55
54 private readonly transcodingProfiles: { 56 private readonly transcodingProfiles: {
@@ -377,7 +379,7 @@ export class RegisterHelpers {
377 const { npmName, type, obj, key } = parameters 379 const { npmName, type, obj, key } = parameters
378 380
379 if (!obj[key]) { 381 if (!obj[key]) {
380 logger.warn('Cannot delete %s %s by plugin %s: key does not exist.', type, npmName, key) 382 logger.warn('Cannot delete %s by plugin %s: key %s does not exist.', type, npmName, key)
381 return false 383 return false
382 } 384 }
383 385
@@ -388,7 +390,15 @@ export class RegisterHelpers {
388 } 390 }
389 } 391 }
390 392
391 this.updatedVideoConstants[type][npmName].deleted.push({ key, label: obj[key] }) 393 const updatedConstants = this.updatedVideoConstants[type][npmName]
394
395 const alreadyAdded = updatedConstants.added.find(a => a.key === key)
396 if (alreadyAdded) {
397 updatedConstants.added.filter(a => a.key !== key)
398 } else if (obj[key]) {
399 updatedConstants.deleted.push({ key, label: obj[key] })
400 }
401
392 delete obj[key] 402 delete obj[key]
393 403
394 return true 404 return true
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index da620b607..2a9241249 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -1,12 +1,12 @@
1import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
2import { sendUndoCacheFile } from './activitypub/send'
3import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
4import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models'
5import { CONFIG } from '@server/initializers/config'
6import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
7import { ActorFollowModel } from '@server/models/activitypub/actor-follow' 3import { CONFIG } from '@server/initializers/config'
8import { Activity } from '@shared/models' 4import { ActorFollowModel } from '@server/models/actor/actor-follow'
9import { getServerActor } from '@server/models/application/application' 5import { getServerActor } from '@server/models/application/application'
6import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models'
7import { Activity } from '@shared/models'
8import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
9import { sendUndoCacheFile } from './activitypub/send'
10 10
11async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { 11async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) {
12 const serverActor = await getServerActor() 12 const serverActor = await getServerActor()
diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts
index 598c0211f..1b80316e9 100644
--- a/server/lib/schedulers/actor-follow-scheduler.ts
+++ b/server/lib/schedulers/actor-follow-scheduler.ts
@@ -1,9 +1,9 @@
1import { isTestInstance } from '../../helpers/core-utils' 1import { isTestInstance } from '../../helpers/core-utils'
2import { logger } from '../../helpers/logger' 2import { logger } from '../../helpers/logger'
3import { ActorFollowModel } from '../../models/activitypub/actor-follow'
4import { AbstractScheduler } from './abstract-scheduler'
5import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 3import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { ActorFollowModel } from '../../models/actor/actor-follow'
6import { ActorFollowScoreCache } from '../files-cache' 5import { ActorFollowScoreCache } from '../files-cache'
6import { AbstractScheduler } from './abstract-scheduler'
7 7
8export class ActorFollowScheduler extends AbstractScheduler { 8export class ActorFollowScheduler extends AbstractScheduler {
9 9
diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts
index 0b8cd1389..aaa5feed5 100644
--- a/server/lib/schedulers/auto-follow-index-instances.ts
+++ b/server/lib/schedulers/auto-follow-index-instances.ts
@@ -1,7 +1,7 @@
1import { chunk } from 'lodash' 1import { chunk } from 'lodash'
2import { doJSONRequest } from '@server/helpers/requests' 2import { doJSONRequest } from '@server/helpers/requests'
3import { JobQueue } from '@server/lib/job-queue' 3import { JobQueue } from '@server/lib/job-queue'
4import { ActorFollowModel } from '@server/models/activitypub/actor-follow' 4import { ActorFollowModel } from '@server/models/actor/actor-follow'
5import { getServerActor } from '@server/models/application/application' 5import { getServerActor } from '@server/models/application/application'
6import { logger } from '../../helpers/logger' 6import { logger } from '../../helpers/logger'
7import { CONFIG } from '../../initializers/config' 7import { CONFIG } from '../../initializers/config'
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts
index 17a42b2c4..225669ea2 100644
--- a/server/lib/schedulers/remove-old-history-scheduler.ts
+++ b/server/lib/schedulers/remove-old-history-scheduler.ts
@@ -1,7 +1,7 @@
1import { logger } from '../../helpers/logger' 1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler' 2import { AbstractScheduler } from './abstract-scheduler'
3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { UserVideoHistoryModel } from '../../models/account/user-video-history' 4import { UserVideoHistoryModel } from '../../models/user/user-video-history'
5import { CONFIG } from '../../initializers/config' 5import { CONFIG } from '../../initializers/config'
6 6
7export class RemoveOldHistoryScheduler extends AbstractScheduler { 7export class RemoveOldHistoryScheduler extends AbstractScheduler {
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 3e75babcb..af69bda89 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -1,12 +1,12 @@
1import { VideoModel } from '@server/models/video/video'
2import { MVideoFullLight } from '@server/types/models'
1import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler' 4import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
5import { sequelizeTypescript } from '../../initializers/database'
3import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' 6import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
4import { retryTransactionWrapper } from '../../helpers/database-utils'
5import { federateVideoIfNeeded } from '../activitypub/videos' 7import { federateVideoIfNeeded } from '../activitypub/videos'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { Notifier } from '../notifier' 8import { Notifier } from '../notifier'
8import { sequelizeTypescript } from '../../initializers/database' 9import { AbstractScheduler } from './abstract-scheduler'
9import { MVideoFullLight } from '@server/types/models'
10 10
11export class UpdateVideosScheduler extends AbstractScheduler { 11export class UpdateVideosScheduler extends AbstractScheduler {
12 12
@@ -19,18 +19,19 @@ export class UpdateVideosScheduler extends AbstractScheduler {
19 } 19 }
20 20
21 protected async internalExecute () { 21 protected async internalExecute () {
22 return retryTransactionWrapper(this.updateVideos.bind(this)) 22 return this.updateVideos()
23 } 23 }
24 24
25 private async updateVideos () { 25 private async updateVideos () {
26 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined 26 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
27 27
28 const publishedVideos = await sequelizeTypescript.transaction(async t => { 28 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
29 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) 29 const publishedVideos: MVideoFullLight[] = []
30 const publishedVideos: MVideoFullLight[] = [] 30
31 for (const schedule of schedules) {
32 await sequelizeTypescript.transaction(async t => {
33 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(schedule.videoId, t)
31 34
32 for (const schedule of schedules) {
33 const video = schedule.Video
34 logger.info('Executing scheduled video update on %s.', video.uuid) 35 logger.info('Executing scheduled video update on %s.', video.uuid)
35 36
36 if (schedule.privacy) { 37 if (schedule.privacy) {
@@ -42,16 +43,13 @@ export class UpdateVideosScheduler extends AbstractScheduler {
42 await federateVideoIfNeeded(video, isNewVideo, t) 43 await federateVideoIfNeeded(video, isNewVideo, t)
43 44
44 if (wasConfidentialVideo) { 45 if (wasConfidentialVideo) {
45 const videoToPublish: MVideoFullLight = Object.assign(video, { ScheduleVideoUpdate: schedule, UserVideoHistories: [] }) 46 publishedVideos.push(video)
46 publishedVideos.push(videoToPublish)
47 } 47 }
48 } 48 }
49 49
50 await schedule.destroy({ transaction: t }) 50 await schedule.destroy({ transaction: t })
51 } 51 })
52 52 }
53 return publishedVideos
54 })
55 53
56 for (const v of publishedVideos) { 54 for (const v of publishedVideos) {
57 Notifier.Instance.notifyOnNewVideoIfNeeded(v) 55 Notifier.Instance.notifyOnNewVideoIfNeeded(v)
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 59b55cccc..b5a5eb697 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -23,7 +23,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../.
23import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 23import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
24import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 24import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
25import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' 25import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
26import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' 26import { getOrCreateAPVideo } from '../activitypub/videos'
27import { downloadPlaylistSegments } from '../hls' 27import { downloadPlaylistSegments } from '../hls'
28import { removeVideoRedundancy } from '../redundancy' 28import { removeVideoRedundancy } from '../redundancy'
29import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' 29import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths'
@@ -351,7 +351,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
351 syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, 351 syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true },
352 fetchType: 'all' as 'all' 352 fetchType: 'all' as 'all'
353 } 353 }
354 const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions) 354 const { video } = await getOrCreateAPVideo(getVideoOptions)
355 355
356 return video 356 return video
357 } 357 }
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
index aefe6aba4..898691c13 100644
--- a/server/lib/schedulers/youtube-dl-update-scheduler.ts
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -1,6 +1,6 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { YoutubeDL } from '@server/helpers/youtube-dl'
2import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 2import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
3import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' 3import { AbstractScheduler } from './abstract-scheduler'
4 4
5export class YoutubeDlUpdateScheduler extends AbstractScheduler { 5export class YoutubeDlUpdateScheduler extends AbstractScheduler {
6 6
@@ -13,7 +13,7 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler {
13 } 13 }
14 14
15 protected internalExecute () { 15 protected internalExecute () {
16 return updateYoutubeDLBinary() 16 return YoutubeDL.updateYoutubeDLBinary()
17 } 17 }
18 18
19 static get Instance () { 19 static get Instance () {
diff --git a/server/lib/search.ts b/server/lib/search.ts
new file mode 100644
index 000000000..b643a4055
--- /dev/null
+++ b/server/lib/search.ts
@@ -0,0 +1,50 @@
1import * as express from 'express'
2import { CONFIG } from '@server/initializers/config'
3import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
4import { getServerActor } from '@server/models/application/application'
5import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
6import { SearchTargetQuery } from '@shared/models'
7
8function isSearchIndexSearch (query: SearchTargetQuery) {
9 if (query.searchTarget === 'search-index') return true
10
11 const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
12
13 if (searchIndexConfig.ENABLED !== true) return false
14
15 if (searchIndexConfig.DISABLE_LOCAL_SEARCH) return true
16 if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true
17
18 return false
19}
20
21async function buildMutedForSearchIndex (res: express.Response) {
22 const serverActor = await getServerActor()
23 const accountIds = [ serverActor.Account.id ]
24
25 if (res.locals.oauth) {
26 accountIds.push(res.locals.oauth.token.User.Account.id)
27 }
28
29 const [ blockedHosts, blockedAccounts ] = await Promise.all([
30 ServerBlocklistModel.listHostsBlockedBy(accountIds),
31 AccountBlocklistModel.listHandlesBlockedBy(accountIds)
32 ])
33
34 return {
35 blockedHosts,
36 blockedAccounts
37 }
38}
39
40function isURISearch (search: string) {
41 if (!search) return false
42
43 return search.startsWith('http://') || search.startsWith('https://')
44}
45
46export {
47 isSearchIndexSearch,
48 buildMutedForSearchIndex,
49 isURISearch
50}
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
new file mode 100644
index 000000000..80d87a9d3
--- /dev/null
+++ b/server/lib/server-config-manager.ts
@@ -0,0 +1,304 @@
1import { getServerCommit } from '@server/helpers/utils'
2import { CONFIG, isEmailEnabled } from '@server/initializers/config'
3import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants'
4import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup'
5import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
6import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models'
7import { Hooks } from './plugins/hooks'
8import { PluginManager } from './plugins/plugin-manager'
9import { getThemeOrDefault } from './plugins/theme-utils'
10import { VideoTranscodingProfilesManager } from './transcoding/video-transcoding-profiles'
11
12/**
13 *
14 * Used to send the server config to clients (using REST/API or plugins API)
15 * We need a singleton class to manage config state depending on external events (to build menu entries etc)
16 *
17 */
18
19class ServerConfigManager {
20
21 private static instance: ServerConfigManager
22
23 private serverCommit: string
24
25 private homepageEnabled = false
26
27 private constructor () {}
28
29 async init () {
30 const instanceHomepage = await ActorCustomPageModel.loadInstanceHomepage()
31
32 this.updateHomepageState(instanceHomepage?.content)
33 }
34
35 updateHomepageState (content: string) {
36 this.homepageEnabled = !!content
37 }
38
39 async getHTMLServerConfig (): Promise<HTMLServerConfig> {
40 if (this.serverCommit === undefined) this.serverCommit = await getServerCommit()
41
42 const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME)
43
44 return {
45 instance: {
46 name: CONFIG.INSTANCE.NAME,
47 shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION,
48 isNSFW: CONFIG.INSTANCE.IS_NSFW,
49 defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
50 defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE,
51 customizations: {
52 javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT,
53 css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS
54 }
55 },
56 search: {
57 remoteUri: {
58 users: CONFIG.SEARCH.REMOTE_URI.USERS,
59 anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS
60 },
61 searchIndex: {
62 enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED,
63 url: CONFIG.SEARCH.SEARCH_INDEX.URL,
64 disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH,
65 isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH
66 }
67 },
68 plugin: {
69 registered: this.getRegisteredPlugins(),
70 registeredExternalAuths: this.getExternalAuthsPlugins(),
71 registeredIdAndPassAuths: this.getIdAndPassAuthPlugins()
72 },
73 theme: {
74 registered: this.getRegisteredThemes(),
75 default: defaultTheme
76 },
77 email: {
78 enabled: isEmailEnabled()
79 },
80 contactForm: {
81 enabled: CONFIG.CONTACT_FORM.ENABLED
82 },
83 serverVersion: PEERTUBE_VERSION,
84 serverCommit: this.serverCommit,
85 transcoding: {
86 hls: {
87 enabled: CONFIG.TRANSCODING.HLS.ENABLED
88 },
89 webtorrent: {
90 enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED
91 },
92 enabledResolutions: this.getEnabledResolutions('vod'),
93 profile: CONFIG.TRANSCODING.PROFILE,
94 availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod')
95 },
96 live: {
97 enabled: CONFIG.LIVE.ENABLED,
98
99 allowReplay: CONFIG.LIVE.ALLOW_REPLAY,
100 maxDuration: CONFIG.LIVE.MAX_DURATION,
101 maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES,
102 maxUserLives: CONFIG.LIVE.MAX_USER_LIVES,
103
104 transcoding: {
105 enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
106 enabledResolutions: this.getEnabledResolutions('live'),
107 profile: CONFIG.LIVE.TRANSCODING.PROFILE,
108 availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live')
109 },
110
111 rtmp: {
112 port: CONFIG.LIVE.RTMP.PORT
113 }
114 },
115 import: {
116 videos: {
117 http: {
118 enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
119 },
120 torrent: {
121 enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
122 }
123 }
124 },
125 autoBlacklist: {
126 videos: {
127 ofUsers: {
128 enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED
129 }
130 }
131 },
132 avatar: {
133 file: {
134 size: {
135 max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
136 },
137 extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
138 }
139 },
140 banner: {
141 file: {
142 size: {
143 max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
144 },
145 extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
146 }
147 },
148 video: {
149 image: {
150 extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME,
151 size: {
152 max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max
153 }
154 },
155 file: {
156 extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME
157 }
158 },
159 videoCaption: {
160 file: {
161 size: {
162 max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max
163 },
164 extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME
165 }
166 },
167 user: {
168 videoQuota: CONFIG.USER.VIDEO_QUOTA,
169 videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY
170 },
171 trending: {
172 videos: {
173 intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS,
174 algorithms: {
175 enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED,
176 default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT
177 }
178 }
179 },
180 tracker: {
181 enabled: CONFIG.TRACKER.ENABLED
182 },
183
184 followings: {
185 instance: {
186 autoFollowIndex: {
187 indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL
188 }
189 }
190 },
191
192 broadcastMessage: {
193 enabled: CONFIG.BROADCAST_MESSAGE.ENABLED,
194 message: CONFIG.BROADCAST_MESSAGE.MESSAGE,
195 level: CONFIG.BROADCAST_MESSAGE.LEVEL,
196 dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE
197 },
198
199 homepage: {
200 enabled: this.homepageEnabled
201 }
202 }
203 }
204
205 async getServerConfig (ip?: string): Promise<ServerConfig> {
206 const { allowed } = await Hooks.wrapPromiseFun(
207 isSignupAllowed,
208 {
209 ip
210 },
211 'filter:api.user.signup.allowed.result'
212 )
213
214 const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip)
215
216 const signup = {
217 allowed,
218 allowedForCurrentIP,
219 minimumAge: CONFIG.SIGNUP.MINIMUM_AGE,
220 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION
221 }
222
223 const htmlConfig = await this.getHTMLServerConfig()
224
225 return { ...htmlConfig, signup }
226 }
227
228 getRegisteredThemes () {
229 return PluginManager.Instance.getRegisteredThemes()
230 .map(t => ({
231 name: t.name,
232 version: t.version,
233 description: t.description,
234 css: t.css,
235 clientScripts: t.clientScripts
236 }))
237 }
238
239 getRegisteredPlugins () {
240 return PluginManager.Instance.getRegisteredPlugins()
241 .map(p => ({
242 name: p.name,
243 version: p.version,
244 description: p.description,
245 clientScripts: p.clientScripts
246 }))
247 }
248
249 getEnabledResolutions (type: 'vod' | 'live') {
250 const transcoding = type === 'vod'
251 ? CONFIG.TRANSCODING
252 : CONFIG.LIVE.TRANSCODING
253
254 return Object.keys(transcoding.RESOLUTIONS)
255 .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true)
256 .map(r => parseInt(r, 10))
257 }
258
259 private getIdAndPassAuthPlugins () {
260 const result: RegisteredIdAndPassAuthConfig[] = []
261
262 for (const p of PluginManager.Instance.getIdAndPassAuths()) {
263 for (const auth of p.idAndPassAuths) {
264 result.push({
265 npmName: p.npmName,
266 name: p.name,
267 version: p.version,
268 authName: auth.authName,
269 weight: auth.getWeight()
270 })
271 }
272 }
273
274 return result
275 }
276
277 private getExternalAuthsPlugins () {
278 const result: RegisteredExternalAuthConfig[] = []
279
280 for (const p of PluginManager.Instance.getExternalAuths()) {
281 for (const auth of p.externalAuths) {
282 result.push({
283 npmName: p.npmName,
284 name: p.name,
285 version: p.version,
286 authName: auth.authName,
287 authDisplayName: auth.authDisplayName()
288 })
289 }
290 }
291
292 return result
293 }
294
295 static get Instance () {
296 return this.instance || (this.instance = new this())
297 }
298}
299
300// ---------------------------------------------------------------------------
301
302export {
303 ServerConfigManager
304}
diff --git a/server/lib/signup.ts b/server/lib/signup.ts
new file mode 100644
index 000000000..8fa81e601
--- /dev/null
+++ b/server/lib/signup.ts
@@ -0,0 +1,62 @@
1import { UserModel } from '../models/user/user'
2import * as ipaddr from 'ipaddr.js'
3import { CONFIG } from '../initializers/config'
4
5const isCidr = require('is-cidr')
6
7async function isSignupAllowed (): Promise<{ allowed: boolean, errorMessage?: string }> {
8 if (CONFIG.SIGNUP.ENABLED === false) {
9 return { allowed: false }
10 }
11
12 // No limit and signup is enabled
13 if (CONFIG.SIGNUP.LIMIT === -1) {
14 return { allowed: true }
15 }
16
17 const totalUsers = await UserModel.countTotal()
18
19 return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT }
20}
21
22function isSignupAllowedForCurrentIP (ip: string) {
23 if (!ip) return false
24
25 const addr = ipaddr.parse(ip)
26 const excludeList = [ 'blacklist' ]
27 let matched = ''
28
29 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too
30 if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) {
31 excludeList.push('unknown')
32 }
33
34 if (addr.kind() === 'ipv4') {
35 const addrV4 = ipaddr.IPv4.parse(ip)
36 const rangeList = {
37 whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr))
38 .map(cidr => ipaddr.IPv4.parseCIDR(cidr)),
39 blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr))
40 .map(cidr => ipaddr.IPv4.parseCIDR(cidr))
41 }
42 matched = ipaddr.subnetMatch(addrV4, rangeList, 'unknown')
43 } else if (addr.kind() === 'ipv6') {
44 const addrV6 = ipaddr.IPv6.parse(ip)
45 const rangeList = {
46 whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr))
47 .map(cidr => ipaddr.IPv6.parseCIDR(cidr)),
48 blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr))
49 .map(cidr => ipaddr.IPv6.parseCIDR(cidr))
50 }
51 matched = ipaddr.subnetMatch(addrV6, rangeList, 'unknown')
52 }
53
54 return !excludeList.includes(matched)
55}
56
57// ---------------------------------------------------------------------------
58
59export {
60 isSignupAllowed,
61 isSignupAllowedForCurrentIP
62}
diff --git a/server/lib/stat-manager.ts b/server/lib/stat-manager.ts
index 5d703f610..3c5e0a93e 100644
--- a/server/lib/stat-manager.ts
+++ b/server/lib/stat-manager.ts
@@ -1,6 +1,6 @@
1import { CONFIG } from '@server/initializers/config' 1import { CONFIG } from '@server/initializers/config'
2import { UserModel } from '@server/models/account/user' 2import { UserModel } from '@server/models/user/user'
3import { ActorFollowModel } from '@server/models/activitypub/actor-follow' 3import { ActorFollowModel } from '@server/models/actor/actor-follow'
4import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' 4import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy'
5import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { VideoChannelModel } from '@server/models/video/video-channel' 6import { VideoChannelModel } from '@server/models/video/video-channel'
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index cfee69cfc..c08523988 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -14,7 +14,7 @@ import { getVideoFilePath } from './video-paths'
14 14
15type ImageSize = { height?: number, width?: number } 15type ImageSize = { height?: number, width?: number }
16 16
17function createPlaylistMiniatureFromExisting (options: { 17function updatePlaylistMiniatureFromExisting (options: {
18 inputPath: string 18 inputPath: string
19 playlist: MVideoPlaylistThumbnail 19 playlist: MVideoPlaylistThumbnail
20 automaticallyGenerated: boolean 20 automaticallyGenerated: boolean
@@ -26,7 +26,7 @@ function createPlaylistMiniatureFromExisting (options: {
26 const type = ThumbnailType.MINIATURE 26 const type = ThumbnailType.MINIATURE
27 27
28 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) 28 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal)
29 return createThumbnailFromFunction({ 29 return updateThumbnailFromFunction({
30 thumbnailCreator, 30 thumbnailCreator,
31 filename, 31 filename,
32 height, 32 height,
@@ -37,7 +37,7 @@ function createPlaylistMiniatureFromExisting (options: {
37 }) 37 })
38} 38}
39 39
40function createPlaylistMiniatureFromUrl (options: { 40function updatePlaylistMiniatureFromUrl (options: {
41 downloadUrl: string 41 downloadUrl: string
42 playlist: MVideoPlaylistThumbnail 42 playlist: MVideoPlaylistThumbnail
43 size?: ImageSize 43 size?: ImageSize
@@ -52,10 +52,10 @@ function createPlaylistMiniatureFromUrl (options: {
52 : downloadUrl 52 : downloadUrl
53 53
54 const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height }) 54 const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height })
55 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) 55 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
56} 56}
57 57
58function createVideoMiniatureFromUrl (options: { 58function updateVideoMiniatureFromUrl (options: {
59 downloadUrl: string 59 downloadUrl: string
60 video: MVideoThumbnail 60 video: MVideoThumbnail
61 type: ThumbnailType 61 type: ThumbnailType
@@ -82,10 +82,10 @@ function createVideoMiniatureFromUrl (options: {
82 return Promise.resolve() 82 return Promise.resolve()
83 } 83 }
84 84
85 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) 85 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
86} 86}
87 87
88function createVideoMiniatureFromExisting (options: { 88function updateVideoMiniatureFromExisting (options: {
89 inputPath: string 89 inputPath: string
90 video: MVideoThumbnail 90 video: MVideoThumbnail
91 type: ThumbnailType 91 type: ThumbnailType
@@ -98,7 +98,7 @@ function createVideoMiniatureFromExisting (options: {
98 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) 98 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
99 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) 99 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal)
100 100
101 return createThumbnailFromFunction({ 101 return updateThumbnailFromFunction({
102 thumbnailCreator, 102 thumbnailCreator,
103 filename, 103 filename,
104 height, 104 height,
@@ -123,7 +123,7 @@ function generateVideoMiniature (options: {
123 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) 123 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true)
124 : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) 124 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
125 125
126 return createThumbnailFromFunction({ 126 return updateThumbnailFromFunction({
127 thumbnailCreator, 127 thumbnailCreator,
128 filename, 128 filename,
129 height, 129 height,
@@ -134,7 +134,7 @@ function generateVideoMiniature (options: {
134 }) 134 })
135} 135}
136 136
137function createPlaceholderThumbnail (options: { 137function updatePlaceholderThumbnail (options: {
138 fileUrl: string 138 fileUrl: string
139 video: MVideoThumbnail 139 video: MVideoThumbnail
140 type: ThumbnailType 140 type: ThumbnailType
@@ -165,11 +165,11 @@ function createPlaceholderThumbnail (options: {
165 165
166export { 166export {
167 generateVideoMiniature, 167 generateVideoMiniature,
168 createVideoMiniatureFromUrl, 168 updateVideoMiniatureFromUrl,
169 createVideoMiniatureFromExisting, 169 updateVideoMiniatureFromExisting,
170 createPlaceholderThumbnail, 170 updatePlaceholderThumbnail,
171 createPlaylistMiniatureFromUrl, 171 updatePlaylistMiniatureFromUrl,
172 createPlaylistMiniatureFromExisting 172 updatePlaylistMiniatureFromExisting
173} 173}
174 174
175function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { 175function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) {
@@ -231,7 +231,7 @@ function buildMetadataFromVideo (video: MVideoThumbnail, type: ThumbnailType, si
231 return undefined 231 return undefined
232} 232}
233 233
234async function createThumbnailFromFunction (parameters: { 234async function updateThumbnailFromFunction (parameters: {
235 thumbnailCreator: () => Promise<any> 235 thumbnailCreator: () => Promise<any>
236 filename: string 236 filename: string
237 height: number 237 height: number
diff --git a/server/lib/video-transcoding-profiles.ts b/server/lib/transcoding/video-transcoding-profiles.ts
index 81f5e1962..c5ea72a5f 100644
--- a/server/lib/video-transcoding-profiles.ts
+++ b/server/lib/transcoding/video-transcoding-profiles.ts
@@ -1,6 +1,6 @@
1import { logger } from '@server/helpers/logger' 1import { logger } from '@server/helpers/logger'
2import { AvailableEncoders, EncoderOptionsBuilder, getTargetBitrate, VideoResolution } from '../../shared/models/videos' 2import { AvailableEncoders, EncoderOptionsBuilder, getTargetBitrate, VideoResolution } from '../../../shared/models/videos'
3import { buildStreamSuffix, resetSupportedEncoders } from '../helpers/ffmpeg-utils' 3import { buildStreamSuffix, resetSupportedEncoders } from '../../helpers/ffmpeg-utils'
4import { 4import {
5 canDoQuickAudioTranscode, 5 canDoQuickAudioTranscode,
6 ffprobePromise, 6 ffprobePromise,
@@ -8,8 +8,8 @@ import {
8 getMaxAudioBitrate, 8 getMaxAudioBitrate,
9 getVideoFileBitrate, 9 getVideoFileBitrate,
10 getVideoStreamFromFile 10 getVideoStreamFromFile
11} from '../helpers/ffprobe-utils' 11} from '../../helpers/ffprobe-utils'
12import { VIDEO_TRANSCODING_FPS } from '../initializers/constants' 12import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants'
13 13
14/** 14/**
15 * 15 *
diff --git a/server/lib/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index c949dca2e..1ad63baf3 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -1,19 +1,20 @@
1import { Job } from 'bull' 1import { Job } from 'bull'
2import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' 2import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
3import { basename, extname as extnameUtil, join } from 'path' 3import { basename, extname as extnameUtil, join } from 'path'
4import { toEven } from '@server/helpers/core-utils'
4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
5import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
6import { VideoResolution } from '../../shared/models/videos' 7import { VideoResolution } from '../../../shared/models/videos'
7import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' 8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
8import { transcode, TranscodeOptions, TranscodeOptionsType } from '../helpers/ffmpeg-utils' 9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils'
9import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../helpers/ffprobe-utils' 10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
10import { logger } from '../helpers/logger' 11import { logger } from '../../helpers/logger'
11import { CONFIG } from '../initializers/config' 12import { CONFIG } from '../../initializers/config'
12import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants' 13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants'
13import { VideoFileModel } from '../models/video/video-file' 14import { VideoFileModel } from '../../models/video/video-file'
14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
15import { updateMasterHLSPlaylist, updateSha256VODSegments } from './hls' 16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
16import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from './video-paths' 17import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from '../video-paths'
17import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' 18import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
18 19
19/** 20/**
@@ -35,6 +36,8 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile
35 ? 'quick-transcode' 36 ? 'quick-transcode'
36 : 'video' 37 : 'video'
37 38
39 const resolution = toEven(inputVideoFile.resolution)
40
38 const transcodeOptions: TranscodeOptions = { 41 const transcodeOptions: TranscodeOptions = {
39 type: transcodeType, 42 type: transcodeType,
40 43
@@ -44,7 +47,7 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile
44 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 47 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
45 profile: CONFIG.TRANSCODING.PROFILE, 48 profile: CONFIG.TRANSCODING.PROFILE,
46 49
47 resolution: inputVideoFile.resolution, 50 resolution,
48 51
49 job 52 job
50 } 53 }
@@ -57,7 +60,7 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile
57 60
58 // Important to do this before getVideoFilename() to take in account the new filename 61 // Important to do this before getVideoFilename() to take in account the new filename
59 inputVideoFile.extname = newExtname 62 inputVideoFile.extname = newExtname
60 inputVideoFile.filename = generateVideoFilename(video, false, inputVideoFile.resolution, newExtname) 63 inputVideoFile.filename = generateVideoFilename(video, false, resolution, newExtname)
61 64
62 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 65 const videoOutputPath = getVideoFilePath(video, inputVideoFile)
63 66
@@ -215,16 +218,6 @@ function generateHlsPlaylistResolution (options: {
215 }) 218 })
216} 219}
217 220
218function getEnabledResolutions (type: 'vod' | 'live') {
219 const transcoding = type === 'vod'
220 ? CONFIG.TRANSCODING
221 : CONFIG.LIVE.TRANSCODING
222
223 return Object.keys(transcoding.RESOLUTIONS)
224 .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true)
225 .map(r => parseInt(r, 10))
226}
227
228// --------------------------------------------------------------------------- 221// ---------------------------------------------------------------------------
229 222
230export { 223export {
@@ -232,8 +225,7 @@ export {
232 generateHlsPlaylistResolutionFromTS, 225 generateHlsPlaylistResolutionFromTS,
233 optimizeOriginalVideofile, 226 optimizeOriginalVideofile,
234 transcodeNewWebTorrentResolution, 227 transcodeNewWebTorrentResolution,
235 mergeAudioVideofile, 228 mergeAudioVideofile
236 getEnabledResolutions
237} 229}
238 230
239// --------------------------------------------------------------------------- 231// ---------------------------------------------------------------------------
diff --git a/server/lib/user.ts b/server/lib/user.ts
index 9b0a0a2f1..936403692 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -1,19 +1,21 @@
1import { Transaction } from 'sequelize/types' 1import { Transaction } from 'sequelize/types'
2import { v4 as uuidv4 } from 'uuid' 2import { buildUUID } from '@server/helpers/uuid'
3import { UserModel } from '@server/models/account/user' 3import { UserModel } from '@server/models/user/user'
4import { MActorDefault } from '@server/types/models/actor'
4import { ActivityPubActorType } from '../../shared/models/activitypub' 5import { ActivityPubActorType } from '../../shared/models/activitypub'
5import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' 6import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users'
6import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' 7import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants'
7import { sequelizeTypescript } from '../initializers/database' 8import { sequelizeTypescript } from '../initializers/database'
8import { AccountModel } from '../models/account/account' 9import { AccountModel } from '../models/account/account'
9import { UserNotificationSettingModel } from '../models/account/user-notification-setting' 10import { ActorModel } from '../models/actor/actor'
10import { ActorModel } from '../models/activitypub/actor' 11import { UserNotificationSettingModel } from '../models/user/user-notification-setting'
11import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models' 12import { MAccountDefault, MChannelActor } from '../types/models'
12import { MUser, MUserDefault, MUserId } from '../types/models/user' 13import { MUser, MUserDefault, MUserId } from '../types/models/user'
13import { buildActorInstance, generateAndSaveActorKeys } from './activitypub/actor' 14import { generateAndSaveActorKeys } from './activitypub/actors'
14import { getLocalAccountActivityPubUrl } from './activitypub/url' 15import { getLocalAccountActivityPubUrl } from './activitypub/url'
15import { Emailer } from './emailer' 16import { Emailer } from './emailer'
16import { LiveManager } from './live-manager' 17import { LiveQuotaStore } from './live/live-quota-store'
18import { buildActorInstance } from './local-actor'
17import { Redis } from './redis' 19import { Redis } from './redis'
18import { createLocalVideoChannel } from './video-channel' 20import { createLocalVideoChannel } from './video-channel'
19import { createWatchLaterPlaylist } from './video-playlist' 21import { createWatchLaterPlaylist } from './video-playlist'
@@ -42,11 +44,11 @@ async function createUserAccountAndChannelAndPlaylist (parameters: {
42 displayName: userDisplayName, 44 displayName: userDisplayName,
43 userId: userCreated.id, 45 userId: userCreated.id,
44 applicationId: null, 46 applicationId: null,
45 t: t 47 t
46 }) 48 })
47 userCreated.Account = accountCreated 49 userCreated.Account = accountCreated
48 50
49 const channelAttributes = await buildChannelAttributes(userCreated, channelNames) 51 const channelAttributes = await buildChannelAttributes(userCreated, t, channelNames)
50 const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) 52 const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t)
51 53
52 const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) 54 const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t)
@@ -127,7 +129,7 @@ async function getOriginalVideoFileTotalFromUser (user: MUserId) {
127 129
128 const base = await UserModel.getTotalRawQuery(query, user.id) 130 const base = await UserModel.getTotalRawQuery(query, user.id)
129 131
130 return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) 132 return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id)
131} 133}
132 134
133// Returns cumulative size of all video files uploaded in the last 24 hours. 135// Returns cumulative size of all video files uploaded in the last 24 hours.
@@ -141,10 +143,10 @@ async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) {
141 143
142 const base = await UserModel.getTotalRawQuery(query, user.id) 144 const base = await UserModel.getTotalRawQuery(query, user.id)
143 145
144 return base + LiveManager.Instance.getLiveQuotaUsedByUser(user.id) 146 return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id)
145} 147}
146 148
147async function isAbleToUploadVideo (userId: number, size: number) { 149async function isAbleToUploadVideo (userId: number, newVideoSize: number) {
148 const user = await UserModel.loadById(userId) 150 const user = await UserModel.loadById(userId)
149 151
150 if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) 152 if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true)
@@ -154,8 +156,8 @@ async function isAbleToUploadVideo (userId: number, size: number) {
154 getOriginalVideoFileTotalDailyFromUser(user) 156 getOriginalVideoFileTotalDailyFromUser(user)
155 ]) 157 ])
156 158
157 const uploadedTotal = size + totalBytes 159 const uploadedTotal = newVideoSize + totalBytes
158 const uploadedDaily = size + totalBytesDaily 160 const uploadedDaily = newVideoSize + totalBytesDaily
159 161
160 if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota 162 if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota
161 if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily 163 if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily
@@ -201,14 +203,14 @@ function createDefaultUserNotificationSettings (user: MUserId, t: Transaction |
201 return UserNotificationSettingModel.create(values, { transaction: t }) 203 return UserNotificationSettingModel.create(values, { transaction: t })
202} 204}
203 205
204async function buildChannelAttributes (user: MUser, channelNames?: ChannelNames) { 206async function buildChannelAttributes (user: MUser, transaction?: Transaction, channelNames?: ChannelNames) {
205 if (channelNames) return channelNames 207 if (channelNames) return channelNames
206 208
207 let channelName = user.username + '_channel' 209 let channelName = user.username + '_channel'
208 210
209 // Conflict, generate uuid instead 211 // Conflict, generate uuid instead
210 const actor = await ActorModel.loadLocalByName(channelName) 212 const actor = await ActorModel.loadLocalByName(channelName, transaction)
211 if (actor) channelName = uuidv4() 213 if (actor) channelName = buildUUID()
212 214
213 const videoChannelDisplayName = `Main ${user.username} channel` 215 const videoChannelDisplayName = `Main ${user.username} channel`
214 216
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts
index 37c43c3b0..0984c0d7a 100644
--- a/server/lib/video-blacklist.ts
+++ b/server/lib/video-blacklist.ts
@@ -16,7 +16,7 @@ import { CONFIG } from '../initializers/config'
16import { VideoBlacklistModel } from '../models/video/video-blacklist' 16import { VideoBlacklistModel } from '../models/video/video-blacklist'
17import { sendDeleteVideo } from './activitypub/send' 17import { sendDeleteVideo } from './activitypub/send'
18import { federateVideoIfNeeded } from './activitypub/videos' 18import { federateVideoIfNeeded } from './activitypub/videos'
19import { LiveManager } from './live-manager' 19import { LiveManager } from './live/live-manager'
20import { Notifier } from './notifier' 20import { Notifier } from './notifier'
21import { Hooks } from './plugins/hooks' 21import { Hooks } from './plugins/hooks'
22 22
diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts
index 0476cb2d5..2fd63a8c4 100644
--- a/server/lib/video-channel.ts
+++ b/server/lib/video-channel.ts
@@ -1,17 +1,15 @@
1import * as Sequelize from 'sequelize' 1import * as Sequelize from 'sequelize'
2import { v4 as uuidv4 } from 'uuid'
3import { VideoChannelCreate } from '../../shared/models' 2import { VideoChannelCreate } from '../../shared/models'
4import { VideoModel } from '../models/video/video' 3import { VideoModel } from '../models/video/video'
5import { VideoChannelModel } from '../models/video/video-channel' 4import { VideoChannelModel } from '../models/video/video-channel'
6import { MAccountId, MChannelId } from '../types/models' 5import { MAccountId, MChannelId } from '../types/models'
7import { buildActorInstance } from './activitypub/actor'
8import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' 6import { getLocalVideoChannelActivityPubUrl } from './activitypub/url'
9import { federateVideoIfNeeded } from './activitypub/videos' 7import { federateVideoIfNeeded } from './activitypub/videos'
8import { buildActorInstance } from './local-actor'
10 9
11async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { 10async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) {
12 const uuid = uuidv4()
13 const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) 11 const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name)
14 const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name, uuid) 12 const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name)
15 13
16 const actorInstanceCreated = await actorInstance.save({ transaction: t }) 14 const actorInstanceCreated = await actorInstance.save({ transaction: t })
17 15
diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts
index 736ebb2f8..c76570a5d 100644
--- a/server/lib/video-comment.ts
+++ b/server/lib/video-comment.ts
@@ -3,7 +3,7 @@ import * as Sequelize from 'sequelize'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { sequelizeTypescript } from '@server/initializers/database' 4import { sequelizeTypescript } from '@server/initializers/database'
5import { ResultList } from '../../shared/models' 5import { ResultList } from '../../shared/models'
6import { VideoCommentThreadTree } from '../../shared/models/videos/video-comment.model' 6import { VideoCommentThreadTree } from '../../shared/models/videos/comment/video-comment.model'
7import { VideoCommentModel } from '../models/video/video-comment' 7import { VideoCommentModel } from '../models/video/video-comment'
8import { MAccountDefault, MComment, MCommentOwnerVideo, MCommentOwnerVideoReply, MVideoFullLight } from '../types/models' 8import { MAccountDefault, MComment, MCommentOwnerVideo, MCommentOwnerVideoReply, MVideoFullLight } from '../types/models'
9import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send' 9import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send'
@@ -18,9 +18,9 @@ async function removeComment (videoCommentInstance: MCommentOwnerVideo) {
18 await sendDeleteVideoComment(videoCommentInstance, t) 18 await sendDeleteVideoComment(videoCommentInstance, t)
19 } 19 }
20 20
21 markCommentAsDeleted(videoCommentInstance) 21 videoCommentInstance.markAsDeleted()
22 22
23 await videoCommentInstance.save() 23 await videoCommentInstance.save({ transaction: t })
24 }) 24 })
25 25
26 logger.info('Video comment %d deleted.', videoCommentInstance.id) 26 logger.info('Video comment %d deleted.', videoCommentInstance.id)
@@ -95,17 +95,10 @@ function buildFormattedCommentTree (resultList: ResultList<VideoCommentModel>):
95 return thread 95 return thread
96} 96}
97 97
98function markCommentAsDeleted (comment: MComment): void {
99 comment.text = ''
100 comment.deletedAt = new Date()
101 comment.accountId = null
102}
103
104// --------------------------------------------------------------------------- 98// ---------------------------------------------------------------------------
105 99
106export { 100export {
107 removeComment, 101 removeComment,
108 createVideoComment, 102 createVideoComment,
109 buildFormattedCommentTree, 103 buildFormattedCommentTree
110 markCommentAsDeleted
111} 104}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 21e4b7ff2..daf998704 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -10,7 +10,7 @@ import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } fro
10import { federateVideoIfNeeded } from './activitypub/videos' 10import { federateVideoIfNeeded } from './activitypub/videos'
11import { JobQueue } from './job-queue/job-queue' 11import { JobQueue } from './job-queue/job-queue'
12import { Notifier } from './notifier' 12import { Notifier } from './notifier'
13import { createVideoMiniatureFromExisting } from './thumbnail' 13import { updateVideoMiniatureFromExisting } from './thumbnail'
14 14
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
16 return { 16 return {
@@ -28,6 +28,8 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil
28 privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, 28 privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
29 channelId: channelId, 29 channelId: channelId,
30 originallyPublishedAt: videoInfo.originallyPublishedAt 30 originallyPublishedAt: videoInfo.originallyPublishedAt
31 ? new Date(videoInfo.originallyPublishedAt)
32 : null
31 } 33 }
32} 34}
33 35
@@ -52,7 +54,7 @@ async function buildVideoThumbnailsFromReq (options: {
52 const fields = files?.[p.fieldName] 54 const fields = files?.[p.fieldName]
53 55
54 if (fields) { 56 if (fields) {
55 return createVideoMiniatureFromExisting({ 57 return updateVideoMiniatureFromExisting({
56 inputPath: fields[0].path, 58 inputPath: fields[0].path,
57 video, 59 video,
58 type: p.type, 60 type: p.type,