aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actors/get.ts37
-rw-r--r--server/lib/activitypub/actors/shared/object-to-model-attributes.ts4
-rw-r--r--server/lib/activitypub/actors/updater.ts10
-rw-r--r--server/lib/activitypub/cache-file.ts3
-rw-r--r--server/lib/activitypub/playlists/create-update.ts2
-rw-r--r--server/lib/activitypub/playlists/shared/object-to-model-attributes.ts2
-rw-r--r--server/lib/activitypub/process/process-create.ts4
-rw-r--r--server/lib/activitypub/process/process-dislike.ts3
-rw-r--r--server/lib/activitypub/process/process-flag.ts11
-rw-r--r--server/lib/auth/oauth-model.ts1
-rw-r--r--server/lib/auth/oauth.ts3
-rw-r--r--server/lib/blocklist.ts4
-rw-r--r--server/lib/client-html.ts12
-rw-r--r--server/lib/emailer.ts3
-rw-r--r--server/lib/hls.ts2
-rw-r--r--server/lib/job-queue/handlers/activitypub-cleaner.ts65
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts35
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts2
-rw-r--r--server/lib/job-queue/handlers/video-import.ts23
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts2
-rw-r--r--server/lib/live/shared/muxing-session.ts2
-rw-r--r--server/lib/local-actor.ts4
-rw-r--r--server/lib/moderation.ts29
-rw-r--r--server/lib/notifier/shared/comment/comment-mention.ts4
-rw-r--r--server/lib/paths.ts2
-rw-r--r--server/lib/plugins/plugin-helpers-builder.ts60
-rw-r--r--server/lib/plugins/plugin-manager.ts24
-rw-r--r--server/lib/redis.ts220
-rw-r--r--server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts35
-rw-r--r--server/lib/server-config-manager.ts25
-rw-r--r--server/lib/transcoding/video-transcoding-profiles.ts27
-rw-r--r--server/lib/transcoding/video-transcoding.ts1
-rw-r--r--server/lib/uploadx.ts14
-rw-r--r--server/lib/user.ts2
-rw-r--r--server/lib/video-path-manager.ts2
-rw-r--r--server/lib/video-state.ts11
-rw-r--r--server/lib/video.ts7
37 files changed, 381 insertions, 316 deletions
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts
index 8681ea02a..4200ddb4d 100644
--- a/server/lib/activitypub/actors/get.ts
+++ b/server/lib/activitypub/actors/get.ts
@@ -68,9 +68,28 @@ async function getOrCreateAPActor (
68 return actorRefreshed 68 return actorRefreshed
69} 69}
70 70
71function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) {
72 const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person')
73 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl)
74
75 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
76 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
77 }
78
79 try {
80 // Don't recurse another time
81 const recurseIfNeeded = false
82 return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded)
83 } catch (err) {
84 logger.error('Cannot get or create account attributed to video channel ' + actorUrl)
85 throw new Error(err)
86 }
87}
88
71// --------------------------------------------------------------------------- 89// ---------------------------------------------------------------------------
72 90
73export { 91export {
92 getOrCreateAPOwner,
74 getOrCreateAPActor 93 getOrCreateAPActor
75} 94}
76 95
@@ -88,24 +107,6 @@ async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType)
88 return actor 107 return actor
89} 108}
90 109
91function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) {
92 const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person')
93 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl)
94
95 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
96 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
97 }
98
99 try {
100 // Don't recurse another time
101 const recurseIfNeeded = false
102 return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded)
103 } catch (err) {
104 logger.error('Cannot get or create account attributed to video channel ' + actorUrl)
105 throw new Error(err)
106 }
107}
108
109async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { 110async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) {
110 if ((created === true || refreshed === true) && updateCollections === true) { 111 if ((created === true || refreshed === true) && updateCollections === true) {
111 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } 112 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
index 1612b3ad0..23bc972e5 100644
--- a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts
@@ -1,9 +1,9 @@
1import { getLowercaseExtension } from '@server/helpers/core-utils'
2import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' 1import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
3import { buildUUID } from '@server/helpers/uuid'
4import { MIMETYPES } from '@server/initializers/constants' 2import { MIMETYPES } from '@server/initializers/constants'
5import { ActorModel } from '@server/models/actor/actor' 3import { ActorModel } from '@server/models/actor/actor'
6import { FilteredModelAttributes } from '@server/types' 4import { FilteredModelAttributes } from '@server/types'
5import { getLowercaseExtension } from '@shared/core-utils'
6import { buildUUID } from '@shared/extra-utils'
7import { ActivityPubActor, ActorImageType } from '@shared/models' 7import { ActivityPubActor, ActorImageType } from '@shared/models'
8 8
9function getActorAttributesFromObject ( 9function getActorAttributesFromObject (
diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts
index de5e03eee..042438d9c 100644
--- a/server/lib/activitypub/actors/updater.ts
+++ b/server/lib/activitypub/actors/updater.ts
@@ -1,8 +1,10 @@
1import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' 1import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { AccountModel } from '@server/models/account/account'
3import { VideoChannelModel } from '@server/models/video/video-channel' 4import { VideoChannelModel } from '@server/models/video/video-channel'
4import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' 5import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models'
5import { ActivityPubActor, ActorImageType } from '@shared/models' 6import { ActivityPubActor, ActorImageType } from '@shared/models'
7import { getOrCreateAPOwner } from './get'
6import { updateActorImageInstance } from './image' 8import { updateActorImageInstance } from './image'
7import { fetchActorFollowsCount } from './shared' 9import { fetchActorFollowsCount } from './shared'
8import { getImageInfoFromObject } from './shared/object-to-model-attributes' 10import { getImageInfoFromObject } from './shared/object-to-model-attributes'
@@ -36,7 +38,13 @@ export class APActorUpdater {
36 this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername 38 this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername
37 this.accountOrChannel.description = this.actorObject.summary 39 this.accountOrChannel.description = this.actorObject.summary
38 40
39 if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support 41 if (this.accountOrChannel instanceof VideoChannelModel) {
42 const owner = await getOrCreateAPOwner(this.actorObject, this.actorObject.url)
43 this.accountOrChannel.accountId = owner.Account.id
44 this.accountOrChannel.Account = owner.Account as AccountModel
45
46 this.accountOrChannel.support = this.actorObject.support
47 }
40 48
41 await runInReadCommittedTransaction(async t => { 49 await runInReadCommittedTransaction(async t => {
42 await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t) 50 await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t)
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index a16d2cd93..c3acd7112 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,7 +1,6 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' 2import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models'
3import { CacheFileObject } from '../../../shared/index' 3import { CacheFileObject, VideoStreamingPlaylistType } from '@shared/models'
4import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 4import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6 5
7async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { 6async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts
index b152d709c..ef572c803 100644
--- a/server/lib/activitypub/playlists/create-update.ts
+++ b/server/lib/activitypub/playlists/create-update.ts
@@ -9,7 +9,7 @@ import { VideoPlaylistModel } from '@server/models/video/video-playlist'
9import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' 9import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
10import { FilteredModelAttributes } from '@server/types' 10import { FilteredModelAttributes } from '@server/types'
11import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' 11import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
12import { AttributesOnly } from '@shared/core-utils' 12import { AttributesOnly } from '@shared/typescript-utils'
13import { PlaylistObject } from '@shared/models' 13import { PlaylistObject } from '@shared/models'
14import { getOrCreateAPActor } from '../actors' 14import { getOrCreateAPActor } from '../actors'
15import { crawlCollectionPage } from '../crawl' 15import { crawlCollectionPage } from '../crawl'
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
index 70fd335bc..753b5e660 100644
--- a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
@@ -2,7 +2,7 @@ import { ACTIVITY_PUB } from '@server/initializers/constants'
2import { VideoPlaylistModel } from '@server/models/video/video-playlist' 2import { VideoPlaylistModel } from '@server/models/video/video-playlist'
3import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' 3import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
4import { MVideoId, MVideoPlaylistId } from '@server/types/models' 4import { MVideoId, MVideoPlaylistId } from '@server/types/models'
5import { AttributesOnly } from '@shared/core-utils' 5import { AttributesOnly } from '@shared/typescript-utils'
6import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' 6import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
7 7
8function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { 8function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 93f1d1c59..3e8ad184c 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -1,8 +1,6 @@
1import { isBlockedByServerOrAccount } from '@server/lib/blocklist' 1import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
2import { isRedundancyAccepted } from '@server/lib/redundancy' 2import { isRedundancyAccepted } from '@server/lib/redundancy'
3import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared' 3import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject } from '@shared/models'
4import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
5import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
6import { retryTransactionWrapper } from '../../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../../helpers/database-utils'
7import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
8import { sequelizeTypescript } from '../../../initializers/database' 6import { sequelizeTypescript } from '../../../initializers/database'
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts
index ecc57cd10..2f46b83d1 100644
--- a/server/lib/activitypub/process/process-dislike.ts
+++ b/server/lib/activitypub/process/process-dislike.ts
@@ -1,5 +1,4 @@
1import { ActivityCreate, ActivityDislike } from '../../../../shared' 1import { ActivityCreate, ActivityDislike, DislikeObject } from '@shared/models'
2import { DislikeObject } from '../../../../shared/models/activitypub/objects'
3import { retryTransactionWrapper } from '../../../helpers/database-utils' 2import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { sequelizeTypescript } from '../../../initializers/database' 3import { sequelizeTypescript } from '../../../initializers/database'
5import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 4import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts
index 7ed409d0e..a15d07a62 100644
--- a/server/lib/activitypub/process/process-flag.ts
+++ b/server/lib/activitypub/process/process-flag.ts
@@ -3,7 +3,7 @@ import { AccountModel } from '@server/models/account/account'
3import { VideoModel } from '@server/models/video/video' 3import { VideoModel } from '@server/models/video/video'
4import { VideoCommentModel } from '@server/models/video/video-comment' 4import { VideoCommentModel } from '@server/models/video/video-comment'
5import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' 5import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse'
6import { AbuseObject, AbuseState, ActivityCreate, ActivityFlag } from '../../../../shared' 6import { AbuseObject, AbuseState, ActivityCreate, ActivityFlag } from '@shared/models'
7import { getAPId } from '../../../helpers/activitypub' 7import { getAPId } from '../../../helpers/activitypub'
8import { retryTransactionWrapper } from '../../../helpers/database-utils' 8import { retryTransactionWrapper } from '../../../helpers/database-utils'
9import { logger } from '../../../helpers/logger' 9import { logger } from '../../../helpers/logger'
@@ -75,7 +75,8 @@ async function processCreateAbuse (activity: ActivityCreate | ActivityFlag, byAc
75 endAt, 75 endAt,
76 reporterAccount, 76 reporterAccount,
77 transaction: t, 77 transaction: t,
78 videoInstance: video 78 videoInstance: video,
79 skipNotification: false
79 }) 80 })
80 } 81 }
81 82
@@ -84,7 +85,8 @@ async function processCreateAbuse (activity: ActivityCreate | ActivityFlag, byAc
84 baseAbuse, 85 baseAbuse,
85 reporterAccount, 86 reporterAccount,
86 transaction: t, 87 transaction: t,
87 commentInstance: videoComment 88 commentInstance: videoComment,
89 skipNotification: false
88 }) 90 })
89 } 91 }
90 92
@@ -92,7 +94,8 @@ async function processCreateAbuse (activity: ActivityCreate | ActivityFlag, byAc
92 baseAbuse, 94 baseAbuse,
93 reporterAccount, 95 reporterAccount,
94 transaction: t, 96 transaction: t,
95 accountInstance: flaggedAccount 97 accountInstance: flaggedAccount,
98 skipNotification: false
96 }) 99 })
97 }) 100 })
98 } catch (err) { 101 } catch (err) {
diff --git a/server/lib/auth/oauth-model.ts b/server/lib/auth/oauth-model.ts
index f2ef0a78a..b68cce6d2 100644
--- a/server/lib/auth/oauth-model.ts
+++ b/server/lib/auth/oauth-model.ts
@@ -226,6 +226,7 @@ async function createUserFromExternal (pluginAuth: string, options: {
226 password: null, 226 password: null,
227 email: options.email, 227 email: options.email,
228 nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, 228 nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
229 p2pEnabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED,
229 autoPlayVideo: true, 230 autoPlayVideo: true,
230 role: options.role, 231 role: options.role,
231 videoQuota: CONFIG.USER.VIDEO_QUOTA, 232 videoQuota: CONFIG.USER.VIDEO_QUOTA,
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts
index 497773536..2bf7a6361 100644
--- a/server/lib/auth/oauth.ts
+++ b/server/lib/auth/oauth.ts
@@ -8,8 +8,9 @@ import {
8 UnauthorizedClientError, 8 UnauthorizedClientError,
9 UnsupportedGrantTypeError 9 UnsupportedGrantTypeError
10} from 'oauth2-server' 10} from 'oauth2-server'
11import { randomBytesPromise, sha1 } from '@server/helpers/core-utils' 11import { randomBytesPromise } from '@server/helpers/core-utils'
12import { MOAuthClient } from '@server/types/models' 12import { MOAuthClient } from '@server/types/models'
13import { sha1 } from '@shared/extra-utils'
13import { OAUTH_LIFETIME } from '../../initializers/constants' 14import { OAUTH_LIFETIME } from '../../initializers/constants'
14import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' 15import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model'
15 16
diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts
index d6b684015..98273a6ea 100644
--- a/server/lib/blocklist.ts
+++ b/server/lib/blocklist.ts
@@ -40,12 +40,12 @@ async function isBlockedByServerOrAccount (targetAccount: MAccountServer, userAc
40 40
41 if (userAccount) sourceAccounts.push(userAccount.id) 41 if (userAccount) sourceAccounts.push(userAccount.id)
42 42
43 const accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(sourceAccounts, targetAccount.id) 43 const accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, targetAccount.id)
44 if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) { 44 if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) {
45 return true 45 return true
46 } 46 }
47 47
48 const instanceMutedHash = await ServerBlocklistModel.isServerMutedByMulti(sourceAccounts, targetAccount.Actor.serverId) 48 const instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, targetAccount.Actor.serverId)
49 if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) { 49 if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) {
50 return true 50 return true
51 } 51 }
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index 360b4667f..74788af52 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -3,12 +3,14 @@ import { readFile } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import validator from 'validator' 4import validator from 'validator'
5import { toCompleteUUID } from '@server/helpers/custom-validators/misc' 5import { toCompleteUUID } from '@server/helpers/custom-validators/misc'
6import { root } from '@shared/core-utils'
6import { escapeHTML } from '@shared/core-utils/renderer' 7import { escapeHTML } from '@shared/core-utils/renderer'
8import { sha256 } from '@shared/extra-utils'
7import { HTMLServerConfig } from '@shared/models' 9import { HTMLServerConfig } from '@shared/models'
8import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' 10import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n'
9import { HttpStatusCode } from '../../shared/models/http/http-error-codes' 11import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
10import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' 12import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos'
11import { isTestInstance, sha256 } from '../helpers/core-utils' 13import { isTestInstance } from '../helpers/core-utils'
12import { logger } from '../helpers/logger' 14import { logger } from '../helpers/logger'
13import { mdToPlainText } from '../helpers/markdown' 15import { mdToPlainText } from '../helpers/markdown'
14import { CONFIG } from '../initializers/config' 16import { CONFIG } from '../initializers/config'
@@ -343,15 +345,11 @@ class ClientHtml {
343 { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] } 345 { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] }
344 ) 346 )
345 347
346 return join(__dirname, '../../../client/dist/' + buildFileLocale(lang) + '/index.html') 348 return join(root(), 'client', 'dist', buildFileLocale(lang), 'index.html')
347 } 349 }
348 350
349 private static getEmbedPath () { 351 private static getEmbedPath () {
350 return join(__dirname, '../../../client/dist/standalone/videos/embed.html') 352 return join(root(), 'client', 'dist', 'standalone', 'videos', 'embed.html')
351 }
352
353 private static addHtmlLang (htmlStringPage: string, paramLang: string) {
354 return htmlStringPage.replace('<html>', `<html lang="${paramLang}">`)
355 } 353 }
356 354
357 private static addManifestContentHash (htmlStringPage: string) { 355 private static addManifestContentHash (htmlStringPage: string) {
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts
index 60284ea28..ebad43650 100644
--- a/server/lib/emailer.ts
+++ b/server/lib/emailer.ts
@@ -4,7 +4,8 @@ import { createTransport, Transporter } from 'nodemailer'
4import { join } from 'path' 4import { join } from 'path'
5import { EmailPayload } from '@shared/models' 5import { EmailPayload } from '@shared/models'
6import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model' 6import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model'
7import { isTestInstance, root } from '../helpers/core-utils' 7import { isTestInstance } from '../helpers/core-utils'
8import { root } from '@shared/core-utils'
8import { bunyanLogger, logger } from '../helpers/logger' 9import { bunyanLogger, logger } from '../helpers/logger'
9import { CONFIG, isEmailEnabled } from '../initializers/config' 10import { CONFIG, isEmailEnabled } from '../initializers/config'
10import { WEBSERVER } from '../initializers/constants' 11import { WEBSERVER } from '../initializers/constants'
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index f2fe893a9..1574ff27b 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -2,7 +2,7 @@ import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat,
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
5import { sha256 } from '../helpers/core-utils' 5import { sha256 } from '@shared/extra-utils'
6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' 6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
7import { logger } from '../helpers/logger' 7import { logger } from '../helpers/logger'
8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' 8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts
index d5e4508fe..509dd1cb5 100644
--- a/server/lib/job-queue/handlers/activitypub-cleaner.ts
+++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts
@@ -8,36 +8,35 @@ import {
8} from '@server/helpers/custom-validators/activitypub/activity' 8} from '@server/helpers/custom-validators/activitypub/activity'
9import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' 9import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
10import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' 10import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
11import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants' 11import { AP_CLEANER } from '@server/initializers/constants'
12import { Redis } from '@server/lib/redis'
12import { VideoModel } from '@server/models/video/video' 13import { VideoModel } from '@server/models/video/video'
13import { VideoCommentModel } from '@server/models/video/video-comment' 14import { VideoCommentModel } from '@server/models/video/video-comment'
14import { VideoShareModel } from '@server/models/video/video-share' 15import { VideoShareModel } from '@server/models/video/video-share'
15import { HttpStatusCode } from '@shared/models' 16import { HttpStatusCode } from '@shared/models'
16import { logger } from '../../../helpers/logger' 17import { logger, loggerTagsFactory } from '../../../helpers/logger'
17import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 18import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
18 19
20const lTags = loggerTagsFactory('ap-cleaner')
21
19// Job to clean remote interactions off local videos 22// Job to clean remote interactions off local videos
20 23
21async function processActivityPubCleaner (_job: Job) { 24async function processActivityPubCleaner (_job: Job) {
22 logger.info('Processing ActivityPub cleaner.') 25 logger.info('Processing ActivityPub cleaner.', lTags())
23 26
24 { 27 {
25 const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() 28 const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
26 const { bodyValidator, deleter, updater } = rateOptionsFactory() 29 const { bodyValidator, deleter, updater } = rateOptionsFactory()
27 30
28 await map(rateUrls, async rateUrl => { 31 await map(rateUrls, async rateUrl => {
29 try { 32 const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
30 const result = await updateObjectIfNeeded(rateUrl, bodyValidator, updater, deleter)
31 33
32 if (result?.status === 'deleted') { 34 if (result?.status === 'deleted') {
33 const { videoId, type } = result.data 35 const { videoId, type } = result.data
34 36
35 await VideoModel.updateRatesOf(videoId, type, undefined) 37 await VideoModel.updateRatesOf(videoId, type, undefined)
36 }
37 } catch (err) {
38 logger.warn('Cannot update/delete remote AP rate %s.', rateUrl, { err })
39 } 38 }
40 }, { concurrency: AP_CLEANER_CONCURRENCY }) 39 }, { concurrency: AP_CLEANER.CONCURRENCY })
41 } 40 }
42 41
43 { 42 {
@@ -45,12 +44,8 @@ async function processActivityPubCleaner (_job: Job) {
45 const { bodyValidator, deleter, updater } = shareOptionsFactory() 44 const { bodyValidator, deleter, updater } = shareOptionsFactory()
46 45
47 await map(shareUrls, async shareUrl => { 46 await map(shareUrls, async shareUrl => {
48 try { 47 await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
49 await updateObjectIfNeeded(shareUrl, bodyValidator, updater, deleter) 48 }, { concurrency: AP_CLEANER.CONCURRENCY })
50 } catch (err) {
51 logger.warn('Cannot update/delete remote AP share %s.', shareUrl, { err })
52 }
53 }, { concurrency: AP_CLEANER_CONCURRENCY })
54 } 49 }
55 50
56 { 51 {
@@ -58,12 +53,8 @@ async function processActivityPubCleaner (_job: Job) {
58 const { bodyValidator, deleter, updater } = commentOptionsFactory() 53 const { bodyValidator, deleter, updater } = commentOptionsFactory()
59 54
60 await map(commentUrls, async commentUrl => { 55 await map(commentUrls, async commentUrl => {
61 try { 56 await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
62 await updateObjectIfNeeded(commentUrl, bodyValidator, updater, deleter) 57 }, { concurrency: AP_CLEANER.CONCURRENCY })
63 } catch (err) {
64 logger.warn('Cannot update/delete remote AP comment %s.', commentUrl, { err })
65 }
66 }, { concurrency: AP_CLEANER_CONCURRENCY })
67 } 58 }
68} 59}
69 60
@@ -75,14 +66,16 @@ export {
75 66
76// --------------------------------------------------------------------------- 67// ---------------------------------------------------------------------------
77 68
78async function updateObjectIfNeeded <T> ( 69async function updateObjectIfNeeded <T> (options: {
79 url: string, 70 url: string
80 bodyValidator: (body: any) => boolean, 71 bodyValidator: (body: any) => boolean
81 updater: (url: string, newUrl: string) => Promise<T>, 72 updater: (url: string, newUrl: string) => Promise<T>
82 deleter: (url: string) => Promise<T> 73 deleter: (url: string) => Promise<T> }
83): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { 74): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
75 const { url, bodyValidator, updater, deleter } = options
76
84 const on404OrTombstone = async () => { 77 const on404OrTombstone = async () => {
85 logger.info('Removing remote AP object %s.', url) 78 logger.info('Removing remote AP object %s.', url, lTags(url))
86 const data = await deleter(url) 79 const data = await deleter(url)
87 80
88 return { status: 'deleted' as 'deleted', data } 81 return { status: 'deleted' as 'deleted', data }
@@ -104,7 +97,7 @@ async function updateObjectIfNeeded <T> (
104 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) 97 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
105 } 98 }
106 99
107 logger.info('Updating remote AP object %s.', url) 100 logger.info('Updating remote AP object %s.', url, lTags(url))
108 const data = await updater(url, newUrl) 101 const data = await updater(url, newUrl)
109 102
110 return { status: 'updated', data } 103 return { status: 'updated', data }
@@ -117,7 +110,15 @@ async function updateObjectIfNeeded <T> (
117 return on404OrTombstone() 110 return on404OrTombstone()
118 } 111 }
119 112
120 throw err 113 logger.debug('Remote AP object %s is unavailable.', url, lTags(url))
114
115 const unavailability = await Redis.Instance.addAPUnavailability(url)
116 if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) {
117 logger.info('Removing unavailable AP resource %s.', url, lTags(url))
118 return on404OrTombstone()
119 }
120
121 return null
121 } 122 }
122} 123}
123 124
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 54a7c566b..9e39322a8 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -2,16 +2,16 @@ import { Job } from 'bull'
2import { remove } from 'fs-extra' 2import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { logger } from '@server/helpers/logger' 4import { logger } from '@server/helpers/logger'
5import { updateTorrentUrls } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' 8import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
10import { moveToNextState } from '@server/lib/video-state' 10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
12import { VideoJobInfoModel } from '@server/models/video/video-job-info' 12import { VideoJobInfoModel } from '@server/models/video/video-job-info'
13import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' 13import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
14import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared' 14import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
15 15
16export async function processMoveToObjectStorage (job: Job) { 16export async function processMoveToObjectStorage (job: Job) {
17 const payload = job.data as MoveObjectStoragePayload 17 const payload = job.data as MoveObjectStoragePayload
@@ -24,18 +24,25 @@ export async function processMoveToObjectStorage (job: Job) {
24 return undefined 24 return undefined
25 } 25 }
26 26
27 if (video.VideoFiles) { 27 try {
28 await moveWebTorrentFiles(video) 28 if (video.VideoFiles) {
29 } 29 await moveWebTorrentFiles(video)
30 }
30 31
31 if (video.VideoStreamingPlaylists) { 32 if (video.VideoStreamingPlaylists) {
32 await moveHLSFiles(video) 33 await moveHLSFiles(video)
33 } 34 }
35
36 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
37 if (pendingMove === 0) {
38 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
39 await doAfterLastJob(video, payload.isNewVideo)
40 }
41 } catch (err) {
42 logger.error('Cannot move video %s to object storage.', video.url, { err })
34 43
35 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') 44 await moveToFailedMoveToObjectStorageState(video)
36 if (pendingMove === 0) { 45 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
37 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
38 await doAfterLastJob(video, payload.isNewVideo)
39 } 46 }
40 47
41 return payload.videoUUID 48 return payload.videoUUID
@@ -113,7 +120,7 @@ async function onFileMoved (options: {
113 file.fileUrl = fileUrl 120 file.fileUrl = fileUrl
114 file.storage = VideoStorage.OBJECT_STORAGE 121 file.storage = VideoStorage.OBJECT_STORAGE
115 122
116 await updateTorrentUrls(videoOrPlaylist, file) 123 await updateTorrentMetadata(videoOrPlaylist, file)
117 await file.save() 124 await file.save()
118 125
119 logger.debug('Removing %s because it\'s now on object storage', oldPath) 126 logger.debug('Removing %s because it\'s now on object storage', oldPath)
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index a91c2ef80..0d9e80cb8 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -1,6 +1,6 @@
1import { Job } from 'bull' 1import { Job } from 'bull'
2import { copy, stat } from 'fs-extra' 2import { copy, stat } from 'fs-extra'
3import { getLowercaseExtension } from '@server/helpers/core-utils' 3import { getLowercaseExtension } from '@shared/core-utils'
4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
5import { CONFIG } from '@server/initializers/config' 5import { CONFIG } from '@server/initializers/config'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 4ce1a6c30..2f74e9fbd 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -1,6 +1,5 @@
1import { Job } from 'bull' 1import { Job } from 'bull'
2import { move, remove, stat } from 'fs-extra' 2import { move, remove, stat } from 'fs-extra'
3import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { retryTransactionWrapper } from '@server/helpers/database-utils' 3import { retryTransactionWrapper } from '@server/helpers/database-utils'
5import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' 4import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
6import { isPostImportVideoAccepted } from '@server/lib/moderation' 5import { isPostImportVideoAccepted } from '@server/lib/moderation'
@@ -13,17 +12,20 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state' 12import { buildNextVideoState } from '@server/lib/video-state'
14import { ThumbnailModel } from '@server/models/video/thumbnail' 13import { ThumbnailModel } from '@server/models/video/thumbnail'
15import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' 14import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
15import { getLowercaseExtension } from '@shared/core-utils'
16import { isAudioFile } from '@shared/extra-utils'
16import { 17import {
18 ThumbnailType,
17 VideoImportPayload, 19 VideoImportPayload,
20 VideoImportState,
18 VideoImportTorrentPayload, 21 VideoImportTorrentPayload,
19 VideoImportTorrentPayloadType, 22 VideoImportTorrentPayloadType,
20 VideoImportYoutubeDLPayload, 23 VideoImportYoutubeDLPayload,
21 VideoImportYoutubeDLPayloadType, 24 VideoImportYoutubeDLPayloadType,
25 VideoResolution,
22 VideoState 26 VideoState
23} from '../../../../shared' 27} from '@shared/models'
24import { VideoImportState } from '../../../../shared/models/videos' 28import { ffprobePromise, getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
25import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
26import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
27import { logger } from '../../../helpers/logger' 29import { logger } from '../../../helpers/logger'
28import { getSecureTorrentName } from '../../../helpers/utils' 30import { getSecureTorrentName } from '../../../helpers/utils'
29import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 31import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
@@ -114,9 +116,14 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
114 throw new Error('The user video quota is exceeded with this video to import.') 116 throw new Error('The user video quota is exceeded with this video to import.')
115 } 117 }
116 118
117 const { resolution } = await getVideoFileResolution(tempVideoPath) 119 const probe = await ffprobePromise(tempVideoPath)
118 const fps = await getVideoFileFPS(tempVideoPath) 120
119 const duration = await getDurationFromVideoFile(tempVideoPath) 121 const { resolution } = await isAudioFile(tempVideoPath, probe)
122 ? { resolution: VideoResolution.H_NOVIDEO }
123 : await getVideoFileResolution(tempVideoPath)
124
125 const fps = await getVideoFileFPS(tempVideoPath, probe)
126 const duration = await getDurationFromVideoFile(tempVideoPath, probe)
120 127
121 // Prepare video file object for creation in database 128 // Prepare video file object for creation in database
122 const fileExt = getLowercaseExtension(tempVideoPath) 129 const fileExt = getLowercaseExtension(tempVideoPath)
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 0edcdcba3..ef3abcbcd 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -12,7 +12,7 @@ import {
12 NewResolutionTranscodingPayload, 12 NewResolutionTranscodingPayload,
13 OptimizeTranscodingPayload, 13 OptimizeTranscodingPayload,
14 VideoTranscodingPayload 14 VideoTranscodingPayload
15} from '../../../../shared' 15} from '@shared/models'
16import { retryTransactionWrapper } from '../../../helpers/database-utils' 16import { retryTransactionWrapper } from '../../../helpers/database-utils'
17import { computeLowerResolutionsToTranscode } from '../../../helpers/ffprobe-utils' 17import { computeLowerResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
18import { logger, loggerTagsFactory } from '../../../helpers/logger' 18import { logger, loggerTagsFactory } from '../../../helpers/logger'
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index eccaefcfa..22a47942a 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -229,7 +229,7 @@ class MuxingSession extends EventEmitter {
229 229
230 const playlistIdMatcher = /^([\d+])-/ 230 const playlistIdMatcher = /^([\d+])-/
231 231
232 const addHandler = async segmentPath => { 232 const addHandler = async (segmentPath: string) => {
233 logger.debug('Live add handler of %s.', segmentPath, this.lTags()) 233 logger.debug('Live add handler of %s.', segmentPath, this.lTags())
234 234
235 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] 235 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts
index 821a92b91..c6826759b 100644
--- a/server/lib/local-actor.ts
+++ b/server/lib/local-actor.ts
@@ -2,9 +2,9 @@ import 'multer'
2import { queue } from 'async' 2import { queue } from 'async'
3import LRUCache from 'lru-cache' 3import LRUCache from 'lru-cache'
4import { join } from 'path' 4import { join } from 'path'
5import { getLowercaseExtension } from '@server/helpers/core-utils'
6import { buildUUID } from '@server/helpers/uuid'
7import { ActorModel } from '@server/models/actor/actor' 5import { ActorModel } from '@server/models/actor/actor'
6import { getLowercaseExtension } from '@shared/core-utils'
7import { buildUUID } from '@shared/extra-utils'
8import { ActivityPubActorType, ActorImageType } from '@shared/models' 8import { ActivityPubActorType, ActorImageType } from '@shared/models'
9import { retryTransactionWrapper } from '../helpers/database-utils' 9import { retryTransactionWrapper } from '../helpers/database-utils'
10import { processImage } from '../helpers/image-utils' 10import { processImage } from '../helpers/image-utils'
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index 456b615b2..c2565f867 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -107,8 +107,9 @@ async function createVideoAbuse (options: {
107 endAt: number 107 endAt: number
108 transaction: Transaction 108 transaction: Transaction
109 reporterAccount: MAccountDefault 109 reporterAccount: MAccountDefault
110 skipNotification: boolean
110}) { 111}) {
111 const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount } = options 112 const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount, skipNotification } = options
112 113
113 const associateFun = async (abuseInstance: MAbuseFull) => { 114 const associateFun = async (abuseInstance: MAbuseFull) => {
114 const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({ 115 const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({
@@ -129,6 +130,7 @@ async function createVideoAbuse (options: {
129 reporterAccount, 130 reporterAccount,
130 flaggedAccount: videoInstance.VideoChannel.Account, 131 flaggedAccount: videoInstance.VideoChannel.Account,
131 transaction, 132 transaction,
133 skipNotification,
132 associateFun 134 associateFun
133 }) 135 })
134} 136}
@@ -138,8 +140,9 @@ function createVideoCommentAbuse (options: {
138 commentInstance: MCommentOwnerVideo 140 commentInstance: MCommentOwnerVideo
139 transaction: Transaction 141 transaction: Transaction
140 reporterAccount: MAccountDefault 142 reporterAccount: MAccountDefault
143 skipNotification: boolean
141}) { 144}) {
142 const { baseAbuse, commentInstance, transaction, reporterAccount } = options 145 const { baseAbuse, commentInstance, transaction, reporterAccount, skipNotification } = options
143 146
144 const associateFun = async (abuseInstance: MAbuseFull) => { 147 const associateFun = async (abuseInstance: MAbuseFull) => {
145 const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({ 148 const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({
@@ -158,6 +161,7 @@ function createVideoCommentAbuse (options: {
158 reporterAccount, 161 reporterAccount,
159 flaggedAccount: commentInstance.Account, 162 flaggedAccount: commentInstance.Account,
160 transaction, 163 transaction,
164 skipNotification,
161 associateFun 165 associateFun
162 }) 166 })
163} 167}
@@ -167,8 +171,9 @@ function createAccountAbuse (options: {
167 accountInstance: MAccountDefault 171 accountInstance: MAccountDefault
168 transaction: Transaction 172 transaction: Transaction
169 reporterAccount: MAccountDefault 173 reporterAccount: MAccountDefault
174 skipNotification: boolean
170}) { 175}) {
171 const { baseAbuse, accountInstance, transaction, reporterAccount } = options 176 const { baseAbuse, accountInstance, transaction, reporterAccount, skipNotification } = options
172 177
173 const associateFun = () => { 178 const associateFun = () => {
174 return Promise.resolve({ isOwned: accountInstance.isOwned() }) 179 return Promise.resolve({ isOwned: accountInstance.isOwned() })
@@ -179,6 +184,7 @@ function createAccountAbuse (options: {
179 reporterAccount, 184 reporterAccount,
180 flaggedAccount: accountInstance, 185 flaggedAccount: accountInstance,
181 transaction, 186 transaction,
187 skipNotification,
182 associateFun 188 associateFun
183 }) 189 })
184} 190}
@@ -207,9 +213,10 @@ async function createAbuse (options: {
207 reporterAccount: MAccountDefault 213 reporterAccount: MAccountDefault
208 flaggedAccount: MAccountLight 214 flaggedAccount: MAccountLight
209 associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean} > 215 associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean} >
216 skipNotification: boolean
210 transaction: Transaction 217 transaction: Transaction
211}) { 218}) {
212 const { base, reporterAccount, flaggedAccount, associateFun, transaction } = options 219 const { base, reporterAccount, flaggedAccount, associateFun, transaction, skipNotification } = options
213 const auditLogger = auditLoggerFactory('abuse') 220 const auditLogger = auditLoggerFactory('abuse')
214 221
215 const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id }) 222 const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id })
@@ -227,13 +234,15 @@ async function createAbuse (options: {
227 const abuseJSON = abuseInstance.toFormattedAdminJSON() 234 const abuseJSON = abuseInstance.toFormattedAdminJSON()
228 auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON)) 235 auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON))
229 236
230 afterCommitIfTransaction(transaction, () => { 237 if (!skipNotification) {
231 Notifier.Instance.notifyOnNewAbuse({ 238 afterCommitIfTransaction(transaction, () => {
232 abuse: abuseJSON, 239 Notifier.Instance.notifyOnNewAbuse({
233 abuseInstance, 240 abuse: abuseJSON,
234 reporter: reporterAccount.Actor.getIdentifier() 241 abuseInstance,
242 reporter: reporterAccount.Actor.getIdentifier()
243 })
235 }) 244 })
236 }) 245 }
237 246
238 logger.info('Abuse report %d created.', abuseInstance.id) 247 logger.info('Abuse report %d created.', abuseInstance.id)
239 248
diff --git a/server/lib/notifier/shared/comment/comment-mention.ts b/server/lib/notifier/shared/comment/comment-mention.ts
index 4f84d8dea..765cbaad9 100644
--- a/server/lib/notifier/shared/comment/comment-mention.ts
+++ b/server/lib/notifier/shared/comment/comment-mention.ts
@@ -47,8 +47,8 @@ export class CommentMention extends AbstractNotification <MCommentOwnerVideo, MU
47 47
48 const sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ]) 48 const sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ])
49 49
50 this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(sourceAccounts, this.payload.accountId) 50 this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, this.payload.accountId)
51 this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByMulti(sourceAccounts, this.payload.Account.Actor.serverId) 51 this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, this.payload.Account.Actor.serverId)
52 } 52 }
53 53
54 log () { 54 log () {
diff --git a/server/lib/paths.ts b/server/lib/paths.ts
index 434e637c6..5a85bea42 100644
--- a/server/lib/paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,9 @@
1import { join } from 'path' 1import { join } from 'path'
2import { buildUUID } from '@server/helpers/uuid'
3import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' 3import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' 4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
6import { removeFragmentedMP4Ext } from '@shared/core-utils' 5import { removeFragmentedMP4Ext } from '@shared/core-utils'
6import { buildUUID } from '@shared/extra-utils'
7 7
8// ################## Video file name ################## 8// ################## Video file name ##################
9 9
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts
index e26776f45..78e4a28ad 100644
--- a/server/lib/plugins/plugin-helpers-builder.ts
+++ b/server/lib/plugins/plugin-helpers-builder.ts
@@ -1,5 +1,6 @@
1import express from 'express' 1import express from 'express'
2import { join } from 'path' 2import { join } from 'path'
3import { ffprobePromise } from '@server/helpers/ffprobe-utils'
3import { buildLogger } from '@server/helpers/logger' 4import { buildLogger } from '@server/helpers/logger'
4import { CONFIG } from '@server/initializers/config' 5import { CONFIG } from '@server/initializers/config'
5import { WEBSERVER } from '@server/initializers/constants' 6import { WEBSERVER } from '@server/initializers/constants'
@@ -9,15 +10,16 @@ import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
9import { getServerActor } from '@server/models/application/application' 10import { getServerActor } from '@server/models/application/application'
10import { ServerModel } from '@server/models/server/server' 11import { ServerModel } from '@server/models/server/server'
11import { ServerBlocklistModel } from '@server/models/server/server-blocklist' 12import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
13import { UserModel } from '@server/models/user/user'
12import { VideoModel } from '@server/models/video/video' 14import { VideoModel } from '@server/models/video/video'
13import { VideoBlacklistModel } from '@server/models/video/video-blacklist' 15import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
14import { MPlugin } from '@server/types/models' 16import { MPlugin } from '@server/types/models'
15import { PeerTubeHelpers } from '@server/types/plugins' 17import { PeerTubeHelpers } from '@server/types/plugins'
16import { VideoBlacklistCreate } from '@shared/models' 18import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
17import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' 19import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
18import { ServerConfigManager } from '../server-config-manager' 20import { ServerConfigManager } from '../server-config-manager'
19import { blacklistVideo, unblacklistVideo } from '../video-blacklist' 21import { blacklistVideo, unblacklistVideo } from '../video-blacklist'
20import { UserModel } from '@server/models/user/user' 22import { VideoPathManager } from '../video-path-manager'
21 23
22function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { 24function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers {
23 const logger = buildPluginLogger(npmName) 25 const logger = buildPluginLogger(npmName)
@@ -85,6 +87,60 @@ function buildVideosHelpers () {
85 87
86 await video.destroy({ transaction: t }) 88 await video.destroy({ transaction: t })
87 }) 89 })
90 },
91
92 ffprobe: (path: string) => {
93 return ffprobePromise(path)
94 },
95
96 getFiles: async (id: number | string) => {
97 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id)
98 if (!video) return undefined
99
100 const webtorrentVideoFiles = (video.VideoFiles || []).map(f => ({
101 path: f.storage === VideoStorage.FILE_SYSTEM
102 ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f)
103 : null,
104 url: f.getFileUrl(video),
105
106 resolution: f.resolution,
107 size: f.size,
108 fps: f.fps
109 }))
110
111 const hls = video.getHLSPlaylist()
112
113 const hlsVideoFiles = hls
114 ? (video.getHLSPlaylist().VideoFiles || []).map(f => {
115 return {
116 path: f.storage === VideoStorage.FILE_SYSTEM
117 ? VideoPathManager.Instance.getFSVideoFileOutputPath(hls, f)
118 : null,
119 url: f.getFileUrl(video),
120 resolution: f.resolution,
121 size: f.size,
122 fps: f.fps
123 }
124 })
125 : []
126
127 const thumbnails = video.Thumbnails.map(t => ({
128 type: t.type,
129 url: t.getFileUrl(video),
130 path: t.getPath()
131 }))
132
133 return {
134 webtorrent: {
135 videoFiles: webtorrentVideoFiles
136 },
137
138 hls: {
139 videoFiles: hlsVideoFiles
140 },
141
142 thumbnails
143 }
88 } 144 }
89 } 145 }
90} 146}
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index d4d2a7edc..39e7f9a5b 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -1,11 +1,17 @@
1import decache from 'decache'
2import express from 'express' 1import express from 'express'
3import { createReadStream, createWriteStream } from 'fs' 2import { createReadStream, createWriteStream } from 'fs'
4import { ensureDir, outputFile, readJSON } from 'fs-extra' 3import { ensureDir, outputFile, readJSON } from 'fs-extra'
5import { basename, join } from 'path' 4import { basename, join } from 'path'
5import { decachePlugin } from '@server/helpers/decache'
6import { MOAuthTokenUser, MUser } from '@server/types/models' 6import { MOAuthTokenUser, MUser } from '@server/types/models'
7import { getCompleteLocale } from '@shared/core-utils' 7import { getCompleteLocale } from '@shared/core-utils'
8import { ClientScript, PluginPackageJson, PluginTranslation, PluginTranslationPaths, RegisterServerHookOptions } from '@shared/models' 8import {
9 ClientScriptJSON,
10 PluginPackageJSON,
11 PluginTranslation,
12 PluginTranslationPathsJSON,
13 RegisterServerHookOptions
14} from '@shared/models'
9import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' 15import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks'
10import { PluginType } from '../../../shared/models/plugins/plugin.type' 16import { PluginType } from '../../../shared/models/plugins/plugin.type'
11import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' 17import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model'
@@ -31,7 +37,7 @@ export interface RegisteredPlugin {
31 path: string 37 path: string
32 38
33 staticDirs: { [name: string]: string } 39 staticDirs: { [name: string]: string }
34 clientScripts: { [name: string]: ClientScript } 40 clientScripts: { [name: string]: ClientScriptJSON }
35 41
36 css: string[] 42 css: string[]
37 43
@@ -392,7 +398,7 @@ export class PluginManager implements ServerHook {
392 registerHelpers = result.registerStore 398 registerHelpers = result.registerStore
393 } 399 }
394 400
395 const clientScripts: { [id: string]: ClientScript } = {} 401 const clientScripts: { [id: string]: ClientScriptJSON } = {}
396 for (const c of packageJSON.clientScripts) { 402 for (const c of packageJSON.clientScripts) {
397 clientScripts[c.script] = c 403 clientScripts[c.script] = c
398 } 404 }
@@ -415,12 +421,12 @@ export class PluginManager implements ServerHook {
415 await this.addTranslations(plugin, npmName, packageJSON.translations) 421 await this.addTranslations(plugin, npmName, packageJSON.translations)
416 } 422 }
417 423
418 private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJson) { 424 private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) {
419 const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) 425 const npmName = PluginModel.buildNpmName(plugin.name, plugin.type)
420 426
421 // Delete cache if needed 427 // Delete cache if needed
422 const modulePath = join(pluginPath, packageJSON.library) 428 const modulePath = join(pluginPath, packageJSON.library)
423 decache(modulePath) 429 decachePlugin(pluginPath, modulePath)
424 const library: PluginLibrary = require(modulePath) 430 const library: PluginLibrary = require(modulePath)
425 431
426 if (!isLibraryCodeValid(library)) { 432 if (!isLibraryCodeValid(library)) {
@@ -442,7 +448,7 @@ export class PluginManager implements ServerHook {
442 448
443 // ###################### Translations ###################### 449 // ###################### Translations ######################
444 450
445 private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPaths) { 451 private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) {
446 for (const locale of Object.keys(translationPaths)) { 452 for (const locale of Object.keys(translationPaths)) {
447 const path = translationPaths[locale] 453 const path = translationPaths[locale]
448 const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) 454 const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path))
@@ -513,7 +519,7 @@ export class PluginManager implements ServerHook {
513 private getPackageJSON (pluginName: string, pluginType: PluginType) { 519 private getPackageJSON (pluginName: string, pluginType: PluginType) {
514 const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json') 520 const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json')
515 521
516 return readJSON(pluginPath) as Promise<PluginPackageJson> 522 return readJSON(pluginPath) as Promise<PluginPackageJSON>
517 } 523 }
518 524
519 private getPluginPath (pluginName: string, pluginType: PluginType) { 525 private getPluginPath (pluginName: string, pluginType: PluginType) {
@@ -572,7 +578,7 @@ export class PluginManager implements ServerHook {
572 } 578 }
573 } 579 }
574 580
575 private sanitizeAndCheckPackageJSONOrThrow (packageJSON: PluginPackageJson, pluginType: PluginType) { 581 private sanitizeAndCheckPackageJSONOrThrow (packageJSON: PluginPackageJSON, pluginType: PluginType) {
576 if (!packageJSON.staticDirs) packageJSON.staticDirs = {} 582 if (!packageJSON.staticDirs) packageJSON.staticDirs = {}
577 if (!packageJSON.css) packageJSON.css = [] 583 if (!packageJSON.css) packageJSON.css = []
578 if (!packageJSON.clientScripts) packageJSON.clientScripts = [] 584 if (!packageJSON.clientScripts) packageJSON.clientScripts = []
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 8aec4b793..52766663a 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -1,31 +1,30 @@
1import express from 'express' 1import { createClient, RedisClientOptions, RedisModules } from 'redis'
2import { createClient, RedisClient } from 'redis' 2import { exists } from '@server/helpers/custom-validators/misc'
3import { sha256 } from '@shared/extra-utils'
3import { logger } from '../helpers/logger' 4import { logger } from '../helpers/logger'
4import { generateRandomString } from '../helpers/utils' 5import { generateRandomString } from '../helpers/utils'
6import { CONFIG } from '../initializers/config'
5import { 7import {
8 AP_CLEANER,
6 CONTACT_FORM_LIFETIME, 9 CONTACT_FORM_LIFETIME,
10 RESUMABLE_UPLOAD_SESSION_LIFETIME,
11 TRACKER_RATE_LIMITS,
7 USER_EMAIL_VERIFY_LIFETIME, 12 USER_EMAIL_VERIFY_LIFETIME,
8 USER_PASSWORD_RESET_LIFETIME,
9 USER_PASSWORD_CREATE_LIFETIME, 13 USER_PASSWORD_CREATE_LIFETIME,
14 USER_PASSWORD_RESET_LIFETIME,
10 VIEW_LIFETIME, 15 VIEW_LIFETIME,
11 WEBSERVER, 16 WEBSERVER
12 TRACKER_RATE_LIMITS,
13 RESUMABLE_UPLOAD_SESSION_LIFETIME
14} from '../initializers/constants' 17} from '../initializers/constants'
15import { CONFIG } from '../initializers/config'
16import { exists } from '@server/helpers/custom-validators/misc'
17 18
18type CachedRoute = { 19// Only used for typings
19 body: string 20const redisClientWrapperForType = () => createClient<{}>()
20 contentType?: string
21 statusCode?: string
22}
23 21
24class Redis { 22class Redis {
25 23
26 private static instance: Redis 24 private static instance: Redis
27 private initialized = false 25 private initialized = false
28 private client: RedisClient 26 private connected = false
27 private client: ReturnType<typeof redisClientWrapperForType>
29 private prefix: string 28 private prefix: string
30 29
31 private constructor () { 30 private constructor () {
@@ -38,26 +37,43 @@ class Redis {
38 37
39 this.client = createClient(Redis.getRedisClientOptions()) 38 this.client = createClient(Redis.getRedisClientOptions())
40 39
41 this.client.on('error', err => { 40 logger.info('Connecting to redis...')
42 logger.error('Error in Redis client.', { err })
43 process.exit(-1)
44 })
45 41
46 if (CONFIG.REDIS.AUTH) { 42 this.client.connect()
47 this.client.auth(CONFIG.REDIS.AUTH) 43 .then(() => {
48 } 44 logger.info('Connected to redis.')
45
46 this.connected = true
47 }).catch(err => {
48 logger.error('Cannot connect to redis', { err })
49 process.exit(-1)
50 })
49 51
50 this.prefix = 'redis-' + WEBSERVER.HOST + '-' 52 this.prefix = 'redis-' + WEBSERVER.HOST + '-'
51 } 53 }
52 54
53 static getRedisClientOptions () { 55 static getRedisClientOptions () {
54 return Object.assign({}, 56 let config: RedisClientOptions<RedisModules, {}> = {
55 (CONFIG.REDIS.AUTH && CONFIG.REDIS.AUTH != null) ? { password: CONFIG.REDIS.AUTH } : {}, 57 socket: {
56 (CONFIG.REDIS.DB) ? { db: CONFIG.REDIS.DB } : {}, 58 connectTimeout: 20000 // Could be slow since node use sync call to compile PeerTube
57 (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) 59 }
58 ? { host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT } 60 }
59 : { path: CONFIG.REDIS.SOCKET } 61
60 ) 62 if (CONFIG.REDIS.AUTH) {
63 config = { ...config, password: CONFIG.REDIS.AUTH }
64 }
65
66 if (CONFIG.REDIS.DB) {
67 config = { ...config, database: CONFIG.REDIS.DB }
68 }
69
70 if (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) {
71 config.socket = { ...config.socket, host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT }
72 } else {
73 config.socket = { ...config.socket, path: CONFIG.REDIS.SOCKET }
74 }
75
76 return config
61 } 77 }
62 78
63 getClient () { 79 getClient () {
@@ -68,6 +84,10 @@ class Redis {
68 return this.prefix 84 return this.prefix
69 } 85 }
70 86
87 isConnected () {
88 return this.connected
89 }
90
71 /* ************ Forgot password ************ */ 91 /* ************ Forgot password ************ */
72 92
73 async setResetPasswordVerificationString (userId: number) { 93 async setResetPasswordVerificationString (userId: number) {
@@ -146,25 +166,6 @@ class Redis {
146 return this.exists(this.generateTrackerBlockIPKey(ip)) 166 return this.exists(this.generateTrackerBlockIPKey(ip))
147 } 167 }
148 168
149 /* ************ API cache ************ */
150
151 async getCachedRoute (req: express.Request) {
152 const cached = await this.getObject(this.generateCachedRouteKey(req))
153
154 return cached as CachedRoute
155 }
156
157 setCachedRoute (req: express.Request, body: any, lifetime: number, contentType?: string, statusCode?: number) {
158 const cached: CachedRoute = Object.assign(
159 {},
160 { body: body.toString() },
161 (contentType) ? { contentType } : null,
162 (statusCode) ? { statusCode: statusCode.toString() } : null
163 )
164
165 return this.setObject(this.generateCachedRouteKey(req), cached, lifetime)
166 }
167
168 /* ************ Video views stats ************ */ 169 /* ************ Video views stats ************ */
169 170
170 addVideoViewStats (videoId: number) { 171 addVideoViewStats (videoId: number) {
@@ -275,12 +276,19 @@ class Redis {
275 return this.deleteKey('resumable-upload-' + uploadId) 276 return this.deleteKey('resumable-upload-' + uploadId)
276 } 277 }
277 278
278 /* ************ Keys generation ************ */ 279 /* ************ AP ressource unavailability ************ */
279 280
280 generateCachedRouteKey (req: express.Request) { 281 async addAPUnavailability (url: string) {
281 return req.method + '-' + req.originalUrl 282 const key = this.generateAPUnavailabilityKey(url)
283
284 const value = await this.increment(key)
285 await this.setExpiration(key, AP_CLEANER.PERIOD * 2)
286
287 return value
282 } 288 }
283 289
290 /* ************ Keys generation ************ */
291
284 private generateLocalVideoViewsKeys (videoId?: Number) { 292 private generateLocalVideoViewsKeys (videoId?: Number) {
285 return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } 293 return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
286 } 294 }
@@ -317,128 +325,52 @@ class Redis {
317 return 'contact-form-' + ip 325 return 'contact-form-' + ip
318 } 326 }
319 327
328 private generateAPUnavailabilityKey (url: string) {
329 return 'ap-unavailability-' + sha256(url)
330 }
331
320 /* ************ Redis helpers ************ */ 332 /* ************ Redis helpers ************ */
321 333
322 private getValue (key: string) { 334 private getValue (key: string) {
323 return new Promise<string>((res, rej) => { 335 return this.client.get(this.prefix + key)
324 this.client.get(this.prefix + key, (err, value) => {
325 if (err) return rej(err)
326
327 return res(value)
328 })
329 })
330 } 336 }
331 337
332 private getSet (key: string) { 338 private getSet (key: string) {
333 return new Promise<string[]>((res, rej) => { 339 return this.client.sMembers(this.prefix + key)
334 this.client.smembers(this.prefix + key, (err, value) => {
335 if (err) return rej(err)
336
337 return res(value)
338 })
339 })
340 } 340 }
341 341
342 private addToSet (key: string, value: string) { 342 private addToSet (key: string, value: string) {
343 return new Promise<void>((res, rej) => { 343 return this.client.sAdd(this.prefix + key, value)
344 this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res())
345 })
346 } 344 }
347 345
348 private deleteFromSet (key: string, value: string) { 346 private deleteFromSet (key: string, value: string) {
349 return new Promise<void>((res, rej) => { 347 return this.client.sRem(this.prefix + key, value)
350 this.client.srem(this.prefix + key, value, err => err ? rej(err) : res())
351 })
352 } 348 }
353 349
354 private deleteKey (key: string) { 350 private deleteKey (key: string) {
355 return new Promise<void>((res, rej) => { 351 return this.client.del(this.prefix + key)
356 this.client.del(this.prefix + key, err => err ? rej(err) : res())
357 })
358 }
359
360 private deleteFieldInHash (key: string, field: string) {
361 return new Promise<void>((res, rej) => {
362 this.client.hdel(this.prefix + key, field, err => err ? rej(err) : res())
363 })
364 } 352 }
365 353
366 private setValue (key: string, value: string, expirationMilliseconds: number) { 354 private async setValue (key: string, value: string, expirationMilliseconds: number) {
367 return new Promise<void>((res, rej) => { 355 const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds })
368 this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => {
369 if (err) return rej(err)
370 356
371 if (ok !== 'OK') return rej(new Error('Redis set result is not OK.')) 357 if (result !== 'OK') throw new Error('Redis set result is not OK.')
372
373 return res()
374 })
375 })
376 } 358 }
377 359
378 private removeValue (key: string) { 360 private removeValue (key: string) {
379 return new Promise<void>((res, rej) => { 361 return this.client.del(this.prefix + key)
380 this.client.del(this.prefix + key, err => {
381 if (err) return rej(err)
382
383 return res()
384 })
385 })
386 }
387
388 private setObject (key: string, obj: { [id: string]: string }, expirationMilliseconds: number) {
389 return new Promise<void>((res, rej) => {
390 this.client.hmset(this.prefix + key, obj, (err, ok) => {
391 if (err) return rej(err)
392 if (!ok) return rej(new Error('Redis mset result is not OK.'))
393
394 this.client.pexpire(this.prefix + key, expirationMilliseconds, (err, ok) => {
395 if (err) return rej(err)
396 if (!ok) return rej(new Error('Redis expiration result is not OK.'))
397
398 return res()
399 })
400 })
401 })
402 }
403
404 private getObject (key: string) {
405 return new Promise<{ [id: string]: string }>((res, rej) => {
406 this.client.hgetall(this.prefix + key, (err, value) => {
407 if (err) return rej(err)
408
409 return res(value)
410 })
411 })
412 }
413
414 private setValueInHash (key: string, field: string, value: string) {
415 return new Promise<void>((res, rej) => {
416 this.client.hset(this.prefix + key, field, value, (err) => {
417 if (err) return rej(err)
418
419 return res()
420 })
421 })
422 } 362 }
423 363
424 private increment (key: string) { 364 private increment (key: string) {
425 return new Promise<number>((res, rej) => { 365 return this.client.incr(this.prefix + key)
426 this.client.incr(this.prefix + key, (err, value) => {
427 if (err) return rej(err)
428
429 return res(value)
430 })
431 })
432 } 366 }
433 367
434 private exists (key: string) { 368 private exists (key: string) {
435 return new Promise<boolean>((res, rej) => { 369 return this.client.exists(this.prefix + key)
436 this.client.exists(this.prefix + key, (err, existsNumber) => { 370 }
437 if (err) return rej(err)
438 371
439 return res(existsNumber === 1) 372 private setExpiration (key: string, ms: number) {
440 }) 373 return this.client.expire(this.prefix + key, ms / 1000)
441 })
442 } 374 }
443 375
444 static get Instance () { 376 static get Instance () {
diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
index d6e561cad..61e93eafa 100644
--- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
+++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
@@ -1,9 +1,7 @@
1import { map } from 'bluebird' 1
2import { readdir, remove, stat } from 'fs-extra'
3import { logger, loggerTagsFactory } from '@server/helpers/logger' 2import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { getResumableUploadPath } from '@server/helpers/upload'
5import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' 3import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants'
6import { METAFILE_EXTNAME } from '@uploadx/core' 4import { uploadx } from '../uploadx'
7import { AbstractScheduler } from './abstract-scheduler' 5import { AbstractScheduler } from './abstract-scheduler'
8 6
9const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') 7const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
@@ -22,36 +20,17 @@ export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
22 } 20 }
23 21
24 protected async internalExecute () { 22 protected async internalExecute () {
25 const path = getResumableUploadPath() 23 logger.debug('Removing dangling resumable uploads', lTags())
26 const files = await readdir(path)
27
28 const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME))
29 24
30 if (metafiles.length === 0) return 25 const now = new Date().getTime()
31
32 logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags())
33 26
34 try { 27 try {
35 await map(metafiles, metafile => { 28 // Remove files that were not updated since the last execution
36 return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs) 29 await uploadx.storage.purge(now - this.lastExecutionTimeMs)
37 }, { concurrency: 5 })
38 } catch (error) { 30 } catch (error) {
39 logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) 31 logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
40 } finally { 32 } finally {
41 this.lastExecutionTimeMs = new Date().getTime() 33 this.lastExecutionTimeMs = now
42 }
43 }
44
45 private async deleteIfOlderThan (metafile: string, olderThan: number) {
46 const metafilePath = getResumableUploadPath(metafile)
47 const statResult = await stat(metafilePath)
48
49 // Delete uploads that started since a long time
50 if (statResult.ctimeMs < olderThan) {
51 await remove(metafilePath)
52
53 const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '')
54 await remove(datafile)
55 } 34 }
56 } 35 }
57 36
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts
index bdf6492f9..d97f21eb7 100644
--- a/server/lib/server-config-manager.ts
+++ b/server/lib/server-config-manager.ts
@@ -3,6 +3,7 @@ import { CONFIG, isEmailEnabled } from '@server/initializers/config'
3import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' 3import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants'
4import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup' 4import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup'
5import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' 5import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
6import { PluginModel } from '@server/models/server/plugin'
6import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models' 7import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models'
7import { Hooks } from './plugins/hooks' 8import { Hooks } from './plugins/hooks'
8import { PluginManager } from './plugins/plugin-manager' 9import { PluginManager } from './plugins/plugin-manager'
@@ -47,6 +48,28 @@ class ServerConfigManager {
47 miniature: { 48 miniature: {
48 preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME 49 preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME
49 } 50 }
51 },
52 menu: {
53 login: {
54 redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH
55 }
56 }
57 },
58
59 defaults: {
60 publish: {
61 downloadEnabled: CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
62 commentsEnabled: CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
63 privacy: CONFIG.DEFAULTS.PUBLISH.PRIVACY,
64 licence: CONFIG.DEFAULTS.PUBLISH.LICENCE
65 },
66 p2p: {
67 webapp: {
68 enabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED
69 },
70 embed: {
71 enabled: CONFIG.DEFAULTS.P2P.EMBED.ENABLED
72 }
50 } 73 }
51 }, 74 },
52 75
@@ -247,6 +270,7 @@ class ServerConfigManager {
247 getRegisteredThemes () { 270 getRegisteredThemes () {
248 return PluginManager.Instance.getRegisteredThemes() 271 return PluginManager.Instance.getRegisteredThemes()
249 .map(t => ({ 272 .map(t => ({
273 npmName: PluginModel.buildNpmName(t.name, t.type),
250 name: t.name, 274 name: t.name,
251 version: t.version, 275 version: t.version,
252 description: t.description, 276 description: t.description,
@@ -258,6 +282,7 @@ class ServerConfigManager {
258 getRegisteredPlugins () { 282 getRegisteredPlugins () {
259 return PluginManager.Instance.getRegisteredPlugins() 283 return PluginManager.Instance.getRegisteredPlugins()
260 .map(p => ({ 284 .map(p => ({
285 npmName: PluginModel.buildNpmName(p.name, p.type),
261 name: p.name, 286 name: p.name,
262 version: p.version, 287 version: p.version,
263 description: p.description, 288 description: p.description,
diff --git a/server/lib/transcoding/video-transcoding-profiles.ts b/server/lib/transcoding/video-transcoding-profiles.ts
index 34a364415..dcc8d4c5c 100644
--- a/server/lib/transcoding/video-transcoding-profiles.ts
+++ b/server/lib/transcoding/video-transcoding-profiles.ts
@@ -23,10 +23,9 @@ const defaultX264VODOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOpt
23 23
24 return { 24 return {
25 outputOptions: [ 25 outputOptions: [
26 `-preset veryfast`, 26 ...getCommonOutputOptions(targetBitrate),
27 `-r ${fps}`, 27
28 `-maxrate ${targetBitrate}`, 28 `-r ${fps}`
29 `-bufsize ${targetBitrate * 2}`
30 ] 29 ]
31 } 30 }
32} 31}
@@ -38,11 +37,10 @@ const defaultX264LiveOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOp
38 37
39 return { 38 return {
40 outputOptions: [ 39 outputOptions: [
41 `-preset veryfast`, 40 ...getCommonOutputOptions(targetBitrate),
41
42 `${buildStreamSuffix('-r:v', streamNum)} ${fps}`, 42 `${buildStreamSuffix('-r:v', streamNum)} ${fps}`,
43 `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}`, 43 `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}`
44 `-maxrate ${targetBitrate}`,
45 `-bufsize ${targetBitrate * 2}`
46 ] 44 ]
47 } 45 }
48} 46}
@@ -257,3 +255,16 @@ function capBitrate (inputBitrate: number, targetBitrate: number) {
257 255
258 return Math.min(targetBitrate, inputBitrateWithMargin) 256 return Math.min(targetBitrate, inputBitrateWithMargin)
259} 257}
258
259function getCommonOutputOptions (targetBitrate: number) {
260 return [
261 `-preset veryfast`,
262 `-maxrate ${targetBitrate}`,
263 `-bufsize ${targetBitrate * 2}`,
264
265 // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it
266 `-b_strategy 1`,
267 // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
268 `-bf 16`
269 ]
270}
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index d0db05216..9942a067b 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -169,6 +169,7 @@ function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolutio
169 169
170 // Important to do this before getVideoFilename() to take in account the new file extension 170 // Important to do this before getVideoFilename() to take in account the new file extension
171 inputVideoFile.extname = newExtname 171 inputVideoFile.extname = newExtname
172 inputVideoFile.resolution = resolution
172 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) 173 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
173 174
174 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 175 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts
new file mode 100644
index 000000000..36f5a556c
--- /dev/null
+++ b/server/lib/uploadx.ts
@@ -0,0 +1,14 @@
1import express from 'express'
2import { getResumableUploadPath } from '@server/helpers/upload'
3import { Uploadx } from '@uploadx/core'
4
5const uploadx = new Uploadx({
6 directory: getResumableUploadPath(),
7 // Could be big with thumbnails/previews
8 maxMetadataSize: '10MB'
9})
10uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id
11
12export {
13 uploadx
14}
diff --git a/server/lib/user.ts b/server/lib/user.ts
index 936403692..0d292ac90 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -1,7 +1,7 @@
1import { Transaction } from 'sequelize/types' 1import { Transaction } from 'sequelize/types'
2import { buildUUID } from '@server/helpers/uuid'
3import { UserModel } from '@server/models/user/user' 2import { UserModel } from '@server/models/user/user'
4import { MActorDefault } from '@server/types/models/actor' 3import { MActorDefault } from '@server/types/models/actor'
4import { buildUUID } from '@shared/extra-utils'
5import { ActivityPubActorType } from '../../shared/models/activitypub' 5import { ActivityPubActorType } from '../../shared/models/activitypub'
6import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users' 6import { UserNotificationSetting, UserNotificationSettingValue } from '../../shared/models/users'
7import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' 7import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants'
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
index 27058005c..c3f55fd95 100644
--- a/server/lib/video-path-manager.ts
+++ b/server/lib/video-path-manager.ts
@@ -1,6 +1,5 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import { extname, join } from 'path' 2import { extname, join } from 'path'
3import { buildUUID } from '@server/helpers/uuid'
4import { extractVideo } from '@server/helpers/video' 3import { extractVideo } from '@server/helpers/video'
5import { CONFIG } from '@server/initializers/config' 4import { CONFIG } from '@server/initializers/config'
6import { 5import {
@@ -11,6 +10,7 @@ import {
11 MVideoFileVideo, 10 MVideoFileVideo,
12 MVideoUUID 11 MVideoUUID
13} from '@server/types/models' 12} from '@server/types/models'
13import { buildUUID } from '@shared/extra-utils'
14import { VideoStorage } from '@shared/models' 14import { VideoStorage } from '@shared/models'
15import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' 15import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
16import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' 16import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
index e420991cd..97ff540ed 100644
--- a/server/lib/video-state.ts
+++ b/server/lib/video-state.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
4import { sequelizeTypescript } from '@server/initializers/database' 4import { sequelizeTypescript } from '@server/initializers/database'
5import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info' 6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { MVideoFullLight, MVideoUUID } from '@server/types/models' 7import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models'
8import { VideoState } from '@shared/models' 8import { VideoState } from '@shared/models'
9import { federateVideoIfNeeded } from './activitypub/videos' 9import { federateVideoIfNeeded } from './activitypub/videos'
10import { Notifier } from './notifier' 10import { Notifier } from './notifier'
@@ -79,18 +79,25 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
79 } 79 }
80} 80}
81 81
82function moveToFailedTranscodingState (video: MVideoFullLight) { 82function moveToFailedTranscodingState (video: MVideo) {
83 if (video.state === VideoState.TRANSCODING_FAILED) return 83 if (video.state === VideoState.TRANSCODING_FAILED) return
84 84
85 return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) 85 return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
86} 86}
87 87
88function moveToFailedMoveToObjectStorageState (video: MVideo) {
89 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return
90
91 return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
92}
93
88// --------------------------------------------------------------------------- 94// ---------------------------------------------------------------------------
89 95
90export { 96export {
91 buildNextVideoState, 97 buildNextVideoState,
92 moveToExternalStorageState, 98 moveToExternalStorageState,
93 moveToFailedTranscodingState, 99 moveToFailedTranscodingState,
100 moveToFailedMoveToObjectStorageState,
94 moveToNextState 101 moveToNextState
95} 102}
96 103
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 1cfe4f27c..e5af028ea 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -9,16 +9,17 @@ import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID
9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' 9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
10import { CreateJobOptions, JobQueue } from './job-queue/job-queue' 10import { CreateJobOptions, JobQueue } from './job-queue/job-queue'
11import { updateVideoMiniatureFromExisting } from './thumbnail' 11import { updateVideoMiniatureFromExisting } from './thumbnail'
12import { CONFIG } from '@server/initializers/config'
12 13
13function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 14function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
14 return { 15 return {
15 name: videoInfo.name, 16 name: videoInfo.name,
16 remote: false, 17 remote: false,
17 category: videoInfo.category, 18 category: videoInfo.category,
18 licence: videoInfo.licence, 19 licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
19 language: videoInfo.language, 20 language: videoInfo.language,
20 commentsEnabled: videoInfo.commentsEnabled !== false, // If the value is not "false", the default is "true" 21 commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
21 downloadEnabled: videoInfo.downloadEnabled !== false, 22 downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
22 waitTranscoding: videoInfo.waitTranscoding || false, 23 waitTranscoding: videoInfo.waitTranscoding || false,
23 nsfw: videoInfo.nsfw || false, 24 nsfw: videoInfo.nsfw || false,
24 description: videoInfo.description, 25 description: videoInfo.description,
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-19 01:54:45 +0000