aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/users/index.ts4
-rw-r--r--server/controllers/api/users/token.ts7
-rw-r--r--server/controllers/api/users/two-factor.ts95
-rw-r--r--server/controllers/index.ts1
-rw-r--r--server/controllers/live.ts32
-rw-r--r--server/controllers/well-known.ts7
-rw-r--r--server/helpers/core-utils.ts14
-rw-r--r--server/helpers/otp.ts58
-rw-r--r--server/helpers/peertube-crypto.ts49
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts15
-rw-r--r--server/initializers/checker-after-init.ts7
-rw-r--r--server/initializers/checker-before-init.ts3
-rw-r--r--server/initializers/config.ts6
-rw-r--r--server/initializers/constants.ts23
-rw-r--r--server/initializers/migrations/0745-user-otp.ts29
-rw-r--r--server/lib/activitypub/process/process-create.ts2
-rw-r--r--server/lib/activitypub/video-comments.ts35
-rw-r--r--server/lib/auth/oauth.ts27
-rw-r--r--server/lib/hls.ts6
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts10
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts22
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts27
-rw-r--r--server/lib/live/live-manager.ts12
-rw-r--r--server/lib/live/live-segment-sha-store.ts75
-rw-r--r--server/lib/live/live-utils.ts67
-rw-r--r--server/lib/live/shared/muxing-session.ts106
-rw-r--r--server/lib/moderation.ts42
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts25
-rw-r--r--server/lib/object-storage/videos.ts37
-rw-r--r--server/lib/plugins/plugin-helpers-builder.ts28
-rw-r--r--server/lib/plugins/plugin-manager.ts31
-rw-r--r--server/lib/plugins/register-helpers.ts21
-rw-r--r--server/lib/redis.ts25
-rw-r--r--server/lib/schedulers/video-channel-sync-latest-scheduler.ts35
-rw-r--r--server/lib/sync-channel.ts90
-rw-r--r--server/middlewares/validators/shared/index.ts1
-rw-r--r--server/middlewares/validators/shared/users.ts62
-rw-r--r--server/middlewares/validators/two-factor.ts81
-rw-r--r--server/middlewares/validators/users.ts118
-rw-r--r--server/middlewares/validators/videos/video-comments.ts5
-rw-r--r--server/models/user/user.ts9
-rw-r--r--server/models/video/video-job-info.ts6
-rw-r--r--server/models/video/video-streaming-playlist.ts24
-rw-r--r--server/tests/api/check-params/index.ts5
-rw-r--r--server/tests/api/check-params/two-factor.ts288
-rw-r--r--server/tests/api/live/live-fast-restream.ts2
-rw-r--r--server/tests/api/live/live.ts91
-rw-r--r--server/tests/api/notifications/admin-notifications.ts6
-rw-r--r--server/tests/api/notifications/moderation-notifications.ts2
-rw-r--r--server/tests/api/object-storage/live.ts183
-rw-r--r--server/tests/api/redundancy/redundancy.ts14
-rw-r--r--server/tests/api/users/index.ts1
-rw-r--r--server/tests/api/users/two-factor.ts200
-rw-r--r--server/tests/api/users/users-multiple-servers.ts2
-rw-r--r--server/tests/api/videos/multiple-servers.ts2
-rw-r--r--server/tests/api/videos/video-files.ts2
-rw-r--r--server/tests/api/videos/video-playlists.ts4
-rw-r--r--server/tests/api/videos/video-privacy.ts2
-rw-r--r--server/tests/api/videos/videos-common-filters.ts2
-rw-r--r--server/tests/external-plugins/akismet.ts160
-rw-r--r--server/tests/external-plugins/auth-ldap.ts8
-rw-r--r--server/tests/external-plugins/index.ts1
-rw-r--r--server/tests/fixtures/peertube-plugin-test-four/main.js16
-rw-r--r--server/tests/fixtures/peertube-plugin-test-websocket/main.js36
-rw-r--r--server/tests/fixtures/peertube-plugin-test-websocket/package.json20
-rw-r--r--server/tests/fixtures/peertube-plugin-test/main.js9
-rw-r--r--server/tests/helpers/crypto.ts33
-rw-r--r--server/tests/helpers/index.ts5
-rw-r--r--server/tests/misc-endpoints.ts30
-rw-r--r--server/tests/plugins/filter-hooks.ts383
-rw-r--r--server/tests/plugins/index.ts1
-rw-r--r--server/tests/plugins/plugin-helpers.ts27
-rw-r--r--server/tests/plugins/plugin-websocket.ts70
-rw-r--r--server/tests/shared/actors.ts8
-rw-r--r--server/tests/shared/directories.ts8
-rw-r--r--server/tests/shared/live.ts119
-rw-r--r--server/tests/shared/playlists.ts9
-rw-r--r--server/tests/shared/streaming-playlists.ts13
-rw-r--r--server/types/plugins/index.ts1
-rw-r--r--server/types/plugins/register-server-option.model.ts21
-rw-r--r--server/types/plugins/register-server-websocket-route.model.ts8
81 files changed, 2502 insertions, 669 deletions
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 0b27d5277..a8677a1d3 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -51,6 +51,7 @@ import { myVideosHistoryRouter } from './my-history'
51import { myNotificationsRouter } from './my-notifications' 51import { myNotificationsRouter } from './my-notifications'
52import { mySubscriptionsRouter } from './my-subscriptions' 52import { mySubscriptionsRouter } from './my-subscriptions'
53import { myVideoPlaylistsRouter } from './my-video-playlists' 53import { myVideoPlaylistsRouter } from './my-video-playlists'
54import { twoFactorRouter } from './two-factor'
54 55
55const auditLogger = auditLoggerFactory('users') 56const auditLogger = auditLoggerFactory('users')
56 57
@@ -66,6 +67,7 @@ const askSendEmailLimiter = buildRateLimiter({
66}) 67})
67 68
68const usersRouter = express.Router() 69const usersRouter = express.Router()
70usersRouter.use('/', twoFactorRouter)
69usersRouter.use('/', tokensRouter) 71usersRouter.use('/', tokensRouter)
70usersRouter.use('/', myNotificationsRouter) 72usersRouter.use('/', myNotificationsRouter)
71usersRouter.use('/', mySubscriptionsRouter) 73usersRouter.use('/', mySubscriptionsRouter)
@@ -343,7 +345,7 @@ async function askResetUserPassword (req: express.Request, res: express.Response
343 345
344 const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) 346 const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id)
345 const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString 347 const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString
346 await Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url) 348 Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url)
347 349
348 return res.status(HttpStatusCode.NO_CONTENT_204).end() 350 return res.status(HttpStatusCode.NO_CONTENT_204).end()
349} 351}
diff --git a/server/controllers/api/users/token.ts b/server/controllers/api/users/token.ts
index 012a49791..c6afea67c 100644
--- a/server/controllers/api/users/token.ts
+++ b/server/controllers/api/users/token.ts
@@ -1,8 +1,9 @@
1import express from 'express' 1import express from 'express'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { OTP } from '@server/initializers/constants'
4import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' 5import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth'
5import { handleOAuthToken } from '@server/lib/auth/oauth' 6import { handleOAuthToken, MissingTwoFactorError } from '@server/lib/auth/oauth'
6import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' 7import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model'
7import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
8import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares' 9import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares'
@@ -79,6 +80,10 @@ async function handleToken (req: express.Request, res: express.Response, next: e
79 } catch (err) { 80 } catch (err) {
80 logger.warn('Login error', { err }) 81 logger.warn('Login error', { err })
81 82
83 if (err instanceof MissingTwoFactorError) {
84 res.set(OTP.HEADER_NAME, OTP.HEADER_REQUIRED_VALUE)
85 }
86
82 return res.fail({ 87 return res.fail({
83 status: err.code, 88 status: err.code,
84 message: err.message, 89 message: err.message,
diff --git a/server/controllers/api/users/two-factor.ts b/server/controllers/api/users/two-factor.ts
new file mode 100644
index 000000000..e6ae9e4dd
--- /dev/null
+++ b/server/controllers/api/users/two-factor.ts
@@ -0,0 +1,95 @@
1import express from 'express'
2import { generateOTPSecret, isOTPValid } from '@server/helpers/otp'
3import { encrypt } from '@server/helpers/peertube-crypto'
4import { CONFIG } from '@server/initializers/config'
5import { Redis } from '@server/lib/redis'
6import { asyncMiddleware, authenticate, usersCheckCurrentPasswordFactory } from '@server/middlewares'
7import {
8 confirmTwoFactorValidator,
9 disableTwoFactorValidator,
10 requestOrConfirmTwoFactorValidator
11} from '@server/middlewares/validators/two-factor'
12import { HttpStatusCode, TwoFactorEnableResult } from '@shared/models'
13
14const twoFactorRouter = express.Router()
15
16twoFactorRouter.post('/:id/two-factor/request',
17 authenticate,
18 asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)),
19 asyncMiddleware(requestOrConfirmTwoFactorValidator),
20 asyncMiddleware(requestTwoFactor)
21)
22
23twoFactorRouter.post('/:id/two-factor/confirm-request',
24 authenticate,
25 asyncMiddleware(requestOrConfirmTwoFactorValidator),
26 confirmTwoFactorValidator,
27 asyncMiddleware(confirmRequestTwoFactor)
28)
29
30twoFactorRouter.post('/:id/two-factor/disable',
31 authenticate,
32 asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)),
33 asyncMiddleware(disableTwoFactorValidator),
34 asyncMiddleware(disableTwoFactor)
35)
36
37// ---------------------------------------------------------------------------
38
39export {
40 twoFactorRouter
41}
42
43// ---------------------------------------------------------------------------
44
45async function requestTwoFactor (req: express.Request, res: express.Response) {
46 const user = res.locals.user
47
48 const { secret, uri } = generateOTPSecret(user.email)
49
50 const encryptedSecret = await encrypt(secret, CONFIG.SECRETS.PEERTUBE)
51 const requestToken = await Redis.Instance.setTwoFactorRequest(user.id, encryptedSecret)
52
53 return res.json({
54 otpRequest: {
55 requestToken,
56 secret,
57 uri
58 }
59 } as TwoFactorEnableResult)
60}
61
62async function confirmRequestTwoFactor (req: express.Request, res: express.Response) {
63 const requestToken = req.body.requestToken
64 const otpToken = req.body.otpToken
65 const user = res.locals.user
66
67 const encryptedSecret = await Redis.Instance.getTwoFactorRequestToken(user.id, requestToken)
68 if (!encryptedSecret) {
69 return res.fail({
70 message: 'Invalid request token',
71 status: HttpStatusCode.FORBIDDEN_403
72 })
73 }
74
75 if (await isOTPValid({ encryptedSecret, token: otpToken }) !== true) {
76 return res.fail({
77 message: 'Invalid OTP token',
78 status: HttpStatusCode.FORBIDDEN_403
79 })
80 }
81
82 user.otpSecret = encryptedSecret
83 await user.save()
84
85 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
86}
87
88async function disableTwoFactor (req: express.Request, res: express.Response) {
89 const user = res.locals.user
90
91 user.otpSecret = null
92 await user.save()
93
94 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
95}
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index e8833d58c..8574a9e7b 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -6,7 +6,6 @@ export * from './feeds'
6export * from './services' 6export * from './services'
7export * from './static' 7export * from './static'
8export * from './lazy-static' 8export * from './lazy-static'
9export * from './live'
10export * from './misc' 9export * from './misc'
11export * from './webfinger' 10export * from './webfinger'
12export * from './tracker' 11export * from './tracker'
diff --git a/server/controllers/live.ts b/server/controllers/live.ts
deleted file mode 100644
index 81008f120..000000000
--- a/server/controllers/live.ts
+++ /dev/null
@@ -1,32 +0,0 @@
1import cors from 'cors'
2import express from 'express'
3import { mapToJSON } from '@server/helpers/core-utils'
4import { LiveSegmentShaStore } from '@server/lib/live'
5import { HttpStatusCode } from '@shared/models'
6
7const liveRouter = express.Router()
8
9liveRouter.use('/segments-sha256/:videoUUID',
10 cors(),
11 getSegmentsSha256
12)
13
14// ---------------------------------------------------------------------------
15
16export {
17 liveRouter
18}
19
20// ---------------------------------------------------------------------------
21
22function getSegmentsSha256 (req: express.Request, res: express.Response) {
23 const videoUUID = req.params.videoUUID
24
25 const result = LiveSegmentShaStore.Instance.getSegmentsSha256(videoUUID)
26
27 if (!result) {
28 return res.status(HttpStatusCode.NOT_FOUND_404).end()
29 }
30
31 return res.json(mapToJSON(result))
32}
diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts
index f467bd629..ce5883571 100644
--- a/server/controllers/well-known.ts
+++ b/server/controllers/well-known.ts
@@ -5,6 +5,7 @@ import { root } from '@shared/core-utils'
5import { CONFIG } from '../initializers/config' 5import { CONFIG } from '../initializers/config'
6import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' 6import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
7import { cacheRoute } from '../middlewares/cache/cache' 7import { cacheRoute } from '../middlewares/cache/cache'
8import { handleStaticError } from '@server/middlewares'
8 9
9const wellKnownRouter = express.Router() 10const wellKnownRouter = express.Router()
10 11
@@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta',
69 } 70 }
70) 71)
71 72
73wellKnownRouter.use('/.well-known/',
74 cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN),
75 express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }),
76 handleStaticError
77)
78
72// --------------------------------------------------------------------------- 79// ---------------------------------------------------------------------------
73 80
74export { 81export {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index c762f6a29..73bd994c1 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -6,7 +6,7 @@
6*/ 6*/
7 7
8import { exec, ExecOptions } from 'child_process' 8import { exec, ExecOptions } from 'child_process'
9import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions } from 'crypto' 9import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
10import { truncate } from 'lodash' 10import { truncate } from 'lodash'
11import { pipeline } from 'stream' 11import { pipeline } from 'stream'
12import { URL } from 'url' 12import { URL } from 'url'
@@ -311,7 +311,17 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
311 } 311 }
312} 312}
313 313
314// eslint-disable-next-line max-len
315function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> {
316 return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> {
317 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
318 func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ])
319 })
320 }
321}
322
314const randomBytesPromise = promisify1<number, Buffer>(randomBytes) 323const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
324const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
315const execPromise2 = promisify2<string, any, string>(exec) 325const execPromise2 = promisify2<string, any, string>(exec)
316const execPromise = promisify1<string, string>(exec) 326const execPromise = promisify1<string, string>(exec)
317const pipelinePromise = promisify(pipeline) 327const pipelinePromise = promisify(pipeline)
@@ -339,6 +349,8 @@ export {
339 promisify1, 349 promisify1,
340 promisify2, 350 promisify2,
341 351
352 scryptPromise,
353
342 randomBytesPromise, 354 randomBytesPromise,
343 355
344 generateRSAKeyPairPromise, 356 generateRSAKeyPairPromise,
diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts
new file mode 100644
index 000000000..a32cc9621
--- /dev/null
+++ b/server/helpers/otp.ts
@@ -0,0 +1,58 @@
1import { Secret, TOTP } from 'otpauth'
2import { CONFIG } from '@server/initializers/config'
3import { WEBSERVER } from '@server/initializers/constants'
4import { decrypt } from './peertube-crypto'
5
6async function isOTPValid (options: {
7 encryptedSecret: string
8 token: string
9}) {
10 const { token, encryptedSecret } = options
11
12 const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE)
13
14 const totp = new TOTP({
15 ...baseOTPOptions(),
16
17 secret
18 })
19
20 const delta = totp.validate({
21 token,
22 window: 1
23 })
24
25 if (delta === null) return false
26
27 return true
28}
29
30function generateOTPSecret (email: string) {
31 const totp = new TOTP({
32 ...baseOTPOptions(),
33
34 label: email,
35 secret: new Secret()
36 })
37
38 return {
39 secret: totp.secret.base32,
40 uri: totp.toString()
41 }
42}
43
44export {
45 isOTPValid,
46 generateOTPSecret
47}
48
49// ---------------------------------------------------------------------------
50
51function baseOTPOptions () {
52 return {
53 issuer: WEBSERVER.HOST,
54 algorithm: 'SHA1',
55 digits: 6,
56 period: 30
57 }
58}
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index 8aca50900..ae7d11800 100644
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -1,11 +1,11 @@
1import { compare, genSalt, hash } from 'bcrypt' 1import { compare, genSalt, hash } from 'bcrypt'
2import { createSign, createVerify } from 'crypto' 2import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
3import { Request } from 'express' 3import { Request } from 'express'
4import { cloneDeep } from 'lodash' 4import { cloneDeep } from 'lodash'
5import { sha256 } from '@shared/extra-utils' 5import { sha256 } from '@shared/extra-utils'
6import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' 6import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
7import { MActor } from '../types/models' 7import { MActor } from '../types/models'
8import { generateRSAKeyPairPromise, promisify1, promisify2 } from './core-utils' 8import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils'
9import { jsonld } from './custom-jsonld-signature' 9import { jsonld } from './custom-jsonld-signature'
10import { logger } from './logger' 10import { logger } from './logger'
11 11
@@ -21,9 +21,13 @@ function createPrivateAndPublicKeys () {
21 return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) 21 return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE)
22} 22}
23 23
24// ---------------------------------------------------------------------------
24// User password checks 25// User password checks
26// ---------------------------------------------------------------------------
25 27
26function comparePassword (plainPassword: string, hashPassword: string) { 28function comparePassword (plainPassword: string, hashPassword: string) {
29 if (!plainPassword) return Promise.resolve(false)
30
27 return bcryptComparePromise(plainPassword, hashPassword) 31 return bcryptComparePromise(plainPassword, hashPassword)
28} 32}
29 33
@@ -33,7 +37,9 @@ async function cryptPassword (password: string) {
33 return bcryptHashPromise(password, salt) 37 return bcryptHashPromise(password, salt)
34} 38}
35 39
40// ---------------------------------------------------------------------------
36// HTTP Signature 41// HTTP Signature
42// ---------------------------------------------------------------------------
37 43
38function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { 44function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean {
39 if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { 45 if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) {
@@ -62,7 +68,9 @@ function parseHTTPSignature (req: Request, clockSkew?: number) {
62 return parsed 68 return parsed
63} 69}
64 70
71// ---------------------------------------------------------------------------
65// JSONLD 72// JSONLD
73// ---------------------------------------------------------------------------
66 74
67function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> { 75function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> {
68 if (signedDocument.signature.type === 'RsaSignature2017') { 76 if (signedDocument.signature.type === 'RsaSignature2017') {
@@ -112,6 +120,8 @@ async function signJsonLDObject <T> (byActor: MActor, data: T) {
112 return Object.assign(data, { signature }) 120 return Object.assign(data, { signature })
113} 121}
114 122
123// ---------------------------------------------------------------------------
124
115function buildDigest (body: any) { 125function buildDigest (body: any) {
116 const rawBody = typeof body === 'string' ? body : JSON.stringify(body) 126 const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
117 127
@@ -119,6 +129,34 @@ function buildDigest (body: any) {
119} 129}
120 130
121// --------------------------------------------------------------------------- 131// ---------------------------------------------------------------------------
132// Encryption
133// ---------------------------------------------------------------------------
134
135async function encrypt (str: string, secret: string) {
136 const iv = await randomBytesPromise(ENCRYPTION.IV)
137
138 const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
139 const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv)
140
141 let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':'
142 encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING)
143 encrypted += cipher.final(ENCRYPTION.ENCODING)
144
145 return encrypted
146}
147
148async function decrypt (encryptedArg: string, secret: string) {
149 const [ ivStr, encryptedStr ] = encryptedArg.split(':')
150
151 const iv = Buffer.from(ivStr, 'hex')
152 const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
153
154 const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv)
155
156 return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8')
157}
158
159// ---------------------------------------------------------------------------
122 160
123export { 161export {
124 isHTTPSignatureDigestValid, 162 isHTTPSignatureDigestValid,
@@ -129,7 +167,10 @@ export {
129 comparePassword, 167 comparePassword,
130 createPrivateAndPublicKeys, 168 createPrivateAndPublicKeys,
131 cryptPassword, 169 cryptPassword,
132 signJsonLDObject 170 signJsonLDObject,
171
172 encrypt,
173 decrypt
133} 174}
134 175
135// --------------------------------------------------------------------------- 176// ---------------------------------------------------------------------------
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index fc4c40787..a2f630953 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -128,14 +128,14 @@ export class YoutubeDLCLI {
128 const data = await this.run({ url, args: completeArgs, processOptions }) 128 const data = await this.run({ url, args: completeArgs, processOptions })
129 if (!data) return undefined 129 if (!data) return undefined
130 130
131 const info = data.map(this.parseInfo) 131 const info = data.map(d => JSON.parse(d))
132 132
133 return info.length === 1 133 return info.length === 1
134 ? info[0] 134 ? info[0]
135 : info 135 : info
136 } 136 }
137 137
138 getListInfo (options: { 138 async getListInfo (options: {
139 url: string 139 url: string
140 latestVideosCount?: number 140 latestVideosCount?: number
141 processOptions: execa.NodeOptions 141 processOptions: execa.NodeOptions
@@ -151,12 +151,17 @@ export class YoutubeDLCLI {
151 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) 151 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
152 } 152 }
153 153
154 return this.getInfo({ 154 const result = await this.getInfo({
155 url: options.url, 155 url: options.url,
156 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), 156 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
157 processOptions: options.processOptions, 157 processOptions: options.processOptions,
158 additionalYoutubeDLArgs 158 additionalYoutubeDLArgs
159 }) 159 })
160
161 if (!result) return result
162 if (!Array.isArray(result)) return [ result ]
163
164 return result
160 } 165 }
161 166
162 async getSubs (options: { 167 async getSubs (options: {
@@ -241,8 +246,4 @@ export class YoutubeDLCLI {
241 246
242 return args 247 return args
243 } 248 }
244
245 private parseInfo (data: string) {
246 return JSON.parse(data)
247 }
248} 249}
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index 42839d1c9..c83fef425 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -42,6 +42,7 @@ function checkConfig () {
42 logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.') 42 logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.')
43 } 43 }
44 44
45 checkSecretsConfig()
45 checkEmailConfig() 46 checkEmailConfig()
46 checkNSFWPolicyConfig() 47 checkNSFWPolicyConfig()
47 checkLocalRedundancyConfig() 48 checkLocalRedundancyConfig()
@@ -103,6 +104,12 @@ export {
103 104
104// --------------------------------------------------------------------------- 105// ---------------------------------------------------------------------------
105 106
107function checkSecretsConfig () {
108 if (!CONFIG.SECRETS.PEERTUBE) {
109 throw new Error('secrets.peertube is missing in config. Generate one using `openssl rand -hex 32`')
110 }
111}
112
106function checkEmailConfig () { 113function checkEmailConfig () {
107 if (!isEmailEnabled()) { 114 if (!isEmailEnabled()) {
108 if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { 115 if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) {
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 3188903be..c9268b156 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -11,12 +11,13 @@ const config: IConfig = require('config')
11function checkMissedConfig () { 11function checkMissedConfig () {
12 const required = [ 'listen.port', 'listen.hostname', 12 const required = [ 'listen.port', 'listen.hostname',
13 'webserver.https', 'webserver.hostname', 'webserver.port', 13 'webserver.https', 'webserver.hostname', 'webserver.port',
14 'secrets.peertube',
14 'trust_proxy', 15 'trust_proxy',
15 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max', 16 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max',
16 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 17 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
17 'email.body.signature', 'email.subject.prefix', 18 'email.body.signature', 'email.subject.prefix',
18 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 19 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
19 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 20 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known',
20 'log.level', 21 'log.level',
21 'user.video_quota', 'user.video_quota_daily', 22 'user.video_quota', 'user.video_quota_daily',
22 'video_channels.max_per_user', 23 'video_channels.max_per_user',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 2c92bea22..a5a0d4e46 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -20,6 +20,9 @@ const CONFIG = {
20 PORT: config.get<number>('listen.port'), 20 PORT: config.get<number>('listen.port'),
21 HOSTNAME: config.get<string>('listen.hostname') 21 HOSTNAME: config.get<string>('listen.hostname')
22 }, 22 },
23 SECRETS: {
24 PEERTUBE: config.get<string>('secrets.peertube')
25 },
23 DATABASE: { 26 DATABASE: {
24 DBNAME: config.has('database.name') ? config.get<string>('database.name') : 'peertube' + config.get<string>('database.suffix'), 27 DBNAME: config.has('database.name') ? config.get<string>('database.name') : 'peertube' + config.get<string>('database.suffix'),
25 HOSTNAME: config.get<string>('database.hostname'), 28 HOSTNAME: config.get<string>('database.hostname'),
@@ -107,7 +110,8 @@ const CONFIG = {
107 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')), 110 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
108 CACHE_DIR: buildPath(config.get<string>('storage.cache')), 111 CACHE_DIR: buildPath(config.get<string>('storage.cache')),
109 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), 112 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
110 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) 113 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
114 WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
111 }, 115 },
112 OBJECT_STORAGE: { 116 OBJECT_STORAGE: {
113 ENABLED: config.get<boolean>('object_storage.enabled'), 117 ENABLED: config.get<boolean>('object_storage.enabled'),
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7039ab457..cab61948a 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,5 +1,5 @@
1import { RepeatOptions } from 'bullmq' 1import { RepeatOptions } from 'bullmq'
2import { randomBytes } from 'crypto' 2import { Encoding, randomBytes } from 'crypto'
3import { invert } from 'lodash' 3import { invert } from 'lodash'
4import { join } from 'path' 4import { join } from 'path'
5import { randomInt, root } from '@shared/core-utils' 5import { randomInt, root } from '@shared/core-utils'
@@ -25,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
25 25
26// --------------------------------------------------------------------------- 26// ---------------------------------------------------------------------------
27 27
28const LAST_MIGRATION_VERSION = 740 28const LAST_MIGRATION_VERSION = 745
29 29
30// --------------------------------------------------------------------------- 30// ---------------------------------------------------------------------------
31 31
@@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = {
116 ACTIVITY_PUB: { 116 ACTIVITY_PUB: {
117 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example 117 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
118 }, 118 },
119 STATS: '4 hours' 119 STATS: '4 hours',
120 WELL_KNOWN: '1 day'
120} 121}
121 122
122// --------------------------------------------------------------------------- 123// ---------------------------------------------------------------------------
@@ -636,9 +637,18 @@ let PRIVATE_RSA_KEY_SIZE = 2048
636// Password encryption 637// Password encryption
637const BCRYPT_SALT_SIZE = 10 638const BCRYPT_SALT_SIZE = 10
638 639
640const ENCRYPTION = {
641 ALGORITHM: 'aes-256-cbc',
642 IV: 16,
643 SALT: 'peertube',
644 ENCODING: 'hex' as Encoding
645}
646
639const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes 647const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes
640const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days 648const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days
641 649
650const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes
651
642const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes 652const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes
643 653
644const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { 654const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
@@ -804,6 +814,10 @@ const REDUNDANCY = {
804} 814}
805 815
806const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) 816const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
817const OTP = {
818 HEADER_NAME: 'x-peertube-otp',
819 HEADER_REQUIRED_VALUE: 'required; app'
820}
807 821
808const ASSETS_PATH = { 822const ASSETS_PATH = {
809 DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'), 823 DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'),
@@ -952,6 +966,7 @@ const VIDEO_FILTERS = {
952export { 966export {
953 WEBSERVER, 967 WEBSERVER,
954 API_VERSION, 968 API_VERSION,
969 ENCRYPTION,
955 VIDEO_LIVE, 970 VIDEO_LIVE,
956 PEERTUBE_VERSION, 971 PEERTUBE_VERSION,
957 LAZY_STATIC_PATHS, 972 LAZY_STATIC_PATHS,
@@ -985,6 +1000,7 @@ export {
985 FOLLOW_STATES, 1000 FOLLOW_STATES,
986 DEFAULT_USER_THEME_NAME, 1001 DEFAULT_USER_THEME_NAME,
987 SERVER_ACTOR_NAME, 1002 SERVER_ACTOR_NAME,
1003 TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
988 PLUGIN_GLOBAL_CSS_FILE_NAME, 1004 PLUGIN_GLOBAL_CSS_FILE_NAME,
989 PLUGIN_GLOBAL_CSS_PATH, 1005 PLUGIN_GLOBAL_CSS_PATH,
990 PRIVATE_RSA_KEY_SIZE, 1006 PRIVATE_RSA_KEY_SIZE,
@@ -1040,6 +1056,7 @@ export {
1040 PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME, 1056 PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME,
1041 ASSETS_PATH, 1057 ASSETS_PATH,
1042 FILES_CONTENT_HASH, 1058 FILES_CONTENT_HASH,
1059 OTP,
1043 loadLanguages, 1060 loadLanguages,
1044 buildLanguages, 1061 buildLanguages,
1045 generateContentHash 1062 generateContentHash
diff --git a/server/initializers/migrations/0745-user-otp.ts b/server/initializers/migrations/0745-user-otp.ts
new file mode 100644
index 000000000..157308ea1
--- /dev/null
+++ b/server/initializers/migrations/0745-user-otp.ts
@@ -0,0 +1,29 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction
5 queryInterface: Sequelize.QueryInterface
6 sequelize: Sequelize.Sequelize
7 db: any
8}): Promise<void> {
9 const { transaction } = utils
10
11 const data = {
12 type: Sequelize.STRING,
13 defaultValue: null,
14 allowNull: true
15 }
16 await utils.queryInterface.addColumn('user', 'otpSecret', data, { transaction })
17
18}
19
20async function down (utils: {
21 queryInterface: Sequelize.QueryInterface
22 transaction: Sequelize.Transaction
23}) {
24}
25
26export {
27 up,
28 down
29}
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 76ed37aae..1e6e8956c 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -109,8 +109,10 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc
109 let video: MVideoAccountLightBlacklistAllFiles 109 let video: MVideoAccountLightBlacklistAllFiles
110 let created: boolean 110 let created: boolean
111 let comment: MCommentOwnerVideo 111 let comment: MCommentOwnerVideo
112
112 try { 113 try {
113 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) 114 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
115 if (!resolveThreadResult) return // Comment not accepted
114 116
115 video = resolveThreadResult.video 117 video = resolveThreadResult.video
116 created = resolveThreadResult.commentCreated 118 created = resolveThreadResult.commentCreated
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 911c7cd30..b65baf0e9 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -4,7 +4,9 @@ import { logger } from '../../helpers/logger'
4import { doJSONRequest } from '../../helpers/requests' 4import { doJSONRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 7import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
8import { isRemoteVideoCommentAccepted } from '../moderation'
9import { Hooks } from '../plugins/hooks'
8import { getOrCreateAPActor } from './actors' 10import { getOrCreateAPActor } from './actors'
9import { checkUrlsSameHost } from './url' 11import { checkUrlsSameHost } from './url'
10import { getOrCreateAPVideo } from './videos' 12import { getOrCreateAPVideo } from './videos'
@@ -103,6 +105,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
103 firstReply.changed('updatedAt', true) 105 firstReply.changed('updatedAt', true)
104 firstReply.Video = video 106 firstReply.Video = video
105 107
108 if (await isRemoteCommentAccepted(firstReply) !== true) {
109 return undefined
110 }
111
106 comments[comments.length - 1] = await firstReply.save() 112 comments[comments.length - 1] = await firstReply.save()
107 113
108 for (let i = comments.length - 2; i >= 0; i--) { 114 for (let i = comments.length - 2; i >= 0; i--) {
@@ -113,6 +119,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
113 comment.changed('updatedAt', true) 119 comment.changed('updatedAt', true)
114 comment.Video = video 120 comment.Video = video
115 121
122 if (await isRemoteCommentAccepted(comment) !== true) {
123 return undefined
124 }
125
116 comments[i] = await comment.save() 126 comments[i] = await comment.save()
117 } 127 }
118 128
@@ -169,3 +179,26 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
169 commentCreated: true 179 commentCreated: true
170 }) 180 })
171} 181}
182
183async function isRemoteCommentAccepted (comment: MComment) {
184 // Already created
185 if (comment.id) return true
186
187 const acceptParameters = {
188 comment
189 }
190
191 const acceptedResult = await Hooks.wrapFun(
192 isRemoteVideoCommentAccepted,
193 acceptParameters,
194 'filter:activity-pub.remote-video-comment.create.accept.result'
195 )
196
197 if (!acceptedResult || acceptedResult.accepted !== true) {
198 logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters })
199
200 return false
201 }
202
203 return true
204}
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts
index fa1887315..35b05ec5a 100644
--- a/server/lib/auth/oauth.ts
+++ b/server/lib/auth/oauth.ts
@@ -9,11 +9,23 @@ import OAuth2Server, {
9 UnsupportedGrantTypeError 9 UnsupportedGrantTypeError
10} from '@node-oauth/oauth2-server' 10} from '@node-oauth/oauth2-server'
11import { randomBytesPromise } from '@server/helpers/core-utils' 11import { randomBytesPromise } from '@server/helpers/core-utils'
12import { isOTPValid } from '@server/helpers/otp'
12import { MOAuthClient } from '@server/types/models' 13import { MOAuthClient } from '@server/types/models'
13import { sha1 } from '@shared/extra-utils' 14import { sha1 } from '@shared/extra-utils'
14import { OAUTH_LIFETIME } from '../../initializers/constants' 15import { HttpStatusCode } from '@shared/models'
16import { OAUTH_LIFETIME, OTP } from '../../initializers/constants'
15import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' 17import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model'
16 18
19class MissingTwoFactorError extends Error {
20 code = HttpStatusCode.UNAUTHORIZED_401
21 name = 'missing_two_factor'
22}
23
24class InvalidTwoFactorError extends Error {
25 code = HttpStatusCode.BAD_REQUEST_400
26 name = 'invalid_two_factor'
27}
28
17/** 29/**
18 * 30 *
19 * Reimplement some functions of OAuth2Server to inject external auth methods 31 * Reimplement some functions of OAuth2Server to inject external auth methods
@@ -94,6 +106,9 @@ function handleOAuthAuthenticate (
94} 106}
95 107
96export { 108export {
109 MissingTwoFactorError,
110 InvalidTwoFactorError,
111
97 handleOAuthToken, 112 handleOAuthToken,
98 handleOAuthAuthenticate 113 handleOAuthAuthenticate
99} 114}
@@ -118,6 +133,16 @@ async function handlePasswordGrant (options: {
118 const user = await getUser(request.body.username, request.body.password, bypassLogin) 133 const user = await getUser(request.body.username, request.body.password, bypassLogin)
119 if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid') 134 if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid')
120 135
136 if (user.otpSecret) {
137 if (!request.headers[OTP.HEADER_NAME]) {
138 throw new MissingTwoFactorError('Missing two factor header')
139 }
140
141 if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) {
142 throw new InvalidTwoFactorError('Invalid two factor header')
143 }
144 }
145
121 const token = await buildToken() 146 const token = await buildToken()
122 147
123 return saveToken(token, client, user, { bypassLogin }) 148 return saveToken(token, client, user, { bypassLogin })
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index a0a5afc0f..a41f1ae48 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -15,7 +15,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
15import { sequelizeTypescript } from '../initializers/database' 15import { sequelizeTypescript } from '../initializers/database'
16import { VideoFileModel } from '../models/video/video-file' 16import { VideoFileModel } from '../models/video/video-file'
17import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 17import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
18import { storeHLSFile } from './object-storage' 18import { storeHLSFileFromFilename } from './object-storage'
19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' 19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
20import { VideoPathManager } from './video-path-manager' 20import { VideoPathManager } from './video-path-manager'
21 21
@@ -95,7 +95,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist
95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
96 96
97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) { 97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
98 playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) 98 playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
99 await remove(masterPlaylistPath) 99 await remove(masterPlaylistPath)
100 } 100 }
101 101
@@ -146,7 +146,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist
146 await outputJSON(outputPath, json) 146 await outputJSON(outputPath, json)
147 147
148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) { 148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
149 playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) 149 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
150 await remove(outputPath) 150 await remove(outputPath)
151 } 151 }
152 152
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 25bdebeea..28c3d325d 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -5,7 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' 8import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' 10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
@@ -88,10 +88,10 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
88 88
89 // Resolution playlist 89 // Resolution playlist
90 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 90 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
91 await storeHLSFile(playlistWithVideo, playlistFilename) 91 await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
92 92
93 // Resolution fragmented file 93 // Resolution fragmented file
94 const fileUrl = await storeHLSFile(playlistWithVideo, file.filename) 94 const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
95 95
96 const oldPath = join(getHLSDirectory(video), file.filename) 96 const oldPath = join(getHLSDirectory(video), file.filename)
97 97
@@ -113,9 +113,9 @@ async function doAfterLastJob (options: {
113 const playlistWithVideo = playlist.withVideo(video) 113 const playlistWithVideo = playlist.withVideo(video)
114 114
115 // Master playlist 115 // Master playlist
116 playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename) 116 playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
117 // Sha256 segments file 117 // Sha256 segments file
118 playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename) 118 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
119 119
120 playlist.storage = VideoStorage.OBJECT_STORAGE 120 playlist.storage = VideoStorage.OBJECT_STORAGE
121 121
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
index 600292844..c3dd8a688 100644
--- a/server/lib/job-queue/handlers/video-channel-import.ts
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel' 5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
7import { MChannelSync } from '@server/types/models' 7import { MChannelSync } from '@server/types/models'
8import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' 8import { VideoChannelImportPayload } from '@shared/models'
9 9
10export async function processVideoChannelImport (job: Job) { 10export async function processVideoChannelImport (job: Job) {
11 const payload = job.data as VideoChannelImportPayload 11 const payload = job.data as VideoChannelImportPayload
@@ -32,17 +32,11 @@ export async function processVideoChannelImport (job: Job) {
32 32
33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) 33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
34 34
35 try { 35 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
36 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) 36
37 37 await synchronizeChannel({
38 await synchronizeChannel({ 38 channel: videoChannel,
39 channel: videoChannel, 39 externalChannelUrl: payload.externalChannelUrl,
40 externalChannelUrl: payload.externalChannelUrl, 40 channelSync
41 channelSync 41 })
42 })
43 } catch (err) {
44 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
45 channelSync.state = VideoChannelSyncState.FAILED
46 await channelSync.save()
47 }
48} 42}
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 8a3ee09a2..7dbffc955 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -4,7 +4,7 @@ import { join } from 'path'
4import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' 4import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg'
5import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 5import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
7import { cleanupPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' 7import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
8import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' 8import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
9import { generateVideoMiniature } from '@server/lib/thumbnail' 9import { generateVideoMiniature } from '@server/lib/thumbnail'
10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' 10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding'
@@ -34,13 +34,13 @@ async function processVideoLiveEnding (job: Job) {
34 const live = await VideoLiveModel.loadByVideoId(payload.videoId) 34 const live = await VideoLiveModel.loadByVideoId(payload.videoId)
35 const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) 35 const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId)
36 36
37 const permanentLive = live.permanentLive
38
39 if (!video || !live || !liveSession) { 37 if (!video || !live || !liveSession) {
40 logError() 38 logError()
41 return 39 return
42 } 40 }
43 41
42 const permanentLive = live.permanentLive
43
44 liveSession.endingProcessed = true 44 liveSession.endingProcessed = true
45 await liveSession.save() 45 await liveSession.save()
46 46
@@ -141,23 +141,22 @@ async function replaceLiveByReplay (options: {
141}) { 141}) {
142 const { video, liveSession, live, permanentLive, replayDirectory } = options 142 const { video, liveSession, live, permanentLive, replayDirectory } = options
143 143
144 await cleanupTMPLiveFiles(video) 144 const videoWithFiles = await VideoModel.loadFull(video.id)
145 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
146
147 await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist)
145 148
146 await live.destroy() 149 await live.destroy()
147 150
148 video.isLive = false 151 videoWithFiles.isLive = false
149 video.waitTranscoding = true 152 videoWithFiles.waitTranscoding = true
150 video.state = VideoState.TO_TRANSCODE 153 videoWithFiles.state = VideoState.TO_TRANSCODE
151 154
152 await video.save() 155 await videoWithFiles.save()
153 156
154 liveSession.replayVideoId = video.id 157 liveSession.replayVideoId = videoWithFiles.id
155 await liveSession.save() 158 await liveSession.save()
156 159
157 // Remove old HLS playlist video files
158 const videoWithFiles = await VideoModel.loadFull(video.id)
159
160 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
161 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) 160 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
162 161
163 // Reset playlist 162 // Reset playlist
@@ -234,7 +233,7 @@ async function cleanupLiveAndFederate (options: {
234 233
235 if (streamingPlaylist) { 234 if (streamingPlaylist) {
236 if (permanentLive) { 235 if (permanentLive) {
237 await cleanupPermanentLive(video, streamingPlaylist) 236 await cleanupAndDestroyPermanentLive(video, streamingPlaylist)
238 } else { 237 } else {
239 await cleanupUnsavedNormalLive(video, streamingPlaylist) 238 await cleanupUnsavedNormalLive(video, streamingPlaylist)
240 } 239 }
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 16715862b..9470b530b 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -21,14 +21,14 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
21import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 21import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
22import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' 22import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models'
23import { pick, wait } from '@shared/core-utils' 23import { pick, wait } from '@shared/core-utils'
24import { LiveVideoError, VideoState, VideoStreamingPlaylistType } from '@shared/models' 24import { LiveVideoError, VideoState, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
25import { federateVideoIfNeeded } from '../activitypub/videos' 25import { federateVideoIfNeeded } from '../activitypub/videos'
26import { JobQueue } from '../job-queue' 26import { JobQueue } from '../job-queue'
27import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' 27import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths'
28import { PeerTubeSocket } from '../peertube-socket' 28import { PeerTubeSocket } from '../peertube-socket'
29import { Hooks } from '../plugins/hooks' 29import { Hooks } from '../plugins/hooks'
30import { LiveQuotaStore } from './live-quota-store' 30import { LiveQuotaStore } from './live-quota-store'
31import { cleanupPermanentLive } from './live-utils' 31import { cleanupAndDestroyPermanentLive } from './live-utils'
32import { MuxingSession } from './shared' 32import { MuxingSession } from './shared'
33 33
34const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') 34const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
@@ -224,7 +224,7 @@ class LiveManager {
224 if (oldStreamingPlaylist) { 224 if (oldStreamingPlaylist) {
225 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) 225 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid)
226 226
227 await cleanupPermanentLive(video, oldStreamingPlaylist) 227 await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
228 } 228 }
229 229
230 this.videoSessions.set(video.id, sessionId) 230 this.videoSessions.set(video.id, sessionId)
@@ -301,7 +301,7 @@ class LiveManager {
301 ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) 301 ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ])
302 }) 302 })
303 303
304 muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) 304 muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
305 305
306 muxingSession.on('bad-socket-health', ({ videoId }) => { 306 muxingSession.on('bad-socket-health', ({ videoId }) => {
307 logger.error( 307 logger.error(
@@ -485,6 +485,10 @@ class LiveManager {
485 485
486 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) 486 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
487 487
488 playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED
489 ? VideoStorage.OBJECT_STORAGE
490 : VideoStorage.FILE_SYSTEM
491
488 return playlist.save() 492 return playlist.save()
489 } 493 }
490 494
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
index 4af6f3ebf..faf03dccf 100644
--- a/server/lib/live/live-segment-sha-store.ts
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -1,62 +1,73 @@
1import { writeJson } from 'fs-extra'
1import { basename } from 'path' 2import { basename } from 'path'
3import { mapToJSON } from '@server/helpers/core-utils'
2import { logger, loggerTagsFactory } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { MStreamingPlaylistVideo } from '@server/types/models'
3import { buildSha256Segment } from '../hls' 6import { buildSha256Segment } from '../hls'
7import { storeHLSFileFromPath } from '../object-storage'
4 8
5const lTags = loggerTagsFactory('live') 9const lTags = loggerTagsFactory('live')
6 10
7class LiveSegmentShaStore { 11class LiveSegmentShaStore {
8 12
9 private static instance: LiveSegmentShaStore 13 private readonly segmentsSha256 = new Map<string, string>()
10 14
11 private readonly segmentsSha256 = new Map<string, Map<string, string>>() 15 private readonly videoUUID: string
12 16 private readonly sha256Path: string
13 private constructor () { 17 private readonly streamingPlaylist: MStreamingPlaylistVideo
18 private readonly sendToObjectStorage: boolean
19
20 constructor (options: {
21 videoUUID: string
22 sha256Path: string
23 streamingPlaylist: MStreamingPlaylistVideo
24 sendToObjectStorage: boolean
25 }) {
26 this.videoUUID = options.videoUUID
27 this.sha256Path = options.sha256Path
28 this.streamingPlaylist = options.streamingPlaylist
29 this.sendToObjectStorage = options.sendToObjectStorage
14 } 30 }
15 31
16 getSegmentsSha256 (videoUUID: string) { 32 async addSegmentSha (segmentPath: string) {
17 return this.segmentsSha256.get(videoUUID) 33 logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID))
18 }
19
20 async addSegmentSha (videoUUID: string, segmentPath: string) {
21 const segmentName = basename(segmentPath)
22 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
23 34
24 const shaResult = await buildSha256Segment(segmentPath) 35 const shaResult = await buildSha256Segment(segmentPath)
25 36
26 if (!this.segmentsSha256.has(videoUUID)) { 37 const segmentName = basename(segmentPath)
27 this.segmentsSha256.set(videoUUID, new Map()) 38 this.segmentsSha256.set(segmentName, shaResult)
28 }
29 39
30 const filesMap = this.segmentsSha256.get(videoUUID) 40 await this.writeToDisk()
31 filesMap.set(segmentName, shaResult)
32 } 41 }
33 42
34 removeSegmentSha (videoUUID: string, segmentPath: string) { 43 async removeSegmentSha (segmentPath: string) {
35 const segmentName = basename(segmentPath) 44 const segmentName = basename(segmentPath)
36 45
37 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) 46 logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID))
38 47
39 const filesMap = this.segmentsSha256.get(videoUUID) 48 if (!this.segmentsSha256.has(segmentName)) {
40 if (!filesMap) { 49 logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID))
41 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
42 return 50 return
43 } 51 }
44 52
45 if (!filesMap.has(segmentName)) { 53 this.segmentsSha256.delete(segmentName)
46 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
47 return
48 }
49 54
50 filesMap.delete(segmentName) 55 await this.writeToDisk()
51 } 56 }
52 57
53 cleanupShaSegments (videoUUID: string) { 58 private async writeToDisk () {
54 this.segmentsSha256.delete(videoUUID) 59 await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256))
55 }
56 60
57 static get Instance () { 61 if (this.sendToObjectStorage) {
58 return this.instance || (this.instance = new this()) 62 const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path)
63
64 if (this.streamingPlaylist.segmentsSha256Url !== url) {
65 this.streamingPlaylist.segmentsSha256Url = url
66 await this.streamingPlaylist.save()
67 }
68 }
59 } 69 }
70
60} 71}
61 72
62export { 73export {
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index bba876642..d2b8e3a55 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,9 +1,10 @@
1import { pathExists, readdir, remove } from 'fs-extra' 1import { pathExists, readdir, remove } from 'fs-extra'
2import { basename, join } from 'path' 2import { basename, join } from 'path'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { MStreamingPlaylist, MVideo } from '@server/types/models' 4import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
5import { VideoStorage } from '@shared/models'
6import { listHLSFileKeysOf, removeHLSFileObjectStorage, removeHLSObjectStorage } from '../object-storage'
5import { getLiveDirectory } from '../paths' 7import { getLiveDirectory } from '../paths'
6import { LiveSegmentShaStore } from './live-segment-sha-store'
7 8
8function buildConcatenatedName (segmentOrPlaylistPath: string) { 9function buildConcatenatedName (segmentOrPlaylistPath: string) {
9 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) 10 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
@@ -11,8 +12,8 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
11 return 'concat-' + num[1] + '.ts' 12 return 'concat-' + num[1] + '.ts'
12} 13}
13 14
14async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 15async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
15 await cleanupTMPLiveFiles(video) 16 await cleanupTMPLiveFiles(video, streamingPlaylist)
16 17
17 await streamingPlaylist.destroy() 18 await streamingPlaylist.destroy()
18} 19}
@@ -20,32 +21,51 @@ async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamin
20async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 21async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
21 const hlsDirectory = getLiveDirectory(video) 22 const hlsDirectory = getLiveDirectory(video)
22 23
24 // We uploaded files to object storage too, remove them
25 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
26 await removeHLSObjectStorage(streamingPlaylist.withVideo(video))
27 }
28
23 await remove(hlsDirectory) 29 await remove(hlsDirectory)
24 30
25 await streamingPlaylist.destroy() 31 await streamingPlaylist.destroy()
32}
26 33
27 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) 34async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
35 await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video))
36
37 await cleanupTMPLiveFilesFromFilesystem(video)
28} 38}
29 39
30async function cleanupTMPLiveFiles (video: MVideo) { 40export {
31 const hlsDirectory = getLiveDirectory(video) 41 cleanupAndDestroyPermanentLive,
42 cleanupUnsavedNormalLive,
43 cleanupTMPLiveFiles,
44 buildConcatenatedName
45}
46
47// ---------------------------------------------------------------------------
32 48
33 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) 49function isTMPLiveFile (name: string) {
50 return name.endsWith('.ts') ||
51 name.endsWith('.m3u8') ||
52 name.endsWith('.json') ||
53 name.endsWith('.mpd') ||
54 name.endsWith('.m4s') ||
55 name.endsWith('.tmp')
56}
57
58async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
59 const hlsDirectory = getLiveDirectory(video)
34 60
35 if (!await pathExists(hlsDirectory)) return 61 if (!await pathExists(hlsDirectory)) return
36 62
37 logger.info('Cleanup TMP live files of %s.', hlsDirectory) 63 logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory)
38 64
39 const files = await readdir(hlsDirectory) 65 const files = await readdir(hlsDirectory)
40 66
41 for (const filename of files) { 67 for (const filename of files) {
42 if ( 68 if (isTMPLiveFile(filename)) {
43 filename.endsWith('.ts') ||
44 filename.endsWith('.m3u8') ||
45 filename.endsWith('.mpd') ||
46 filename.endsWith('.m4s') ||
47 filename.endsWith('.tmp')
48 ) {
49 const p = join(hlsDirectory, filename) 69 const p = join(hlsDirectory, filename)
50 70
51 remove(p) 71 remove(p)
@@ -54,9 +74,14 @@ async function cleanupTMPLiveFiles (video: MVideo) {
54 } 74 }
55} 75}
56 76
57export { 77async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
58 cleanupPermanentLive, 78 if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
59 cleanupUnsavedNormalLive, 79
60 cleanupTMPLiveFiles, 80 const keys = await listHLSFileKeysOf(streamingPlaylist)
61 buildConcatenatedName 81
82 for (const key of keys) {
83 if (isTMPLiveFile(key)) {
84 await removeHLSFileObjectStorage(streamingPlaylist, key)
85 }
86 }
62} 87}
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index 505717dce..4c27d5dd8 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -9,8 +9,10 @@ import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers
9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' 9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
10import { CONFIG } from '@server/initializers/config' 10import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { removeHLSFileObjectStorage, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage'
12import { VideoFileModel } from '@server/models/video/video-file' 13import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 14import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
15import { VideoStorage } from '@shared/models'
14import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' 16import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths'
15import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' 17import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles'
16import { isAbleToUploadVideo } from '../../user' 18import { isAbleToUploadVideo } from '../../user'
@@ -21,7 +23,7 @@ import { buildConcatenatedName } from '../live-utils'
21import memoizee = require('memoizee') 23import memoizee = require('memoizee')
22 24
23interface MuxingSessionEvents { 25interface MuxingSessionEvents {
24 'master-playlist-created': (options: { videoId: number }) => void 26 'live-ready': (options: { videoId: number }) => void
25 27
26 'bad-socket-health': (options: { videoId: number }) => void 28 'bad-socket-health': (options: { videoId: number }) => void
27 'duration-exceeded': (options: { videoId: number }) => void 29 'duration-exceeded': (options: { videoId: number }) => void
@@ -68,12 +70,18 @@ class MuxingSession extends EventEmitter {
68 private readonly outDirectory: string 70 private readonly outDirectory: string
69 private readonly replayDirectory: string 71 private readonly replayDirectory: string
70 72
73 private readonly liveSegmentShaStore: LiveSegmentShaStore
74
71 private readonly lTags: LoggerTagsFn 75 private readonly lTags: LoggerTagsFn
72 76
73 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} 77 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
74 78
75 private tsWatcher: FSWatcher 79 private tsWatcher: FSWatcher
76 private masterWatcher: FSWatcher 80 private masterWatcher: FSWatcher
81 private m3u8Watcher: FSWatcher
82
83 private masterPlaylistCreated = false
84 private liveReady = false
77 85
78 private aborted = false 86 private aborted = false
79 87
@@ -123,6 +131,13 @@ class MuxingSession extends EventEmitter {
123 this.outDirectory = getLiveDirectory(this.videoLive.Video) 131 this.outDirectory = getLiveDirectory(this.videoLive.Video)
124 this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) 132 this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString())
125 133
134 this.liveSegmentShaStore = new LiveSegmentShaStore({
135 videoUUID: this.videoLive.Video.uuid,
136 sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename),
137 streamingPlaylist: this.streamingPlaylist,
138 sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED
139 })
140
126 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) 141 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID)
127 } 142 }
128 143
@@ -159,8 +174,9 @@ class MuxingSession extends EventEmitter {
159 174
160 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) 175 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags())
161 176
162 this.watchTSFiles()
163 this.watchMasterFile() 177 this.watchMasterFile()
178 this.watchTSFiles()
179 this.watchM3U8File()
164 180
165 let ffmpegShellCommand: string 181 let ffmpegShellCommand: string
166 this.ffmpegCommand.on('start', cmdline => { 182 this.ffmpegCommand.on('start', cmdline => {
@@ -219,7 +235,7 @@ class MuxingSession extends EventEmitter {
219 setTimeout(() => { 235 setTimeout(() => {
220 // Wait latest segments generation, and close watchers 236 // Wait latest segments generation, and close watchers
221 237
222 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) 238 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
223 .then(() => { 239 .then(() => {
224 // Process remaining segments hash 240 // Process remaining segments hash
225 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { 241 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
@@ -240,14 +256,41 @@ class MuxingSession extends EventEmitter {
240 private watchMasterFile () { 256 private watchMasterFile () {
241 this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) 257 this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
242 258
243 this.masterWatcher.on('add', () => { 259 this.masterWatcher.on('add', async () => {
244 this.emit('master-playlist-created', { videoId: this.videoId }) 260 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
261 try {
262 const url = await storeHLSFileFromFilename(this.streamingPlaylist, this.streamingPlaylist.playlistFilename)
263
264 this.streamingPlaylist.playlistUrl = url
265 await this.streamingPlaylist.save()
266 } catch (err) {
267 logger.error('Cannot upload live master file to object storage.', { err, ...this.lTags() })
268 }
269 }
270
271 this.masterPlaylistCreated = true
245 272
246 this.masterWatcher.close() 273 this.masterWatcher.close()
247 .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) 274 .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
248 }) 275 })
249 } 276 }
250 277
278 private watchM3U8File () {
279 this.m3u8Watcher = watch(this.outDirectory + '/*.m3u8')
280
281 const onChangeOrAdd = async (m3u8Path: string) => {
282 if (this.streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
283
284 try {
285 await storeHLSFileFromPath(this.streamingPlaylist, m3u8Path)
286 } catch (err) {
287 logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() })
288 }
289 }
290
291 this.m3u8Watcher.on('change', onChangeOrAdd)
292 }
293
251 private watchTSFiles () { 294 private watchTSFiles () {
252 const startStreamDateTime = new Date().getTime() 295 const startStreamDateTime = new Date().getTime()
253 296
@@ -282,7 +325,21 @@ class MuxingSession extends EventEmitter {
282 } 325 }
283 } 326 }
284 327
285 const deleteHandler = (segmentPath: string) => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) 328 const deleteHandler = async (segmentPath: string) => {
329 try {
330 await this.liveSegmentShaStore.removeSegmentSha(segmentPath)
331 } catch (err) {
332 logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() })
333 }
334
335 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
336 try {
337 await removeHLSFileObjectStorage(this.streamingPlaylist, segmentPath)
338 } catch (err) {
339 logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() })
340 }
341 }
342 }
286 343
287 this.tsWatcher.on('add', p => addHandler(p)) 344 this.tsWatcher.on('add', p => addHandler(p))
288 this.tsWatcher.on('unlink', p => deleteHandler(p)) 345 this.tsWatcher.on('unlink', p => deleteHandler(p))
@@ -315,6 +372,7 @@ class MuxingSession extends EventEmitter {
315 extname: '.ts', 372 extname: '.ts',
316 infoHash: null, 373 infoHash: null,
317 fps: this.fps, 374 fps: this.fps,
375 storage: this.streamingPlaylist.storage,
318 videoStreamingPlaylistId: this.streamingPlaylist.id 376 videoStreamingPlaylistId: this.streamingPlaylist.id
319 }) 377 })
320 378
@@ -343,18 +401,36 @@ class MuxingSession extends EventEmitter {
343 } 401 }
344 402
345 private processSegments (segmentPaths: string[]) { 403 private processSegments (segmentPaths: string[]) {
346 mapSeries(segmentPaths, async previousSegment => { 404 mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment))
347 // Add sha hash of previous segments, because ffmpeg should have finished generating them 405 .catch(err => {
348 await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) 406 if (this.aborted) return
407
408 logger.error('Cannot process segments', { err, ...this.lTags() })
409 })
410 }
349 411
350 if (this.saveReplay) { 412 private async processSegment (segmentPath: string) {
351 await this.addSegmentToReplay(previousSegment) 413 // Add sha hash of previous segments, because ffmpeg should have finished generating them
414 await this.liveSegmentShaStore.addSegmentSha(segmentPath)
415
416 if (this.saveReplay) {
417 await this.addSegmentToReplay(segmentPath)
418 }
419
420 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
421 try {
422 await storeHLSFileFromPath(this.streamingPlaylist, segmentPath)
423 } catch (err) {
424 logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })
352 } 425 }
353 }).catch(err => { 426 }
354 if (this.aborted) return
355 427
356 logger.error('Cannot process segments', { err, ...this.lTags() }) 428 // Master playlist and segment JSON file are created, live is ready
357 }) 429 if (this.masterPlaylistCreated && !this.liveReady) {
430 this.liveReady = true
431
432 this.emit('live-ready', { videoId: this.videoId })
433 }
358 } 434 }
359 435
360 private hasClientSocketInBadHealth (sessionId: string) { 436 private hasClientSocketInBadHealth (sessionId: string) {
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index c23f5b6a6..3cc92ca30 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -1,4 +1,4 @@
1import { VideoUploadFile } from 'express' 1import express, { VideoUploadFile } from 'express'
2import { PathLike } from 'fs-extra' 2import { PathLike } from 'fs-extra'
3import { Transaction } from 'sequelize/types' 3import { Transaction } from 'sequelize/types'
4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' 4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
@@ -13,18 +13,15 @@ import {
13 MAbuseFull, 13 MAbuseFull,
14 MAccountDefault, 14 MAccountDefault,
15 MAccountLight, 15 MAccountLight,
16 MComment,
16 MCommentAbuseAccountVideo, 17 MCommentAbuseAccountVideo,
17 MCommentOwnerVideo, 18 MCommentOwnerVideo,
18 MUser, 19 MUser,
19 MVideoAbuseVideoFull, 20 MVideoAbuseVideoFull,
20 MVideoAccountLightBlacklistAllFiles 21 MVideoAccountLightBlacklistAllFiles
21} from '@server/types/models' 22} from '@server/types/models'
22import { ActivityCreate } from '../../shared/models/activitypub'
23import { VideoObject } from '../../shared/models/activitypub/objects'
24import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
25import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' 23import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos'
26import { VideoCommentCreate } from '../../shared/models/videos/comment' 24import { VideoCommentCreate } from '../../shared/models/videos/comment'
27import { ActorModel } from '../models/actor/actor'
28import { UserModel } from '../models/user/user' 25import { UserModel } from '../models/user/user'
29import { VideoModel } from '../models/video/video' 26import { VideoModel } from '../models/video/video'
30import { VideoCommentModel } from '../models/video/video-comment' 27import { VideoCommentModel } from '../models/video/video-comment'
@@ -36,7 +33,9 @@ export type AcceptResult = {
36 errorMessage?: string 33 errorMessage?: string
37} 34}
38 35
39// Can be filtered by plugins 36// ---------------------------------------------------------------------------
37
38// Stub function that can be filtered by plugins
40function isLocalVideoAccepted (object: { 39function isLocalVideoAccepted (object: {
41 videoBody: VideoCreate 40 videoBody: VideoCreate
42 videoFile: VideoUploadFile 41 videoFile: VideoUploadFile
@@ -45,6 +44,9 @@ function isLocalVideoAccepted (object: {
45 return { accepted: true } 44 return { accepted: true }
46} 45}
47 46
47// ---------------------------------------------------------------------------
48
49// Stub function that can be filtered by plugins
48function isLocalLiveVideoAccepted (object: { 50function isLocalLiveVideoAccepted (object: {
49 liveVideoBody: LiveVideoCreate 51 liveVideoBody: LiveVideoCreate
50 user: UserModel 52 user: UserModel
@@ -52,7 +54,11 @@ function isLocalLiveVideoAccepted (object: {
52 return { accepted: true } 54 return { accepted: true }
53} 55}
54 56
57// ---------------------------------------------------------------------------
58
59// Stub function that can be filtered by plugins
55function isLocalVideoThreadAccepted (_object: { 60function isLocalVideoThreadAccepted (_object: {
61 req: express.Request
56 commentBody: VideoCommentCreate 62 commentBody: VideoCommentCreate
57 video: VideoModel 63 video: VideoModel
58 user: UserModel 64 user: UserModel
@@ -60,7 +66,9 @@ function isLocalVideoThreadAccepted (_object: {
60 return { accepted: true } 66 return { accepted: true }
61} 67}
62 68
69// Stub function that can be filtered by plugins
63function isLocalVideoCommentReplyAccepted (_object: { 70function isLocalVideoCommentReplyAccepted (_object: {
71 req: express.Request
64 commentBody: VideoCommentCreate 72 commentBody: VideoCommentCreate
65 parentComment: VideoCommentModel 73 parentComment: VideoCommentModel
66 video: VideoModel 74 video: VideoModel
@@ -69,22 +77,18 @@ function isLocalVideoCommentReplyAccepted (_object: {
69 return { accepted: true } 77 return { accepted: true }
70} 78}
71 79
72function isRemoteVideoAccepted (_object: { 80// ---------------------------------------------------------------------------
73 activity: ActivityCreate
74 videoAP: VideoObject
75 byActor: ActorModel
76}): AcceptResult {
77 return { accepted: true }
78}
79 81
82// Stub function that can be filtered by plugins
80function isRemoteVideoCommentAccepted (_object: { 83function isRemoteVideoCommentAccepted (_object: {
81 activity: ActivityCreate 84 comment: MComment
82 commentAP: VideoCommentObject
83 byActor: ActorModel
84}): AcceptResult { 85}): AcceptResult {
85 return { accepted: true } 86 return { accepted: true }
86} 87}
87 88
89// ---------------------------------------------------------------------------
90
91// Stub function that can be filtered by plugins
88function isPreImportVideoAccepted (object: { 92function isPreImportVideoAccepted (object: {
89 videoImportBody: VideoImportCreate 93 videoImportBody: VideoImportCreate
90 user: MUser 94 user: MUser
@@ -92,6 +96,7 @@ function isPreImportVideoAccepted (object: {
92 return { accepted: true } 96 return { accepted: true }
93} 97}
94 98
99// Stub function that can be filtered by plugins
95function isPostImportVideoAccepted (object: { 100function isPostImportVideoAccepted (object: {
96 videoFilePath: PathLike 101 videoFilePath: PathLike
97 videoFile: VideoFileModel 102 videoFile: VideoFileModel
@@ -100,6 +105,8 @@ function isPostImportVideoAccepted (object: {
100 return { accepted: true } 105 return { accepted: true }
101} 106}
102 107
108// ---------------------------------------------------------------------------
109
103async function createVideoAbuse (options: { 110async function createVideoAbuse (options: {
104 baseAbuse: FilteredModelAttributes<AbuseModel> 111 baseAbuse: FilteredModelAttributes<AbuseModel>
105 videoInstance: MVideoAccountLightBlacklistAllFiles 112 videoInstance: MVideoAccountLightBlacklistAllFiles
@@ -189,12 +196,13 @@ function createAccountAbuse (options: {
189 }) 196 })
190} 197}
191 198
199// ---------------------------------------------------------------------------
200
192export { 201export {
193 isLocalLiveVideoAccepted, 202 isLocalLiveVideoAccepted,
194 203
195 isLocalVideoAccepted, 204 isLocalVideoAccepted,
196 isLocalVideoThreadAccepted, 205 isLocalVideoThreadAccepted,
197 isRemoteVideoAccepted,
198 isRemoteVideoCommentAccepted, 206 isRemoteVideoCommentAccepted,
199 isLocalVideoCommentReplyAccepted, 207 isLocalVideoCommentReplyAccepted,
200 isPreImportVideoAccepted, 208 isPreImportVideoAccepted,
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index 16161362c..c131977e8 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -22,6 +22,24 @@ type BucketInfo = {
22 PREFIX?: string 22 PREFIX?: string
23} 23}
24 24
25async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
26 const s3Client = getClient()
27
28 const commandPrefix = bucketInfo.PREFIX + prefix
29 const listCommand = new ListObjectsV2Command({
30 Bucket: bucketInfo.BUCKET_NAME,
31 Prefix: commandPrefix
32 })
33
34 const listedObjects = await s3Client.send(listCommand)
35
36 if (isArray(listedObjects.Contents) !== true) return []
37
38 return listedObjects.Contents.map(c => c.Key)
39}
40
41// ---------------------------------------------------------------------------
42
25async function storeObject (options: { 43async function storeObject (options: {
26 inputPath: string 44 inputPath: string
27 objectStorageKey: string 45 objectStorageKey: string
@@ -36,6 +54,8 @@ async function storeObject (options: {
36 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) 54 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo })
37} 55}
38 56
57// ---------------------------------------------------------------------------
58
39async function removeObject (filename: string, bucketInfo: BucketInfo) { 59async function removeObject (filename: string, bucketInfo: BucketInfo) {
40 const command = new DeleteObjectCommand({ 60 const command = new DeleteObjectCommand({
41 Bucket: bucketInfo.BUCKET_NAME, 61 Bucket: bucketInfo.BUCKET_NAME,
@@ -89,6 +109,8 @@ async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
89 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) 109 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
90} 110}
91 111
112// ---------------------------------------------------------------------------
113
92async function makeAvailable (options: { 114async function makeAvailable (options: {
93 key: string 115 key: string
94 destination: string 116 destination: string
@@ -122,7 +144,8 @@ export {
122 storeObject, 144 storeObject,
123 removeObject, 145 removeObject,
124 removePrefix, 146 removePrefix,
125 makeAvailable 147 makeAvailable,
148 listKeysOfPrefix
126} 149}
127 150
128// --------------------------------------------------------------------------- 151// ---------------------------------------------------------------------------
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 66e738200..62aae248b 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,19 +1,35 @@
1import { join } from 'path' 1import { basename, join } from 'path'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' 4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 7import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8 8
9function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) { 9function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
10 return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
11}
12
13// ---------------------------------------------------------------------------
14
15function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
10 return storeObject({ 16 return storeObject({
11 inputPath: path ?? join(getHLSDirectory(playlist.Video), filename), 17 inputPath: join(getHLSDirectory(playlist.Video), filename),
12 objectStorageKey: generateHLSObjectStorageKey(playlist, filename), 18 objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
13 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 19 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
14 }) 20 })
15} 21}
16 22
23function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
24 return storeObject({
25 inputPath: path,
26 objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
27 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
28 })
29}
30
31// ---------------------------------------------------------------------------
32
17function storeWebTorrentFile (filename: string) { 33function storeWebTorrentFile (filename: string) {
18 return storeObject({ 34 return storeObject({
19 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), 35 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
@@ -22,6 +38,8 @@ function storeWebTorrentFile (filename: string) {
22 }) 38 })
23} 39}
24 40
41// ---------------------------------------------------------------------------
42
25function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { 43function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
26 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 44 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
27} 45}
@@ -30,10 +48,14 @@ function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename
30 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 48 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
31} 49}
32 50
51// ---------------------------------------------------------------------------
52
33function removeWebTorrentObjectStorage (videoFile: MVideoFile) { 53function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
34 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) 54 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
35} 55}
36 56
57// ---------------------------------------------------------------------------
58
37async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { 59async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
38 const key = generateHLSObjectStorageKey(playlist, filename) 60 const key = generateHLSObjectStorageKey(playlist, filename)
39 61
@@ -62,9 +84,14 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
62 return destination 84 return destination
63} 85}
64 86
87// ---------------------------------------------------------------------------
88
65export { 89export {
90 listHLSFileKeysOf,
91
66 storeWebTorrentFile, 92 storeWebTorrentFile,
67 storeHLSFile, 93 storeHLSFileFromFilename,
94 storeHLSFileFromPath,
68 95
69 removeHLSObjectStorage, 96 removeHLSObjectStorage,
70 removeHLSFileObjectStorage, 97 removeHLSFileObjectStorage,
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts
index 4e799b3d4..7b1def6e3 100644
--- a/server/lib/plugins/plugin-helpers-builder.ts
+++ b/server/lib/plugins/plugin-helpers-builder.ts
@@ -1,4 +1,5 @@
1import express from 'express' 1import express from 'express'
2import { Server } from 'http'
2import { join } from 'path' 3import { join } from 'path'
3import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' 4import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils'
4import { buildLogger } from '@server/helpers/logger' 5import { buildLogger } from '@server/helpers/logger'
@@ -13,15 +14,16 @@ import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
13import { UserModel } from '@server/models/user/user' 14import { UserModel } from '@server/models/user/user'
14import { VideoModel } from '@server/models/video/video' 15import { VideoModel } from '@server/models/video/video'
15import { VideoBlacklistModel } from '@server/models/video/video-blacklist' 16import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
16import { MPlugin } from '@server/types/models' 17import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models'
17import { PeerTubeHelpers } from '@server/types/plugins' 18import { PeerTubeHelpers } from '@server/types/plugins'
18import { VideoBlacklistCreate, VideoStorage } from '@shared/models' 19import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
19import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' 20import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
21import { PeerTubeSocket } from '../peertube-socket'
20import { ServerConfigManager } from '../server-config-manager' 22import { ServerConfigManager } from '../server-config-manager'
21import { blacklistVideo, unblacklistVideo } from '../video-blacklist' 23import { blacklistVideo, unblacklistVideo } from '../video-blacklist'
22import { VideoPathManager } from '../video-path-manager' 24import { VideoPathManager } from '../video-path-manager'
23 25
24function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { 26function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers {
25 const logger = buildPluginLogger(npmName) 27 const logger = buildPluginLogger(npmName)
26 28
27 const database = buildDatabaseHelpers() 29 const database = buildDatabaseHelpers()
@@ -29,12 +31,14 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel
29 31
30 const config = buildConfigHelpers() 32 const config = buildConfigHelpers()
31 33
32 const server = buildServerHelpers() 34 const server = buildServerHelpers(httpServer)
33 35
34 const moderation = buildModerationHelpers() 36 const moderation = buildModerationHelpers()
35 37
36 const plugin = buildPluginRelatedHelpers(pluginModel, npmName) 38 const plugin = buildPluginRelatedHelpers(pluginModel, npmName)
37 39
40 const socket = buildSocketHelpers()
41
38 const user = buildUserHelpers() 42 const user = buildUserHelpers()
39 43
40 return { 44 return {
@@ -45,6 +49,7 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel
45 moderation, 49 moderation,
46 plugin, 50 plugin,
47 server, 51 server,
52 socket,
48 user 53 user
49 } 54 }
50} 55}
@@ -65,8 +70,10 @@ function buildDatabaseHelpers () {
65 } 70 }
66} 71}
67 72
68function buildServerHelpers () { 73function buildServerHelpers (httpServer: Server) {
69 return { 74 return {
75 getHTTPServer: () => httpServer,
76
70 getServerActor: () => getServerActor() 77 getServerActor: () => getServerActor()
71 } 78 }
72} 79}
@@ -214,10 +221,23 @@ function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) {
214 221
215 getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, 222 getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`,
216 223
224 getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`,
225
217 getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) 226 getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName)
218 } 227 }
219} 228}
220 229
230function buildSocketHelpers () {
231 return {
232 sendNotification: (userId: number, notification: UserNotificationModelForApi) => {
233 PeerTubeSocket.Instance.sendNotification(userId, notification)
234 },
235 sendVideoLiveNewState: (video: MVideo) => {
236 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
237 }
238 }
239}
240
221function buildUserHelpers () { 241function buildUserHelpers () {
222 return { 242 return {
223 loadById: (id: number) => { 243 loadById: (id: number) => {
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index a46b97fa4..c4d9b6574 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -1,6 +1,7 @@
1import express from 'express' 1import express from 'express'
2import { createReadStream, createWriteStream } from 'fs' 2import { createReadStream, createWriteStream } from 'fs'
3import { ensureDir, outputFile, readJSON } from 'fs-extra' 3import { ensureDir, outputFile, readJSON } from 'fs-extra'
4import { Server } from 'http'
4import { basename, join } from 'path' 5import { basename, join } from 'path'
5import { decachePlugin } from '@server/helpers/decache' 6import { decachePlugin } from '@server/helpers/decache'
6import { ApplicationModel } from '@server/models/application/application' 7import { ApplicationModel } from '@server/models/application/application'
@@ -67,9 +68,37 @@ export class PluginManager implements ServerHook {
67 private hooks: { [name: string]: HookInformationValue[] } = {} 68 private hooks: { [name: string]: HookInformationValue[] } = {}
68 private translations: PluginLocalesTranslations = {} 69 private translations: PluginLocalesTranslations = {}
69 70
71 private server: Server
72
70 private constructor () { 73 private constructor () {
71 } 74 }
72 75
76 init (server: Server) {
77 this.server = server
78 }
79
80 registerWebSocketRouter () {
81 this.server.on('upgrade', (request, socket, head) => {
82 const url = request.url
83
84 const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`)
85 if (!matched) return
86
87 const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN)
88 const subRoute = matched[3]
89
90 const result = this.getRegisteredPluginOrTheme(npmName)
91 if (!result) return
92
93 const routes = result.registerHelpers.getWebSocketRoutes()
94
95 const wss = routes.find(r => r.route.startsWith(subRoute))
96 if (!wss) return
97
98 wss.handler(request, socket, head)
99 })
100 }
101
73 // ###################### Getters ###################### 102 // ###################### Getters ######################
74 103
75 isRegistered (npmName: string) { 104 isRegistered (npmName: string) {
@@ -581,7 +610,7 @@ export class PluginManager implements ServerHook {
581 }) 610 })
582 } 611 }
583 612
584 const registerHelpers = new RegisterHelpers(npmName, plugin, onHookAdded.bind(this)) 613 const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this))
585 614
586 return { 615 return {
587 registerStore: registerHelpers, 616 registerStore: registerHelpers,
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts
index f4d405676..1aaef3606 100644
--- a/server/lib/plugins/register-helpers.ts
+++ b/server/lib/plugins/register-helpers.ts
@@ -1,4 +1,5 @@
1import express from 'express' 1import express from 'express'
2import { Server } from 'http'
2import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
3import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' 4import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth'
4import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' 5import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory'
@@ -8,7 +9,8 @@ import {
8 RegisterServerAuthExternalResult, 9 RegisterServerAuthExternalResult,
9 RegisterServerAuthPassOptions, 10 RegisterServerAuthPassOptions,
10 RegisterServerExternalAuthenticatedResult, 11 RegisterServerExternalAuthenticatedResult,
11 RegisterServerOptions 12 RegisterServerOptions,
13 RegisterServerWebSocketRouteOptions
12} from '@server/types/plugins' 14} from '@server/types/plugins'
13import { 15import {
14 EncoderOptionsBuilder, 16 EncoderOptionsBuilder,
@@ -49,12 +51,15 @@ export class RegisterHelpers {
49 51
50 private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] 52 private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = []
51 53
54 private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = []
55
52 private readonly router: express.Router 56 private readonly router: express.Router
53 private readonly videoConstantManagerFactory: VideoConstantManagerFactory 57 private readonly videoConstantManagerFactory: VideoConstantManagerFactory
54 58
55 constructor ( 59 constructor (
56 private readonly npmName: string, 60 private readonly npmName: string,
57 private readonly plugin: PluginModel, 61 private readonly plugin: PluginModel,
62 private readonly server: Server,
58 private readonly onHookAdded: (options: RegisterServerHookOptions) => void 63 private readonly onHookAdded: (options: RegisterServerHookOptions) => void
59 ) { 64 ) {
60 this.router = express.Router() 65 this.router = express.Router()
@@ -66,6 +71,7 @@ export class RegisterHelpers {
66 const registerSetting = this.buildRegisterSetting() 71 const registerSetting = this.buildRegisterSetting()
67 72
68 const getRouter = this.buildGetRouter() 73 const getRouter = this.buildGetRouter()
74 const registerWebSocketRoute = this.buildRegisterWebSocketRoute()
69 75
70 const settingsManager = this.buildSettingsManager() 76 const settingsManager = this.buildSettingsManager()
71 const storageManager = this.buildStorageManager() 77 const storageManager = this.buildStorageManager()
@@ -85,13 +91,14 @@ export class RegisterHelpers {
85 const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() 91 const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth()
86 const unregisterExternalAuth = this.buildUnregisterExternalAuth() 92 const unregisterExternalAuth = this.buildUnregisterExternalAuth()
87 93
88 const peertubeHelpers = buildPluginHelpers(this.plugin, this.npmName) 94 const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName)
89 95
90 return { 96 return {
91 registerHook, 97 registerHook,
92 registerSetting, 98 registerSetting,
93 99
94 getRouter, 100 getRouter,
101 registerWebSocketRoute,
95 102
96 settingsManager, 103 settingsManager,
97 storageManager, 104 storageManager,
@@ -180,10 +187,20 @@ export class RegisterHelpers {
180 return this.onSettingsChangeCallbacks 187 return this.onSettingsChangeCallbacks
181 } 188 }
182 189
190 getWebSocketRoutes () {
191 return this.webSocketRoutes
192 }
193
183 private buildGetRouter () { 194 private buildGetRouter () {
184 return () => this.router 195 return () => this.router
185 } 196 }
186 197
198 private buildRegisterWebSocketRoute () {
199 return (options: RegisterServerWebSocketRouteOptions) => {
200 this.webSocketRoutes.push(options)
201 }
202 }
203
187 private buildRegisterSetting () { 204 private buildRegisterSetting () {
188 return (options: RegisterServerSettingOptions) => { 205 return (options: RegisterServerSettingOptions) => {
189 this.settings.push(options) 206 this.settings.push(options)
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 9b3c72300..b7523492a 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -9,6 +9,7 @@ import {
9 CONTACT_FORM_LIFETIME, 9 CONTACT_FORM_LIFETIME,
10 RESUMABLE_UPLOAD_SESSION_LIFETIME, 10 RESUMABLE_UPLOAD_SESSION_LIFETIME,
11 TRACKER_RATE_LIMITS, 11 TRACKER_RATE_LIMITS,
12 TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
12 USER_EMAIL_VERIFY_LIFETIME, 13 USER_EMAIL_VERIFY_LIFETIME,
13 USER_PASSWORD_CREATE_LIFETIME, 14 USER_PASSWORD_CREATE_LIFETIME,
14 USER_PASSWORD_RESET_LIFETIME, 15 USER_PASSWORD_RESET_LIFETIME,
@@ -108,10 +109,24 @@ class Redis {
108 return this.removeValue(this.generateResetPasswordKey(userId)) 109 return this.removeValue(this.generateResetPasswordKey(userId))
109 } 110 }
110 111
111 async getResetPasswordLink (userId: number) { 112 async getResetPasswordVerificationString (userId: number) {
112 return this.getValue(this.generateResetPasswordKey(userId)) 113 return this.getValue(this.generateResetPasswordKey(userId))
113 } 114 }
114 115
116 /* ************ Two factor auth request ************ */
117
118 async setTwoFactorRequest (userId: number, otpSecret: string) {
119 const requestToken = await generateRandomString(32)
120
121 await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME)
122
123 return requestToken
124 }
125
126 async getTwoFactorRequestToken (userId: number, requestToken: string) {
127 return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken))
128 }
129
115 /* ************ Email verification ************ */ 130 /* ************ Email verification ************ */
116 131
117 async setVerifyEmailVerificationString (userId: number) { 132 async setVerifyEmailVerificationString (userId: number) {
@@ -342,6 +357,10 @@ class Redis {
342 return 'reset-password-' + userId 357 return 'reset-password-' + userId
343 } 358 }
344 359
360 private generateTwoFactorRequestKey (userId: number, token: string) {
361 return 'two-factor-request-' + userId + '-' + token
362 }
363
345 private generateVerifyEmailKey (userId: number) { 364 private generateVerifyEmailKey (userId: number) {
346 return 'verify-email-' + userId 365 return 'verify-email-' + userId
347 } 366 }
@@ -391,8 +410,8 @@ class Redis {
391 return JSON.parse(value) 410 return JSON.parse(value)
392 } 411 }
393 412
394 private setObject (key: string, value: { [ id: string ]: number | string }) { 413 private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) {
395 return this.setValue(key, JSON.stringify(value)) 414 return this.setValue(key, JSON.stringify(value), expirationMilliseconds)
396 } 415 }
397 416
398 private async setValue (key: string, value: string, expirationMilliseconds?: number) { 417 private async setValue (key: string, value: string, expirationMilliseconds?: number) {
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
index a527f68b5..efb957fac 100644
--- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { VideoChannelModel } from '@server/models/video/video-channel' 3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
5import { VideoChannelSyncState } from '@shared/models'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { synchronizeChannel } from '../sync-channel' 6import { synchronizeChannel } from '../sync-channel'
8import { AbstractScheduler } from './abstract-scheduler' 7import { AbstractScheduler } from './abstract-scheduler'
@@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
28 for (const sync of channelSyncs) { 27 for (const sync of channelSyncs) {
29 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) 28 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
30 29
31 try { 30 logger.info(
32 logger.info( 31 'Creating video import jobs for "%s" sync with external channel "%s"',
33 'Creating video import jobs for "%s" sync with external channel "%s"', 32 channel.Actor.preferredUsername, sync.externalChannelUrl
34 channel.Actor.preferredUsername, sync.externalChannelUrl 33 )
35 ) 34
36 35 const onlyAfter = sync.lastSyncAt || sync.createdAt
37 const onlyAfter = sync.lastSyncAt || sync.createdAt 36
38 37 await synchronizeChannel({
39 await synchronizeChannel({ 38 channel,
40 channel, 39 externalChannelUrl: sync.externalChannelUrl,
41 externalChannelUrl: sync.externalChannelUrl, 40 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
42 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, 41 channelSync: sync,
43 channelSync: sync, 42 onlyAfter
44 onlyAfter 43 })
45 })
46 } catch (err) {
47 logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
48 sync.state = VideoChannelSyncState.FAILED
49 await sync.save()
50 }
51 } 44 }
52 } 45 }
53 46
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index f91599c14..35af91429 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -24,56 +24,62 @@ export async function synchronizeChannel (options: {
24 await channelSync.save() 24 await channelSync.save()
25 } 25 }
26 26
27 const user = await UserModel.loadByChannelActorId(channel.actorId) 27 try {
28 const youtubeDL = new YoutubeDLWrapper( 28 const user = await UserModel.loadByChannelActorId(channel.actorId)
29 externalChannelUrl, 29 const youtubeDL = new YoutubeDLWrapper(
30 ServerConfigManager.Instance.getEnabledResolutions('vod'), 30 externalChannelUrl,
31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 31 ServerConfigManager.Instance.getEnabledResolutions('vod'),
32 ) 32 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
33 33 )
34 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
35
36 logger.info(
37 'Fetched %d candidate URLs for sync channel %s.',
38 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
39 )
40
41 if (targetUrls.length === 0) {
42 if (channelSync) {
43 channelSync.state = VideoChannelSyncState.SYNCED
44 await channelSync.save()
45 }
46
47 return
48 }
49 34
50 const children: CreateJobArgument[] = [] 35 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
51 36
52 for (const targetUrl of targetUrls) { 37 logger.info(
53 if (await skipImport(channel, targetUrl, onlyAfter)) continue 38 'Fetched %d candidate URLs for sync channel %s.',
39 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
40 )
54 41
55 const { job } = await buildYoutubeDLImport({ 42 if (targetUrls.length === 0) {
56 user, 43 if (channelSync) {
57 channel, 44 channelSync.state = VideoChannelSyncState.SYNCED
58 targetUrl, 45 await channelSync.save()
59 channelSync,
60 importDataOverride: {
61 privacy: VideoPrivacy.PUBLIC
62 } 46 }
63 })
64 47
65 children.push(job) 48 return
66 } 49 }
50
51 const children: CreateJobArgument[] = []
52
53 for (const targetUrl of targetUrls) {
54 if (await skipImport(channel, targetUrl, onlyAfter)) continue
67 55
68 // Will update the channel sync status 56 const { job } = await buildYoutubeDLImport({
69 const parent: CreateJobArgument = { 57 user,
70 type: 'after-video-channel-import', 58 channel,
71 payload: { 59 targetUrl,
72 channelSyncId: channelSync?.id 60 channelSync,
61 importDataOverride: {
62 privacy: VideoPrivacy.PUBLIC
63 }
64 })
65
66 children.push(job)
73 } 67 }
74 }
75 68
76 await JobQueue.Instance.createJobWithChildren(parent, children) 69 // Will update the channel sync status
70 const parent: CreateJobArgument = {
71 type: 'after-video-channel-import',
72 payload: {
73 channelSyncId: channelSync?.id
74 }
75 }
76
77 await JobQueue.Instance.createJobWithChildren(parent, children)
78 } catch (err) {
79 logger.error(`Failed to import channel ${channel.name}`, { err })
80 channelSync.state = VideoChannelSyncState.FAILED
81 await channelSync.save()
82 }
77} 83}
78 84
79// --------------------------------------------------------------------------- 85// ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/shared/index.ts b/server/middlewares/validators/shared/index.ts
index bbd03b248..de98cd442 100644
--- a/server/middlewares/validators/shared/index.ts
+++ b/server/middlewares/validators/shared/index.ts
@@ -1,5 +1,6 @@
1export * from './abuses' 1export * from './abuses'
2export * from './accounts' 2export * from './accounts'
3export * from './users'
3export * from './utils' 4export * from './utils'
4export * from './video-blacklists' 5export * from './video-blacklists'
5export * from './video-captions' 6export * from './video-captions'
diff --git a/server/middlewares/validators/shared/users.ts b/server/middlewares/validators/shared/users.ts
new file mode 100644
index 000000000..fbaa7db0e
--- /dev/null
+++ b/server/middlewares/validators/shared/users.ts
@@ -0,0 +1,62 @@
1import express from 'express'
2import { ActorModel } from '@server/models/actor/actor'
3import { UserModel } from '@server/models/user/user'
4import { MUserDefault } from '@server/types/models'
5import { HttpStatusCode } from '@shared/models'
6
7function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
8 const id = parseInt(idArg + '', 10)
9 return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res)
10}
11
12function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {
13 return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse)
14}
15
16async function checkUserNameOrEmailDoesNotAlreadyExist (username: string, email: string, res: express.Response) {
17 const user = await UserModel.loadByUsernameOrEmail(username, email)
18
19 if (user) {
20 res.fail({
21 status: HttpStatusCode.CONFLICT_409,
22 message: 'User with this username or email already exists.'
23 })
24 return false
25 }
26
27 const actor = await ActorModel.loadLocalByName(username)
28 if (actor) {
29 res.fail({
30 status: HttpStatusCode.CONFLICT_409,
31 message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.'
32 })
33 return false
34 }
35
36 return true
37}
38
39async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) {
40 const user = await finder()
41
42 if (!user) {
43 if (abortResponse === true) {
44 res.fail({
45 status: HttpStatusCode.NOT_FOUND_404,
46 message: 'User not found'
47 })
48 }
49
50 return false
51 }
52
53 res.locals.user = user
54 return true
55}
56
57export {
58 checkUserIdExist,
59 checkUserEmailExist,
60 checkUserNameOrEmailDoesNotAlreadyExist,
61 checkUserExist
62}
diff --git a/server/middlewares/validators/two-factor.ts b/server/middlewares/validators/two-factor.ts
new file mode 100644
index 000000000..106b579b5
--- /dev/null
+++ b/server/middlewares/validators/two-factor.ts
@@ -0,0 +1,81 @@
1import express from 'express'
2import { body, param } from 'express-validator'
3import { HttpStatusCode, UserRight } from '@shared/models'
4import { exists, isIdValid } from '../../helpers/custom-validators/misc'
5import { areValidationErrors, checkUserIdExist } from './shared'
6
7const requestOrConfirmTwoFactorValidator = [
8 param('id').custom(isIdValid),
9
10 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
11 if (areValidationErrors(req, res)) return
12
13 if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return
14
15 if (res.locals.user.otpSecret) {
16 return res.fail({
17 status: HttpStatusCode.BAD_REQUEST_400,
18 message: `Two factor is already enabled.`
19 })
20 }
21
22 return next()
23 }
24]
25
26const confirmTwoFactorValidator = [
27 body('requestToken').custom(exists),
28 body('otpToken').custom(exists),
29
30 (req: express.Request, res: express.Response, next: express.NextFunction) => {
31 if (areValidationErrors(req, res)) return
32
33 return next()
34 }
35]
36
37const disableTwoFactorValidator = [
38 param('id').custom(isIdValid),
39
40 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
41 if (areValidationErrors(req, res)) return
42
43 if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return
44
45 if (!res.locals.user.otpSecret) {
46 return res.fail({
47 status: HttpStatusCode.BAD_REQUEST_400,
48 message: `Two factor is already disabled.`
49 })
50 }
51
52 return next()
53 }
54]
55
56// ---------------------------------------------------------------------------
57
58export {
59 requestOrConfirmTwoFactorValidator,
60 confirmTwoFactorValidator,
61 disableTwoFactorValidator
62}
63
64// ---------------------------------------------------------------------------
65
66async function checkCanEnableOrDisableTwoFactor (userId: number | string, res: express.Response) {
67 const authUser = res.locals.oauth.token.user
68
69 if (!await checkUserIdExist(userId, res)) return
70
71 if (res.locals.user.id !== authUser.id && authUser.hasRight(UserRight.MANAGE_USERS) !== true) {
72 res.fail({
73 status: HttpStatusCode.FORBIDDEN_403,
74 message: `User ${authUser.username} does not have right to change two factor setting of this user.`
75 })
76
77 return false
78 }
79
80 return true
81}
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index 2de5265fb..055af3b64 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -1,9 +1,8 @@
1import express from 'express' 1import express from 'express'
2import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { Hooks } from '@server/lib/plugins/hooks' 3import { Hooks } from '@server/lib/plugins/hooks'
4import { MUserDefault } from '@server/types/models'
5import { HttpStatusCode, UserRegister, UserRight, UserRole } from '@shared/models' 4import { HttpStatusCode, UserRegister, UserRight, UserRole } from '@shared/models'
6import { isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' 5import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
7import { isThemeNameValid } from '../../helpers/custom-validators/plugins' 6import { isThemeNameValid } from '../../helpers/custom-validators/plugins'
8import { 7import {
9 isUserAdminFlagsValid, 8 isUserAdminFlagsValid,
@@ -30,8 +29,15 @@ import { isThemeRegistered } from '../../lib/plugins/theme-utils'
30import { Redis } from '../../lib/redis' 29import { Redis } from '../../lib/redis'
31import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../lib/signup' 30import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../lib/signup'
32import { ActorModel } from '../../models/actor/actor' 31import { ActorModel } from '../../models/actor/actor'
33import { UserModel } from '../../models/user/user' 32import {
34import { areValidationErrors, doesVideoChannelIdExist, doesVideoExist, isValidVideoIdParam } from './shared' 33 areValidationErrors,
34 checkUserEmailExist,
35 checkUserIdExist,
36 checkUserNameOrEmailDoesNotAlreadyExist,
37 doesVideoChannelIdExist,
38 doesVideoExist,
39 isValidVideoIdParam
40} from './shared'
35 41
36const usersListValidator = [ 42const usersListValidator = [
37 query('blocked') 43 query('blocked')
@@ -411,6 +417,13 @@ const usersAskResetPasswordValidator = [
411 return res.status(HttpStatusCode.NO_CONTENT_204).end() 417 return res.status(HttpStatusCode.NO_CONTENT_204).end()
412 } 418 }
413 419
420 if (res.locals.user.pluginAuth) {
421 return res.fail({
422 status: HttpStatusCode.CONFLICT_409,
423 message: 'Cannot recover password of a user that uses a plugin authentication.'
424 })
425 }
426
414 return next() 427 return next()
415 } 428 }
416] 429]
@@ -428,7 +441,7 @@ const usersResetPasswordValidator = [
428 if (!await checkUserIdExist(req.params.id, res)) return 441 if (!await checkUserIdExist(req.params.id, res)) return
429 442
430 const user = res.locals.user 443 const user = res.locals.user
431 const redisVerificationString = await Redis.Instance.getResetPasswordLink(user.id) 444 const redisVerificationString = await Redis.Instance.getResetPasswordVerificationString(user.id)
432 445
433 if (redisVerificationString !== req.body.verificationString) { 446 if (redisVerificationString !== req.body.verificationString) {
434 return res.fail({ 447 return res.fail({
@@ -454,6 +467,13 @@ const usersAskSendVerifyEmailValidator = [
454 return res.status(HttpStatusCode.NO_CONTENT_204).end() 467 return res.status(HttpStatusCode.NO_CONTENT_204).end()
455 } 468 }
456 469
470 if (res.locals.user.pluginAuth) {
471 return res.fail({
472 status: HttpStatusCode.CONFLICT_409,
473 message: 'Cannot ask verification email of a user that uses a plugin authentication.'
474 })
475 }
476
457 return next() 477 return next()
458 } 478 }
459] 479]
@@ -486,6 +506,41 @@ const usersVerifyEmailValidator = [
486 } 506 }
487] 507]
488 508
509const usersCheckCurrentPasswordFactory = (targetUserIdGetter: (req: express.Request) => number | string) => {
510 return [
511 body('currentPassword').optional().custom(exists),
512
513 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
514 if (areValidationErrors(req, res)) return
515
516 const user = res.locals.oauth.token.User
517 const isAdminOrModerator = user.role === UserRole.ADMINISTRATOR || user.role === UserRole.MODERATOR
518 const targetUserId = parseInt(targetUserIdGetter(req) + '')
519
520 // Admin/moderator action on another user, skip the password check
521 if (isAdminOrModerator && targetUserId !== user.id) {
522 return next()
523 }
524
525 if (!req.body.currentPassword) {
526 return res.fail({
527 status: HttpStatusCode.BAD_REQUEST_400,
528 message: 'currentPassword is missing'
529 })
530 }
531
532 if (await user.isPasswordMatch(req.body.currentPassword) !== true) {
533 return res.fail({
534 status: HttpStatusCode.FORBIDDEN_403,
535 message: 'currentPassword is invalid.'
536 })
537 }
538
539 return next()
540 }
541 ]
542}
543
489const userAutocompleteValidator = [ 544const userAutocompleteValidator = [
490 param('search') 545 param('search')
491 .isString() 546 .isString()
@@ -553,6 +608,7 @@ export {
553 usersUpdateValidator, 608 usersUpdateValidator,
554 usersUpdateMeValidator, 609 usersUpdateMeValidator,
555 usersVideoRatingValidator, 610 usersVideoRatingValidator,
611 usersCheckCurrentPasswordFactory,
556 ensureUserRegistrationAllowed, 612 ensureUserRegistrationAllowed,
557 ensureUserRegistrationAllowedForIP, 613 ensureUserRegistrationAllowedForIP,
558 usersGetValidator, 614 usersGetValidator,
@@ -566,55 +622,3 @@ export {
566 ensureCanModerateUser, 622 ensureCanModerateUser,
567 ensureCanManageChannelOrAccount 623 ensureCanManageChannelOrAccount
568} 624}
569
570// ---------------------------------------------------------------------------
571
572function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
573 const id = parseInt(idArg + '', 10)
574 return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res)
575}
576
577function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {
578 return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse)
579}
580
581async function checkUserNameOrEmailDoesNotAlreadyExist (username: string, email: string, res: express.Response) {
582 const user = await UserModel.loadByUsernameOrEmail(username, email)
583
584 if (user) {
585 res.fail({
586 status: HttpStatusCode.CONFLICT_409,
587 message: 'User with this username or email already exists.'
588 })
589 return false
590 }
591
592 const actor = await ActorModel.loadLocalByName(username)
593 if (actor) {
594 res.fail({
595 status: HttpStatusCode.CONFLICT_409,
596 message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.'
597 })
598 return false
599 }
600
601 return true
602}
603
604async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) {
605 const user = await finder()
606
607 if (!user) {
608 if (abortResponse === true) {
609 res.fail({
610 status: HttpStatusCode.NOT_FOUND_404,
611 message: 'User not found'
612 })
613 }
614
615 return false
616 }
617
618 res.locals.user = user
619 return true
620}
diff --git a/server/middlewares/validators/videos/video-comments.ts b/server/middlewares/validators/videos/video-comments.ts
index 69062701b..133feb7bd 100644
--- a/server/middlewares/validators/videos/video-comments.ts
+++ b/server/middlewares/validators/videos/video-comments.ts
@@ -208,7 +208,8 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon
208 const acceptParameters = { 208 const acceptParameters = {
209 video, 209 video,
210 commentBody: req.body, 210 commentBody: req.body,
211 user: res.locals.oauth.token.User 211 user: res.locals.oauth.token.User,
212 req
212 } 213 }
213 214
214 let acceptedResult: AcceptResult 215 let acceptedResult: AcceptResult
@@ -234,7 +235,7 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon
234 235
235 res.fail({ 236 res.fail({
236 status: HttpStatusCode.FORBIDDEN_403, 237 status: HttpStatusCode.FORBIDDEN_403,
237 message: acceptedResult?.errorMessage || 'Refused local comment' 238 message: acceptedResult?.errorMessage || 'Comment has been rejected.'
238 }) 239 })
239 return false 240 return false
240 } 241 }
diff --git a/server/models/user/user.ts b/server/models/user/user.ts
index 1a7c84390..34329580b 100644
--- a/server/models/user/user.ts
+++ b/server/models/user/user.ts
@@ -403,6 +403,11 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> {
403 @Column 403 @Column
404 lastLoginDate: Date 404 lastLoginDate: Date
405 405
406 @AllowNull(true)
407 @Default(null)
408 @Column
409 otpSecret: string
410
406 @CreatedAt 411 @CreatedAt
407 createdAt: Date 412 createdAt: Date
408 413
@@ -935,7 +940,9 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> {
935 940
936 pluginAuth: this.pluginAuth, 941 pluginAuth: this.pluginAuth,
937 942
938 lastLoginDate: this.lastLoginDate 943 lastLoginDate: this.lastLoginDate,
944
945 twoFactorEnabled: !!this.otpSecret
939 } 946 }
940 947
941 if (parameters.withAdminFlags) { 948 if (parameters.withAdminFlags) {
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 7497addf1..740f6b5c6 100644
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -84,7 +84,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
84 static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { 84 static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> {
85 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } 85 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
86 86
87 const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` 87 const result = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
88 UPDATE 88 UPDATE
89 "videoJobInfo" 89 "videoJobInfo"
90 SET 90 SET
@@ -97,7 +97,9 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
97 "${column}"; 97 "${column}";
98 `, options) 98 `, options)
99 99
100 return pendingMove 100 if (result.length === 0) return undefined
101
102 return result[0].pendingMove
101 } 103 }
102 104
103 static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> { 105 static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index f587989dc..2b6771f27 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -245,21 +245,25 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
245 } 245 }
246 246
247 getMasterPlaylistUrl (video: MVideo) { 247 getMasterPlaylistUrl (video: MVideo) {
248 if (this.storage === VideoStorage.OBJECT_STORAGE) { 248 if (video.isOwned()) {
249 return getHLSPublicFileUrl(this.playlistUrl) 249 if (this.storage === VideoStorage.OBJECT_STORAGE) {
250 } 250 return getHLSPublicFileUrl(this.playlistUrl)
251 }
251 252
252 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) 253 return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
254 }
253 255
254 return this.playlistUrl 256 return this.playlistUrl
255 } 257 }
256 258
257 getSha256SegmentsUrl (video: MVideo) { 259 getSha256SegmentsUrl (video: MVideo) {
258 if (this.storage === VideoStorage.OBJECT_STORAGE) { 260 if (video.isOwned()) {
259 return getHLSPublicFileUrl(this.segmentsSha256Url) 261 if (this.storage === VideoStorage.OBJECT_STORAGE) {
260 } 262 return getHLSPublicFileUrl(this.segmentsSha256Url)
263 }
261 264
262 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) 265 return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid)
266 }
263 267
264 return this.segmentsSha256Url 268 return this.segmentsSha256Url
265 } 269 }
@@ -287,9 +291,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
287 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) 291 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
288 } 292 }
289 293
290 private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) { 294 private getSha256SegmentsStaticPath (videoUUID: string) {
291 if (isLive) return join('/live', 'segments-sha256', videoUUID)
292
293 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename) 295 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
294 } 296 }
295} 297}
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index cd7a38459..33dc8fb76 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -2,6 +2,7 @@ import './abuses'
2import './accounts' 2import './accounts'
3import './blocklist' 3import './blocklist'
4import './bulk' 4import './bulk'
5import './channel-import-videos'
5import './config' 6import './config'
6import './contact-form' 7import './contact-form'
7import './custom-pages' 8import './custom-pages'
@@ -17,6 +18,7 @@ import './redundancy'
17import './search' 18import './search'
18import './services' 19import './services'
19import './transcoding' 20import './transcoding'
21import './two-factor'
20import './upload-quota' 22import './upload-quota'
21import './user-notifications' 23import './user-notifications'
22import './user-subscriptions' 24import './user-subscriptions'
@@ -24,12 +26,11 @@ import './users-admin'
24import './users' 26import './users'
25import './video-blacklist' 27import './video-blacklist'
26import './video-captions' 28import './video-captions'
29import './video-channel-syncs'
27import './video-channels' 30import './video-channels'
28import './video-comments' 31import './video-comments'
29import './video-files' 32import './video-files'
30import './video-imports' 33import './video-imports'
31import './video-channel-syncs'
32import './channel-import-videos'
33import './video-playlists' 34import './video-playlists'
34import './video-source' 35import './video-source'
35import './video-studio' 36import './video-studio'
diff --git a/server/tests/api/check-params/two-factor.ts b/server/tests/api/check-params/two-factor.ts
new file mode 100644
index 000000000..f8365f1b5
--- /dev/null
+++ b/server/tests/api/check-params/two-factor.ts
@@ -0,0 +1,288 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { HttpStatusCode } from '@shared/models'
4import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands'
5
6describe('Test two factor API validators', function () {
7 let server: PeerTubeServer
8
9 let rootId: number
10 let rootPassword: string
11 let rootRequestToken: string
12 let rootOTPToken: string
13
14 let userId: number
15 let userToken = ''
16 let userPassword: string
17 let userRequestToken: string
18 let userOTPToken: string
19
20 // ---------------------------------------------------------------
21
22 before(async function () {
23 this.timeout(30000)
24
25 {
26 server = await createSingleServer(1)
27 await setAccessTokensToServers([ server ])
28 }
29
30 {
31 const result = await server.users.generate('user1')
32 userToken = result.token
33 userId = result.userId
34 userPassword = result.password
35 }
36
37 {
38 const { id } = await server.users.getMyInfo()
39 rootId = id
40 rootPassword = server.store.user.password
41 }
42 })
43
44 describe('When requesting two factor', function () {
45
46 it('Should fail with an unknown user id', async function () {
47 await server.twoFactor.request({ userId: 42, currentPassword: rootPassword, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
48 })
49
50 it('Should fail with an invalid user id', async function () {
51 await server.twoFactor.request({
52 userId: 'invalid' as any,
53 currentPassword: rootPassword,
54 expectedStatus: HttpStatusCode.BAD_REQUEST_400
55 })
56 })
57
58 it('Should fail to request another user two factor without the appropriate rights', async function () {
59 await server.twoFactor.request({
60 userId: rootId,
61 token: userToken,
62 currentPassword: userPassword,
63 expectedStatus: HttpStatusCode.FORBIDDEN_403
64 })
65 })
66
67 it('Should succeed to request another user two factor with the appropriate rights', async function () {
68 await server.twoFactor.request({ userId, currentPassword: rootPassword })
69 })
70
71 it('Should fail to request two factor without a password', async function () {
72 await server.twoFactor.request({
73 userId,
74 token: userToken,
75 currentPassword: undefined,
76 expectedStatus: HttpStatusCode.BAD_REQUEST_400
77 })
78 })
79
80 it('Should fail to request two factor with an incorrect password', async function () {
81 await server.twoFactor.request({
82 userId,
83 token: userToken,
84 currentPassword: rootPassword,
85 expectedStatus: HttpStatusCode.FORBIDDEN_403
86 })
87 })
88
89 it('Should succeed to request two factor without a password when targeting a remote user with an admin account', async function () {
90 await server.twoFactor.request({ userId })
91 })
92
93 it('Should fail to request two factor without a password when targeting myself with an admin account', async function () {
94 await server.twoFactor.request({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
95 await server.twoFactor.request({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
96 })
97
98 it('Should succeed to request my two factor auth', async function () {
99 {
100 const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword })
101 userRequestToken = otpRequest.requestToken
102 userOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate()
103 }
104
105 {
106 const { otpRequest } = await server.twoFactor.request({ userId: rootId, currentPassword: rootPassword })
107 rootRequestToken = otpRequest.requestToken
108 rootOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate()
109 }
110 })
111 })
112
113 describe('When confirming two factor request', function () {
114
115 it('Should fail with an unknown user id', async function () {
116 await server.twoFactor.confirmRequest({
117 userId: 42,
118 requestToken: rootRequestToken,
119 otpToken: rootOTPToken,
120 expectedStatus: HttpStatusCode.NOT_FOUND_404
121 })
122 })
123
124 it('Should fail with an invalid user id', async function () {
125 await server.twoFactor.confirmRequest({
126 userId: 'invalid' as any,
127 requestToken: rootRequestToken,
128 otpToken: rootOTPToken,
129 expectedStatus: HttpStatusCode.BAD_REQUEST_400
130 })
131 })
132
133 it('Should fail to confirm another user two factor request without the appropriate rights', async function () {
134 await server.twoFactor.confirmRequest({
135 userId: rootId,
136 token: userToken,
137 requestToken: rootRequestToken,
138 otpToken: rootOTPToken,
139 expectedStatus: HttpStatusCode.FORBIDDEN_403
140 })
141 })
142
143 it('Should fail without request token', async function () {
144 await server.twoFactor.confirmRequest({
145 userId,
146 requestToken: undefined,
147 otpToken: userOTPToken,
148 expectedStatus: HttpStatusCode.BAD_REQUEST_400
149 })
150 })
151
152 it('Should fail with an invalid request token', async function () {
153 await server.twoFactor.confirmRequest({
154 userId,
155 requestToken: 'toto',
156 otpToken: userOTPToken,
157 expectedStatus: HttpStatusCode.FORBIDDEN_403
158 })
159 })
160
161 it('Should fail with request token of another user', async function () {
162 await server.twoFactor.confirmRequest({
163 userId,
164 requestToken: rootRequestToken,
165 otpToken: userOTPToken,
166 expectedStatus: HttpStatusCode.FORBIDDEN_403
167 })
168 })
169
170 it('Should fail without an otp token', async function () {
171 await server.twoFactor.confirmRequest({
172 userId,
173 requestToken: userRequestToken,
174 otpToken: undefined,
175 expectedStatus: HttpStatusCode.BAD_REQUEST_400
176 })
177 })
178
179 it('Should fail with a bad otp token', async function () {
180 await server.twoFactor.confirmRequest({
181 userId,
182 requestToken: userRequestToken,
183 otpToken: '123456',
184 expectedStatus: HttpStatusCode.FORBIDDEN_403
185 })
186 })
187
188 it('Should succeed to confirm another user two factor request with the appropriate rights', async function () {
189 await server.twoFactor.confirmRequest({
190 userId,
191 requestToken: userRequestToken,
192 otpToken: userOTPToken
193 })
194
195 // Reinit
196 await server.twoFactor.disable({ userId, currentPassword: rootPassword })
197 })
198
199 it('Should succeed to confirm my two factor request', async function () {
200 await server.twoFactor.confirmRequest({
201 userId,
202 token: userToken,
203 requestToken: userRequestToken,
204 otpToken: userOTPToken
205 })
206 })
207
208 it('Should fail to confirm again two factor request', async function () {
209 await server.twoFactor.confirmRequest({
210 userId,
211 token: userToken,
212 requestToken: userRequestToken,
213 otpToken: userOTPToken,
214 expectedStatus: HttpStatusCode.BAD_REQUEST_400
215 })
216 })
217 })
218
219 describe('When disabling two factor', function () {
220
221 it('Should fail with an unknown user id', async function () {
222 await server.twoFactor.disable({
223 userId: 42,
224 currentPassword: rootPassword,
225 expectedStatus: HttpStatusCode.NOT_FOUND_404
226 })
227 })
228
229 it('Should fail with an invalid user id', async function () {
230 await server.twoFactor.disable({
231 userId: 'invalid' as any,
232 currentPassword: rootPassword,
233 expectedStatus: HttpStatusCode.BAD_REQUEST_400
234 })
235 })
236
237 it('Should fail to disable another user two factor without the appropriate rights', async function () {
238 await server.twoFactor.disable({
239 userId: rootId,
240 token: userToken,
241 currentPassword: userPassword,
242 expectedStatus: HttpStatusCode.FORBIDDEN_403
243 })
244 })
245
246 it('Should fail to disable two factor with an incorrect password', async function () {
247 await server.twoFactor.disable({
248 userId,
249 token: userToken,
250 currentPassword: rootPassword,
251 expectedStatus: HttpStatusCode.FORBIDDEN_403
252 })
253 })
254
255 it('Should succeed to disable two factor without a password when targeting a remote user with an admin account', async function () {
256 await server.twoFactor.disable({ userId })
257 await server.twoFactor.requestAndConfirm({ userId })
258 })
259
260 it('Should fail to disable two factor without a password when targeting myself with an admin account', async function () {
261 await server.twoFactor.disable({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
262 await server.twoFactor.disable({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
263 })
264
265 it('Should succeed to disable another user two factor with the appropriate rights', async function () {
266 await server.twoFactor.disable({ userId, currentPassword: rootPassword })
267
268 await server.twoFactor.requestAndConfirm({ userId })
269 })
270
271 it('Should succeed to update my two factor auth', async function () {
272 await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword })
273 })
274
275 it('Should fail to disable again two factor', async function () {
276 await server.twoFactor.disable({
277 userId,
278 token: userToken,
279 currentPassword: userPassword,
280 expectedStatus: HttpStatusCode.BAD_REQUEST_400
281 })
282 })
283 })
284
285 after(async function () {
286 await cleanupTests([ server ])
287 })
288})
diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts
index 502959258..3ea6be9ff 100644
--- a/server/tests/api/live/live-fast-restream.ts
+++ b/server/tests/api/live/live-fast-restream.ts
@@ -59,7 +59,7 @@ describe('Fast restream in live', function () {
59 const video = await server.videos.get({ id: liveId }) 59 const video = await server.videos.get({ id: liveId })
60 expect(video.streamingPlaylists).to.have.lengthOf(1) 60 expect(video.streamingPlaylists).to.have.lengthOf(1)
61 61
62 await server.live.getSegment({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) 62 await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 })
63 await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) 63 await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200)
64 await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) 64 await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200)
65 65
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index c436f0f01..5dd2bd9ab 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -3,7 +3,7 @@
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { basename, join } from 'path' 4import { basename, join } from 'path'
5import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg' 5import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
6import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared' 6import { testImage, testVideoResolutions } from '@server/tests/shared'
7import { getAllFiles, wait } from '@shared/core-utils' 7import { getAllFiles, wait } from '@shared/core-utils'
8import { 8import {
9 HttpStatusCode, 9 HttpStatusCode,
@@ -372,46 +372,6 @@ describe('Test live', function () {
372 return uuid 372 return uuid
373 } 373 }
374 374
375 async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
376 for (const server of servers) {
377 const { data } = await server.videos.list()
378 expect(data.find(v => v.uuid === liveVideoId)).to.exist
379
380 const video = await server.videos.get({ id: liveVideoId })
381
382 expect(video.streamingPlaylists).to.have.lengthOf(1)
383
384 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
385 expect(hlsPlaylist).to.exist
386
387 // Only finite files are displayed
388 expect(hlsPlaylist.files).to.have.lengthOf(0)
389
390 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
391
392 for (let i = 0; i < resolutions.length; i++) {
393 const segmentNum = 3
394 const segmentName = `${i}-00000${segmentNum}.ts`
395 await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum })
396
397 const subPlaylist = await servers[0].streamingPlaylists.get({
398 url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`
399 })
400
401 expect(subPlaylist).to.contain(segmentName)
402
403 const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
404 await checkLiveSegmentHash({
405 server,
406 baseUrlSegment: baseUrlAndPath,
407 videoUUID: video.uuid,
408 segmentName,
409 hlsPlaylist
410 })
411 }
412 }
413 }
414
415 function updateConf (resolutions: number[]) { 375 function updateConf (resolutions: number[]) {
416 return servers[0].config.updateCustomSubConfig({ 376 return servers[0].config.updateCustomSubConfig({
417 newConfig: { 377 newConfig: {
@@ -449,7 +409,14 @@ describe('Test live', function () {
449 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 409 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
450 await waitJobs(servers) 410 await waitJobs(servers)
451 411
452 await testVideoResolutions(liveVideoId, [ 720 ]) 412 await testVideoResolutions({
413 originServer: servers[0],
414 servers,
415 liveVideoId,
416 resolutions: [ 720 ],
417 objectStorage: false,
418 transcoded: true
419 })
453 420
454 await stopFfmpeg(ffmpegCommand) 421 await stopFfmpeg(ffmpegCommand)
455 }) 422 })
@@ -477,7 +444,14 @@ describe('Test live', function () {
477 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 444 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
478 await waitJobs(servers) 445 await waitJobs(servers)
479 446
480 await testVideoResolutions(liveVideoId, resolutions.concat([ 720 ])) 447 await testVideoResolutions({
448 originServer: servers[0],
449 servers,
450 liveVideoId,
451 resolutions: resolutions.concat([ 720 ]),
452 objectStorage: false,
453 transcoded: true
454 })
481 455
482 await stopFfmpeg(ffmpegCommand) 456 await stopFfmpeg(ffmpegCommand)
483 }) 457 })
@@ -522,7 +496,14 @@ describe('Test live', function () {
522 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 496 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
523 await waitJobs(servers) 497 await waitJobs(servers)
524 498
525 await testVideoResolutions(liveVideoId, resolutions) 499 await testVideoResolutions({
500 originServer: servers[0],
501 servers,
502 liveVideoId,
503 resolutions,
504 objectStorage: false,
505 transcoded: true
506 })
526 507
527 await stopFfmpeg(ffmpegCommand) 508 await stopFfmpeg(ffmpegCommand)
528 await commands[0].waitUntilEnded({ videoId: liveVideoId }) 509 await commands[0].waitUntilEnded({ videoId: liveVideoId })
@@ -538,7 +519,7 @@ describe('Test live', function () {
538 } 519 }
539 520
540 const minBitrateLimits = { 521 const minBitrateLimits = {
541 720: 5500 * 1000, 522 720: 5000 * 1000,
542 360: 1000 * 1000, 523 360: 1000 * 1000,
543 240: 550 * 1000 524 240: 550 * 1000
544 } 525 }
@@ -569,7 +550,7 @@ describe('Test live', function () {
569 if (resolution >= 720) { 550 if (resolution >= 720) {
570 expect(file.fps).to.be.approximately(60, 10) 551 expect(file.fps).to.be.approximately(60, 10)
571 } else { 552 } else {
572 expect(file.fps).to.be.approximately(30, 2) 553 expect(file.fps).to.be.approximately(30, 3)
573 } 554 }
574 555
575 const filename = basename(file.fileUrl) 556 const filename = basename(file.fileUrl)
@@ -611,7 +592,14 @@ describe('Test live', function () {
611 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 592 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
612 await waitJobs(servers) 593 await waitJobs(servers)
613 594
614 await testVideoResolutions(liveVideoId, resolutions) 595 await testVideoResolutions({
596 originServer: servers[0],
597 servers,
598 liveVideoId,
599 resolutions,
600 objectStorage: false,
601 transcoded: true
602 })
615 603
616 await stopFfmpeg(ffmpegCommand) 604 await stopFfmpeg(ffmpegCommand)
617 await commands[0].waitUntilEnded({ videoId: liveVideoId }) 605 await commands[0].waitUntilEnded({ videoId: liveVideoId })
@@ -640,7 +628,14 @@ describe('Test live', function () {
640 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 628 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
641 await waitJobs(servers) 629 await waitJobs(servers)
642 630
643 await testVideoResolutions(liveVideoId, [ 720 ]) 631 await testVideoResolutions({
632 originServer: servers[0],
633 servers,
634 liveVideoId,
635 resolutions: [ 720 ],
636 objectStorage: false,
637 transcoded: true
638 })
644 639
645 await stopFfmpeg(ffmpegCommand) 640 await stopFfmpeg(ffmpegCommand)
646 await commands[0].waitUntilEnded({ videoId: liveVideoId }) 641 await commands[0].waitUntilEnded({ videoId: liveVideoId })
diff --git a/server/tests/api/notifications/admin-notifications.ts b/server/tests/api/notifications/admin-notifications.ts
index b3bb4888e..07c981a37 100644
--- a/server/tests/api/notifications/admin-notifications.ts
+++ b/server/tests/api/notifications/admin-notifications.ts
@@ -37,7 +37,7 @@ describe('Test admin notifications', function () {
37 plugins: { 37 plugins: {
38 index: { 38 index: {
39 enabled: true, 39 enabled: true,
40 check_latest_versions_interval: '5 seconds' 40 check_latest_versions_interval: '3 seconds'
41 } 41 }
42 } 42 }
43 } 43 }
@@ -62,7 +62,7 @@ describe('Test admin notifications', function () {
62 62
63 describe('Latest PeerTube version notification', function () { 63 describe('Latest PeerTube version notification', function () {
64 64
65 it('Should not send a notification to admins if there is not a new version', async function () { 65 it('Should not send a notification to admins if there is no new version', async function () {
66 this.timeout(30000) 66 this.timeout(30000)
67 67
68 joinPeerTubeServer.setLatestVersion('1.4.2') 68 joinPeerTubeServer.setLatestVersion('1.4.2')
@@ -71,7 +71,7 @@ describe('Test admin notifications', function () {
71 await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' }) 71 await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' })
72 }) 72 })
73 73
74 it('Should send a notification to admins on new plugin version', async function () { 74 it('Should send a notification to admins on new version', async function () {
75 this.timeout(30000) 75 this.timeout(30000)
76 76
77 joinPeerTubeServer.setLatestVersion('15.4.2') 77 joinPeerTubeServer.setLatestVersion('15.4.2')
diff --git a/server/tests/api/notifications/moderation-notifications.ts b/server/tests/api/notifications/moderation-notifications.ts
index d8a7d576e..fc953f144 100644
--- a/server/tests/api/notifications/moderation-notifications.ts
+++ b/server/tests/api/notifications/moderation-notifications.ts
@@ -382,7 +382,7 @@ describe('Test moderation notifications', function () {
382 }) 382 })
383 383
384 it('Should send a notification only to admin when there is a new instance follower', async function () { 384 it('Should send a notification only to admin when there is a new instance follower', async function () {
385 this.timeout(20000) 385 this.timeout(60000)
386 386
387 await servers[2].follows.follow({ hosts: [ servers[0].url ] }) 387 await servers[2].follows.follow({ hosts: [ servers[0].url ] })
388 388
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
index 0958ffe0f..7e16b4c89 100644
--- a/server/tests/api/object-storage/live.ts
+++ b/server/tests/api/object-storage/live.ts
@@ -1,9 +1,9 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectStartWith } from '@server/tests/shared' 4import { expectStartWith, testVideoResolutions } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models'
7import { 7import {
8 createMultipleServers, 8 createMultipleServers,
9 doubleFollow, 9 doubleFollow,
@@ -35,41 +35,43 @@ async function createLive (server: PeerTubeServer, permanent: boolean) {
35 return uuid 35 return uuid
36} 36}
37 37
38async function checkFiles (files: VideoFile[]) { 38async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) {
39 for (const file of files) { 39 for (const server of servers) {
40 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 40 const video = await server.videos.get({ id: videoUUID })
41 41
42 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 42 expect(video.files).to.have.lengthOf(0)
43 } 43 expect(video.streamingPlaylists).to.have.lengthOf(1)
44}
45 44
46async function getFiles (server: PeerTubeServer, videoUUID: string) { 45 const files = video.streamingPlaylists[0].files
47 const video = await server.videos.get({ id: videoUUID }) 46 expect(files).to.have.lengthOf(numberOfFiles)
48 47
49 expect(video.files).to.have.lengthOf(0) 48 for (const file of files) {
50 expect(video.streamingPlaylists).to.have.lengthOf(1) 49 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
51 50
52 return video.streamingPlaylists[0].files 51 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
52 }
53 }
53} 54}
54 55
55async function streamAndEnd (servers: PeerTubeServer[], liveUUID: string) { 56async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) {
56 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveUUID }) 57 const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`)
57 await waitUntilLivePublishedOnAllServers(servers, liveUUID)
58
59 const videoLiveDetails = await servers[0].videos.get({ id: liveUUID })
60 const liveDetails = await servers[0].live.get({ videoId: liveUUID })
61 58
62 await stopFfmpeg(ffmpegCommand) 59 for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) {
63 60 await server.live.getPlaylistFile({
64 if (liveDetails.permanentLive) { 61 videoUUID,
65 await waitUntilLiveWaitingOnAllServers(servers, liveUUID) 62 playlistName,
66 } else { 63 expectedStatus: HttpStatusCode.NOT_FOUND_404,
67 await waitUntilLiveReplacedByReplayOnAllServers(servers, liveUUID) 64 objectStorage: true
65 })
68 } 66 }
69 67
70 await waitJobs(servers) 68 await server.live.getSegmentFile({
71 69 videoUUID,
72 return { videoLiveDetails, liveDetails } 70 playlistNumber: 0,
71 segment: 0,
72 objectStorage: true,
73 expectedStatus: HttpStatusCode.NOT_FOUND_404
74 })
73} 75}
74 76
75describe('Object storage for lives', function () { 77describe('Object storage for lives', function () {
@@ -100,57 +102,124 @@ describe('Object storage for lives', function () {
100 videoUUID = await createLive(servers[0], false) 102 videoUUID = await createLive(servers[0], false)
101 }) 103 })
102 104
103 it('Should create a live and save the replay on object storage', async function () { 105 it('Should create a live and publish it on object storage', async function () {
106 this.timeout(220000)
107
108 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
109 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
110
111 await testVideoResolutions({
112 originServer: servers[0],
113 servers,
114 liveVideoId: videoUUID,
115 resolutions: [ 720 ],
116 transcoded: false,
117 objectStorage: true
118 })
119
120 await stopFfmpeg(ffmpegCommand)
121 })
122
123 it('Should have saved the replay on object storage', async function () {
104 this.timeout(220000) 124 this.timeout(220000)
105 125
106 await streamAndEnd(servers, videoUUID) 126 await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID)
127 await waitJobs(servers)
107 128
108 for (const server of servers) { 129 await checkFilesExist(servers, videoUUID, 1)
109 const files = await getFiles(server, videoUUID) 130 })
110 expect(files).to.have.lengthOf(1)
111 131
112 await checkFiles(files) 132 it('Should have cleaned up live files from object storage', async function () {
113 } 133 await checkFilesCleanup(servers[0], videoUUID, [ 720 ])
114 }) 134 })
115 }) 135 })
116 136
117 describe('With live transcoding', async function () { 137 describe('With live transcoding', async function () {
118 let videoUUIDPermanent: string 138 const resolutions = [ 720, 480, 360, 240, 144 ]
119 let videoUUIDNonPermanent: string
120 139
121 before(async function () { 140 before(async function () {
122 await servers[0].config.enableLive({ transcoding: true }) 141 await servers[0].config.enableLive({ transcoding: true })
123
124 videoUUIDPermanent = await createLive(servers[0], true)
125 videoUUIDNonPermanent = await createLive(servers[0], false)
126 }) 142 })
127 143
128 it('Should create a live and save the replay on object storage', async function () { 144 describe('Normal replay', function () {
129 this.timeout(240000) 145 let videoUUIDNonPermanent: string
146
147 before(async function () {
148 videoUUIDNonPermanent = await createLive(servers[0], false)
149 })
150
151 it('Should create a live and publish it on object storage', async function () {
152 this.timeout(240000)
153
154 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent })
155 await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent)
156
157 await testVideoResolutions({
158 originServer: servers[0],
159 servers,
160 liveVideoId: videoUUIDNonPermanent,
161 resolutions,
162 transcoded: true,
163 objectStorage: true
164 })
165
166 await stopFfmpeg(ffmpegCommand)
167 })
130 168
131 await streamAndEnd(servers, videoUUIDNonPermanent) 169 it('Should have saved the replay on object storage', async function () {
170 this.timeout(220000)
132 171
133 for (const server of servers) { 172 await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent)
134 const files = await getFiles(server, videoUUIDNonPermanent) 173 await waitJobs(servers)
135 expect(files).to.have.lengthOf(5)
136 174
137 await checkFiles(files) 175 await checkFilesExist(servers, videoUUIDNonPermanent, 5)
138 } 176 })
177
178 it('Should have cleaned up live files from object storage', async function () {
179 await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions)
180 })
139 }) 181 })
140 182
141 it('Should create a live and save the replay of permanent live on object storage', async function () { 183 describe('Permanent replay', function () {
142 this.timeout(240000) 184 let videoUUIDPermanent: string
185
186 before(async function () {
187 videoUUIDPermanent = await createLive(servers[0], true)
188 })
189
190 it('Should create a live and publish it on object storage', async function () {
191 this.timeout(240000)
192
193 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent })
194 await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent)
195
196 await testVideoResolutions({
197 originServer: servers[0],
198 servers,
199 liveVideoId: videoUUIDPermanent,
200 resolutions,
201 transcoded: true,
202 objectStorage: true
203 })
204
205 await stopFfmpeg(ffmpegCommand)
206 })
207
208 it('Should have saved the replay on object storage', async function () {
209 this.timeout(220000)
143 210
144 const { videoLiveDetails } = await streamAndEnd(servers, videoUUIDPermanent) 211 await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent)
212 await waitJobs(servers)
145 213
146 const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) 214 const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent })
215 const replay = await findExternalSavedVideo(servers[0], videoLiveDetails)
147 216
148 for (const server of servers) { 217 await checkFilesExist(servers, replay.uuid, 5)
149 const files = await getFiles(server, replay.uuid) 218 })
150 expect(files).to.have.lengthOf(5)
151 219
152 await checkFiles(files) 220 it('Should have cleaned up live files from object storage', async function () {
153 } 221 await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions)
222 })
154 }) 223 })
155 }) 224 })
156 225
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 5abed358f..f349a7a76 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -5,7 +5,7 @@ import { readdir } from 'fs-extra'
5import magnetUtil from 'magnet-uri' 5import magnetUtil from 'magnet-uri'
6import { basename, join } from 'path' 6import { basename, join } from 'path'
7import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared' 7import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared'
8import { root, wait } from '@shared/core-utils' 8import { wait } from '@shared/core-utils'
9import { 9import {
10 HttpStatusCode, 10 HttpStatusCode,
11 VideoDetails, 11 VideoDetails,
@@ -159,12 +159,12 @@ async function check2Webseeds (videoUUID?: string) {
159 const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) 159 const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
160 160
161 const directories = [ 161 const directories = [
162 'test' + servers[0].internalServerNumber + '/redundancy', 162 servers[0].getDirectoryPath('redundancy'),
163 'test' + servers[1].internalServerNumber + '/videos' 163 servers[1].getDirectoryPath('videos')
164 ] 164 ]
165 165
166 for (const directory of directories) { 166 for (const directory of directories) {
167 const files = await readdir(join(root(), directory)) 167 const files = await readdir(directory)
168 expect(files).to.have.length.at.least(4) 168 expect(files).to.have.length.at.least(4)
169 169
170 // Ensure we files exist on disk 170 // Ensure we files exist on disk
@@ -214,12 +214,12 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
214 const { hlsFilenames } = await ensureSameFilenames(videoUUID) 214 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
215 215
216 const directories = [ 216 const directories = [
217 'test' + servers[0].internalServerNumber + '/redundancy/hls', 217 servers[0].getDirectoryPath('redundancy/hls'),
218 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' 218 servers[1].getDirectoryPath('streaming-playlists/hls')
219 ] 219 ]
220 220
221 for (const directory of directories) { 221 for (const directory of directories) {
222 const files = await readdir(join(root(), directory, videoUUID)) 222 const files = await readdir(join(directory, videoUUID))
223 expect(files).to.have.length.at.least(4) 223 expect(files).to.have.length.at.least(4)
224 224
225 // Ensure we files exist on disk 225 // Ensure we files exist on disk
diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts
index c65152c6f..643f1a531 100644
--- a/server/tests/api/users/index.ts
+++ b/server/tests/api/users/index.ts
@@ -1,3 +1,4 @@
1import './two-factor'
1import './user-subscriptions' 2import './user-subscriptions'
2import './user-videos' 3import './user-videos'
3import './users' 4import './users'
diff --git a/server/tests/api/users/two-factor.ts b/server/tests/api/users/two-factor.ts
new file mode 100644
index 000000000..0dcab9e17
--- /dev/null
+++ b/server/tests/api/users/two-factor.ts
@@ -0,0 +1,200 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { expectStartWith } from '@server/tests/shared'
5import { HttpStatusCode } from '@shared/models'
6import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands'
7
8async function login (options: {
9 server: PeerTubeServer
10 username: string
11 password: string
12 otpToken?: string
13 expectedStatus?: HttpStatusCode
14}) {
15 const { server, username, password, otpToken, expectedStatus } = options
16
17 const user = { username, password }
18 const { res, body: { access_token: token } } = await server.login.loginAndGetResponse({ user, otpToken, expectedStatus })
19
20 return { res, token }
21}
22
23describe('Test users', function () {
24 let server: PeerTubeServer
25 let otpSecret: string
26 let requestToken: string
27
28 const userUsername = 'user1'
29 let userId: number
30 let userPassword: string
31 let userToken: string
32
33 before(async function () {
34 this.timeout(30000)
35
36 server = await createSingleServer(1)
37
38 await setAccessTokensToServers([ server ])
39 const res = await server.users.generate(userUsername)
40 userId = res.userId
41 userPassword = res.password
42 userToken = res.token
43 })
44
45 it('Should not add the header on login if two factor is not enabled', async function () {
46 const { res, token } = await login({ server, username: userUsername, password: userPassword })
47
48 expect(res.header['x-peertube-otp']).to.not.exist
49
50 await server.users.getMyInfo({ token })
51 })
52
53 it('Should request two factor and get the secret and uri', async function () {
54 const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword })
55
56 expect(otpRequest.requestToken).to.exist
57
58 expect(otpRequest.secret).to.exist
59 expect(otpRequest.secret).to.have.lengthOf(32)
60
61 expect(otpRequest.uri).to.exist
62 expectStartWith(otpRequest.uri, 'otpauth://')
63 expect(otpRequest.uri).to.include(otpRequest.secret)
64
65 requestToken = otpRequest.requestToken
66 otpSecret = otpRequest.secret
67 })
68
69 it('Should not have two factor confirmed yet', async function () {
70 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
71 expect(twoFactorEnabled).to.be.false
72 })
73
74 it('Should confirm two factor', async function () {
75 await server.twoFactor.confirmRequest({
76 userId,
77 token: userToken,
78 otpToken: TwoFactorCommand.buildOTP({ secret: otpSecret }).generate(),
79 requestToken
80 })
81 })
82
83 it('Should not add the header on login if two factor is enabled and password is incorrect', async function () {
84 const { res, token } = await login({ server, username: userUsername, password: 'fake', expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
85
86 expect(res.header['x-peertube-otp']).to.not.exist
87 expect(token).to.not.exist
88 })
89
90 it('Should add the header on login if two factor is enabled and password is correct', async function () {
91 const { res, token } = await login({
92 server,
93 username: userUsername,
94 password: userPassword,
95 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
96 })
97
98 expect(res.header['x-peertube-otp']).to.exist
99 expect(token).to.not.exist
100
101 await server.users.getMyInfo({ token })
102 })
103
104 it('Should not login with correct password and incorrect otp secret', async function () {
105 const otp = TwoFactorCommand.buildOTP({ secret: 'a'.repeat(32) })
106
107 const { res, token } = await login({
108 server,
109 username: userUsername,
110 password: userPassword,
111 otpToken: otp.generate(),
112 expectedStatus: HttpStatusCode.BAD_REQUEST_400
113 })
114
115 expect(res.header['x-peertube-otp']).to.not.exist
116 expect(token).to.not.exist
117 })
118
119 it('Should not login with correct password and incorrect otp code', async function () {
120 const { res, token } = await login({
121 server,
122 username: userUsername,
123 password: userPassword,
124 otpToken: '123456',
125 expectedStatus: HttpStatusCode.BAD_REQUEST_400
126 })
127
128 expect(res.header['x-peertube-otp']).to.not.exist
129 expect(token).to.not.exist
130 })
131
132 it('Should not login with incorrect password and correct otp code', async function () {
133 const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate()
134
135 const { res, token } = await login({
136 server,
137 username: userUsername,
138 password: 'fake',
139 otpToken,
140 expectedStatus: HttpStatusCode.BAD_REQUEST_400
141 })
142
143 expect(res.header['x-peertube-otp']).to.not.exist
144 expect(token).to.not.exist
145 })
146
147 it('Should correctly login with correct password and otp code', async function () {
148 const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate()
149
150 const { res, token } = await login({ server, username: userUsername, password: userPassword, otpToken })
151
152 expect(res.header['x-peertube-otp']).to.not.exist
153 expect(token).to.exist
154
155 await server.users.getMyInfo({ token })
156 })
157
158 it('Should have two factor enabled when getting my info', async function () {
159 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
160 expect(twoFactorEnabled).to.be.true
161 })
162
163 it('Should disable two factor and be able to login without otp token', async function () {
164 await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword })
165
166 const { res, token } = await login({ server, username: userUsername, password: userPassword })
167 expect(res.header['x-peertube-otp']).to.not.exist
168
169 await server.users.getMyInfo({ token })
170 })
171
172 it('Should have two factor disabled when getting my info', async function () {
173 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
174 expect(twoFactorEnabled).to.be.false
175 })
176
177 it('Should enable two factor auth without password from an admin', async function () {
178 const { otpRequest } = await server.twoFactor.request({ userId })
179
180 await server.twoFactor.confirmRequest({
181 userId,
182 otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate(),
183 requestToken: otpRequest.requestToken
184 })
185
186 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
187 expect(twoFactorEnabled).to.be.true
188 })
189
190 it('Should disable two factor auth without password from an admin', async function () {
191 await server.twoFactor.disable({ userId })
192
193 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
194 expect(twoFactorEnabled).to.be.false
195 })
196
197 after(async function () {
198 await cleanupTests([ server ])
199 })
200})
diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts
index 62d668d1e..188e6f137 100644
--- a/server/tests/api/users/users-multiple-servers.ts
+++ b/server/tests/api/users/users-multiple-servers.ts
@@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () {
197 it('Should not have actor files', async () => { 197 it('Should not have actor files', async () => {
198 for (const server of servers) { 198 for (const server of servers) {
199 for (const userAvatarFilename of userAvatarFilenames) { 199 for (const userAvatarFilename of userAvatarFilenames) {
200 await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber) 200 await checkActorFilesWereRemoved(userAvatarFilename, server)
201 } 201 }
202 } 202 }
203 }) 203 })
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index d47807a79..2ad749fd4 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -156,7 +156,7 @@ describe('Test multiple servers', function () {
156 }) 156 })
157 157
158 it('Should upload the video on server 2 and propagate on each server', async function () { 158 it('Should upload the video on server 2 and propagate on each server', async function () {
159 this.timeout(100000) 159 this.timeout(240000)
160 160
161 const user = { 161 const user = {
162 username: 'user1', 162 username: 'user1',
diff --git a/server/tests/api/videos/video-files.ts b/server/tests/api/videos/video-files.ts
index 10277b9cf..c0b886aad 100644
--- a/server/tests/api/videos/video-files.ts
+++ b/server/tests/api/videos/video-files.ts
@@ -33,7 +33,7 @@ describe('Test videos files', function () {
33 let validId2: string 33 let validId2: string
34 34
35 before(async function () { 35 before(async function () {
36 this.timeout(120_000) 36 this.timeout(360_000)
37 37
38 { 38 {
39 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) 39 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts
index 47b8c7b1e..9d223de48 100644
--- a/server/tests/api/videos/video-playlists.ts
+++ b/server/tests/api/videos/video-playlists.ts
@@ -70,7 +70,7 @@ describe('Test video playlists', function () {
70 let commands: PlaylistsCommand[] 70 let commands: PlaylistsCommand[]
71 71
72 before(async function () { 72 before(async function () {
73 this.timeout(120000) 73 this.timeout(240000)
74 74
75 servers = await createMultipleServers(3) 75 servers = await createMultipleServers(3)
76 76
@@ -1049,7 +1049,7 @@ describe('Test video playlists', function () {
1049 this.timeout(30000) 1049 this.timeout(30000)
1050 1050
1051 for (const server of servers) { 1051 for (const server of servers) {
1052 await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber) 1052 await checkPlaylistFilesWereRemoved(playlistServer1UUID, server)
1053 } 1053 }
1054 }) 1054 })
1055 1055
diff --git a/server/tests/api/videos/video-privacy.ts b/server/tests/api/videos/video-privacy.ts
index b18c71c94..92f5dab3c 100644
--- a/server/tests/api/videos/video-privacy.ts
+++ b/server/tests/api/videos/video-privacy.ts
@@ -45,7 +45,7 @@ describe('Test video privacy', function () {
45 describe('Private and internal videos', function () { 45 describe('Private and internal videos', function () {
46 46
47 it('Should upload a private and internal videos on server 1', async function () { 47 it('Should upload a private and internal videos on server 1', async function () {
48 this.timeout(10000) 48 this.timeout(50000)
49 49
50 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { 50 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
51 const attributes = { privacy } 51 const attributes = { privacy }
diff --git a/server/tests/api/videos/videos-common-filters.ts b/server/tests/api/videos/videos-common-filters.ts
index e7fc15e42..b176d90ab 100644
--- a/server/tests/api/videos/videos-common-filters.ts
+++ b/server/tests/api/videos/videos-common-filters.ts
@@ -232,7 +232,7 @@ describe('Test videos filter', function () {
232 }) 232 })
233 233
234 it('Should display only remote videos', async function () { 234 it('Should display only remote videos', async function () {
235 this.timeout(40000) 235 this.timeout(120000)
236 236
237 await servers[1].videos.upload({ attributes: { name: 'remote video' } }) 237 await servers[1].videos.upload({ attributes: { name: 'remote video' } })
238 238
diff --git a/server/tests/external-plugins/akismet.ts b/server/tests/external-plugins/akismet.ts
new file mode 100644
index 000000000..974bf0011
--- /dev/null
+++ b/server/tests/external-plugins/akismet.ts
@@ -0,0 +1,160 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { HttpStatusCode } from '@shared/models'
5import {
6 cleanupTests,
7 createMultipleServers,
8 doubleFollow,
9 PeerTubeServer,
10 setAccessTokensToServers,
11 waitJobs
12} from '@shared/server-commands'
13
14describe('Official plugin Akismet', function () {
15 let servers: PeerTubeServer[]
16 let videoUUID: string
17
18 before(async function () {
19 this.timeout(30000)
20
21 servers = await createMultipleServers(2)
22 await setAccessTokensToServers(servers)
23
24 await servers[0].plugins.install({
25 npmName: 'peertube-plugin-akismet'
26 })
27
28 if (!process.env.AKISMET_KEY) throw new Error('Missing AKISMET_KEY from env')
29
30 await servers[0].plugins.updateSettings({
31 npmName: 'peertube-plugin-akismet',
32 settings: {
33 'akismet-api-key': process.env.AKISMET_KEY
34 }
35 })
36
37 await doubleFollow(servers[0], servers[1])
38 })
39
40 describe('Local threads/replies', function () {
41
42 before(async function () {
43 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
44 videoUUID = uuid
45 })
46
47 it('Should not detect a thread as spam', async function () {
48 await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' })
49 })
50
51 it('Should not detect a reply as spam', async function () {
52 await servers[0].comments.addReplyToLastThread({ text: 'reply' })
53 })
54
55 it('Should detect a thread as spam', async function () {
56 await servers[0].comments.createThread({
57 videoId: videoUUID,
58 text: 'akismet-guaranteed-spam',
59 expectedStatus: HttpStatusCode.FORBIDDEN_403
60 })
61 })
62
63 it('Should detect a thread as spam', async function () {
64 await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' })
65 await servers[0].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
66 })
67 })
68
69 describe('Remote threads/replies', function () {
70
71 before(async function () {
72 this.timeout(60000)
73
74 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
75 videoUUID = uuid
76
77 await waitJobs(servers)
78 })
79
80 it('Should not detect a thread as spam', async function () {
81 this.timeout(30000)
82
83 await servers[1].comments.createThread({ videoId: videoUUID, text: 'remote comment 1' })
84 await waitJobs(servers)
85
86 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
87 expect(data).to.have.lengthOf(1)
88 })
89
90 it('Should not detect a reply as spam', async function () {
91 this.timeout(30000)
92
93 await servers[1].comments.addReplyToLastThread({ text: 'I agree with you' })
94 await waitJobs(servers)
95
96 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
97 expect(data).to.have.lengthOf(1)
98
99 const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: data[0].id })
100 expect(tree.children).to.have.lengthOf(1)
101 })
102
103 it('Should detect a thread as spam', async function () {
104 this.timeout(30000)
105
106 await servers[1].comments.createThread({ videoId: videoUUID, text: 'akismet-guaranteed-spam' })
107 await waitJobs(servers)
108
109 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
110 expect(data).to.have.lengthOf(1)
111 })
112
113 it('Should detect a thread as spam', async function () {
114 this.timeout(30000)
115
116 await servers[1].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam' })
117 await waitJobs(servers)
118
119 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
120 expect(data).to.have.lengthOf(1)
121
122 const thread = data[0]
123 const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: thread.id })
124 expect(tree.children).to.have.lengthOf(1)
125 })
126 })
127
128 describe('Signup', function () {
129
130 before(async function () {
131 await servers[0].config.updateExistingSubConfig({
132 newConfig: {
133 signup: {
134 enabled: true
135 }
136 }
137 })
138 })
139
140 it('Should allow signup', async function () {
141 await servers[0].users.register({
142 username: 'user1',
143 displayName: 'user 1'
144 })
145 })
146
147 it('Should detect a signup as SPAM', async function () {
148 await servers[0].users.register({
149 username: 'user2',
150 displayName: 'user 2',
151 email: 'akismet-guaranteed-spam@example.com',
152 expectedStatus: HttpStatusCode.FORBIDDEN_403
153 })
154 })
155 })
156
157 after(async function () {
158 await cleanupTests(servers)
159 })
160})
diff --git a/server/tests/external-plugins/auth-ldap.ts b/server/tests/external-plugins/auth-ldap.ts
index d7f155d2a..6f6a574a0 100644
--- a/server/tests/external-plugins/auth-ldap.ts
+++ b/server/tests/external-plugins/auth-ldap.ts
@@ -94,6 +94,14 @@ describe('Official plugin auth-ldap', function () {
94 await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } }) 94 await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } })
95 }) 95 })
96 96
97 it('Should not be able to ask password reset', async function () {
98 await server.users.askResetPassword({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 })
99 })
100
101 it('Should not be able to ask email verification', async function () {
102 await server.users.askSendVerifyEmail({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 })
103 })
104
97 it('Should not login if the plugin is uninstalled', async function () { 105 it('Should not login if the plugin is uninstalled', async function () {
98 await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' }) 106 await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' })
99 107
diff --git a/server/tests/external-plugins/index.ts b/server/tests/external-plugins/index.ts
index 31d818b51..815bbf1da 100644
--- a/server/tests/external-plugins/index.ts
+++ b/server/tests/external-plugins/index.ts
@@ -1,3 +1,4 @@
1import './akismet'
1import './auth-ldap' 2import './auth-ldap'
2import './auto-block-videos' 3import './auto-block-videos'
3import './auto-mute' 4import './auto-mute'
diff --git a/server/tests/fixtures/peertube-plugin-test-four/main.js b/server/tests/fixtures/peertube-plugin-test-four/main.js
index 5194e3e02..3e848c49e 100644
--- a/server/tests/fixtures/peertube-plugin-test-four/main.js
+++ b/server/tests/fixtures/peertube-plugin-test-four/main.js
@@ -128,6 +128,22 @@ async function register ({
128 128
129 return res.json(result) 129 return res.json(result)
130 }) 130 })
131
132 router.post('/send-notification', async (req, res) => {
133 peertubeHelpers.socket.sendNotification(req.body.userId, {
134 type: 1,
135 userId: req.body.userId
136 })
137
138 return res.sendStatus(201)
139 })
140
141 router.post('/send-video-live-new-state/:uuid', async (req, res) => {
142 const video = await peertubeHelpers.videos.loadByIdOrUUID(req.params.uuid)
143 peertubeHelpers.socket.sendVideoLiveNewState(video)
144
145 return res.sendStatus(201)
146 })
131 } 147 }
132 148
133} 149}
diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/main.js b/server/tests/fixtures/peertube-plugin-test-websocket/main.js
new file mode 100644
index 000000000..3fde76cfe
--- /dev/null
+++ b/server/tests/fixtures/peertube-plugin-test-websocket/main.js
@@ -0,0 +1,36 @@
1const WebSocketServer = require('ws').WebSocketServer
2
3async function register ({
4 registerWebSocketRoute
5}) {
6 const wss = new WebSocketServer({ noServer: true })
7
8 wss.on('connection', function connection(ws) {
9 ws.on('message', function message(data) {
10 if (data.toString() === 'ping') {
11 ws.send('pong')
12 }
13 })
14 })
15
16 registerWebSocketRoute({
17 route: '/toto',
18
19 handler: (request, socket, head) => {
20 wss.handleUpgrade(request, socket, head, ws => {
21 wss.emit('connection', ws, request)
22 })
23 }
24 })
25}
26
27async function unregister () {
28 return
29}
30
31module.exports = {
32 register,
33 unregister
34}
35
36// ###########################################################################
diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/package.json b/server/tests/fixtures/peertube-plugin-test-websocket/package.json
new file mode 100644
index 000000000..89c8baa04
--- /dev/null
+++ b/server/tests/fixtures/peertube-plugin-test-websocket/package.json
@@ -0,0 +1,20 @@
1{
2 "name": "peertube-plugin-test-websocket",
3 "version": "0.0.1",
4 "description": "Plugin test websocket",
5 "engine": {
6 "peertube": ">=1.3.0"
7 },
8 "keywords": [
9 "peertube",
10 "plugin"
11 ],
12 "homepage": "https://github.com/Chocobozzz/PeerTube",
13 "author": "Chocobozzz",
14 "bugs": "https://github.com/Chocobozzz/PeerTube/issues",
15 "library": "./main.js",
16 "staticDirs": {},
17 "css": [],
18 "clientScripts": [],
19 "translations": {}
20}
diff --git a/server/tests/fixtures/peertube-plugin-test/main.js b/server/tests/fixtures/peertube-plugin-test/main.js
index 813482a27..19dccf26e 100644
--- a/server/tests/fixtures/peertube-plugin-test/main.js
+++ b/server/tests/fixtures/peertube-plugin-test/main.js
@@ -178,6 +178,8 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
178 } 178 }
179 }) 179 })
180 180
181 // ---------------------------------------------------------------------------
182
181 registerHook({ 183 registerHook({
182 target: 'filter:api.video-thread.create.accept.result', 184 target: 'filter:api.video-thread.create.accept.result',
183 handler: ({ accepted }, { commentBody }) => checkCommentBadWord(accepted, commentBody) 185 handler: ({ accepted }, { commentBody }) => checkCommentBadWord(accepted, commentBody)
@@ -189,6 +191,13 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
189 }) 191 })
190 192
191 registerHook({ 193 registerHook({
194 target: 'filter:activity-pub.remote-video-comment.create.accept.result',
195 handler: ({ accepted }, { comment }) => checkCommentBadWord(accepted, comment)
196 })
197
198 // ---------------------------------------------------------------------------
199
200 registerHook({
192 target: 'filter:api.video-threads.list.params', 201 target: 'filter:api.video-threads.list.params',
193 handler: obj => addToCount(obj) 202 handler: obj => addToCount(obj)
194 }) 203 })
diff --git a/server/tests/helpers/crypto.ts b/server/tests/helpers/crypto.ts
new file mode 100644
index 000000000..b508c715b
--- /dev/null
+++ b/server/tests/helpers/crypto.ts
@@ -0,0 +1,33 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { decrypt, encrypt } from '@server/helpers/peertube-crypto'
5
6describe('Encrypt/Descrypt', function () {
7
8 it('Should encrypt and decrypt the string', async function () {
9 const secret = 'my_secret'
10 const str = 'my super string'
11
12 const encrypted = await encrypt(str, secret)
13 const decrypted = await decrypt(encrypted, secret)
14
15 expect(str).to.equal(decrypted)
16 })
17
18 it('Should not decrypt without the same secret', async function () {
19 const str = 'my super string'
20
21 const encrypted = await encrypt(str, 'my_secret')
22
23 let error = false
24
25 try {
26 await decrypt(encrypted, 'my_sicret')
27 } catch (err) {
28 error = true
29 }
30
31 expect(error).to.be.true
32 })
33})
diff --git a/server/tests/helpers/index.ts b/server/tests/helpers/index.ts
index 951208842..1f0e3098a 100644
--- a/server/tests/helpers/index.ts
+++ b/server/tests/helpers/index.ts
@@ -1,6 +1,7 @@
1import './image' 1import './comment-model'
2import './core-utils' 2import './core-utils'
3import './crypto'
3import './dns' 4import './dns'
4import './comment-model' 5import './image'
5import './markdown' 6import './markdown'
6import './request' 7import './request'
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 663ac044a..d2072342e 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -1,18 +1,24 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' 4import { writeJson } from 'fs-extra'
5import { join } from 'path'
5import { HttpStatusCode, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
6import { expectLogDoesNotContain } from './shared' 8import { expectLogDoesNotContain } from './shared'
7 9
8describe('Test misc endpoints', function () { 10describe('Test misc endpoints', function () {
9 let server: PeerTubeServer 11 let server: PeerTubeServer
12 let wellKnownPath: string
10 13
11 before(async function () { 14 before(async function () {
12 this.timeout(120000) 15 this.timeout(120000)
13 16
14 server = await createSingleServer(1) 17 server = await createSingleServer(1)
18
15 await setAccessTokensToServers([ server ]) 19 await setAccessTokensToServers([ server ])
20
21 wellKnownPath = server.getDirectoryPath('well-known')
16 }) 22 })
17 23
18 describe('Test a well known endpoints', function () { 24 describe('Test a well known endpoints', function () {
@@ -93,6 +99,28 @@ describe('Test misc endpoints', function () {
93 expect(remoteInteract).to.exist 99 expect(remoteInteract).to.exist
94 expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') 100 expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}')
95 }) 101 })
102
103 it('Should return 404 for non-existing files in /.well-known', async function () {
104 await makeGetRequest({
105 url: server.url,
106 path: '/.well-known/non-existing-file',
107 expectedStatus: HttpStatusCode.NOT_FOUND_404
108 })
109 })
110
111 it('Should return custom file from /.well-known', async function () {
112 const filename = 'existing-file.json'
113
114 await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' })
115
116 const { body } = await makeGetRequest({
117 url: server.url,
118 path: '/.well-known/' + filename,
119 expectedStatus: HttpStatusCode.OK_200
120 })
121
122 expect(body.iThink).to.equal('therefore I am')
123 })
96 }) 124 })
97 125
98 describe('Test classic static endpoints', function () { 126 describe('Test classic static endpoints', function () {
diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts
index 026c7e856..ae4b3cf5f 100644
--- a/server/tests/plugins/filter-hooks.ts
+++ b/server/tests/plugins/filter-hooks.ts
@@ -64,232 +64,289 @@ describe('Test plugin filter hooks', function () {
64 }) 64 })
65 }) 65 })
66 66
67 it('Should run filter:api.videos.list.params', async function () { 67 describe('Videos', function () {
68 const { data } = await servers[0].videos.list({ start: 0, count: 2 })
69 68
70 // 2 plugins do +1 to the count parameter 69 it('Should run filter:api.videos.list.params', async function () {
71 expect(data).to.have.lengthOf(4) 70 const { data } = await servers[0].videos.list({ start: 0, count: 2 })
72 })
73 71
74 it('Should run filter:api.videos.list.result', async function () { 72 // 2 plugins do +1 to the count parameter
75 const { total } = await servers[0].videos.list({ start: 0, count: 0 }) 73 expect(data).to.have.lengthOf(4)
74 })
76 75
77 // Plugin do +1 to the total result 76 it('Should run filter:api.videos.list.result', async function () {
78 expect(total).to.equal(11) 77 const { total } = await servers[0].videos.list({ start: 0, count: 0 })
79 })
80 78
81 it('Should run filter:api.video-playlist.videos.list.params', async function () { 79 // Plugin do +1 to the total result
82 const { data } = await servers[0].playlists.listVideos({ 80 expect(total).to.equal(11)
83 count: 2,
84 playlistId: videoPlaylistUUID
85 }) 81 })
86 82
87 // 1 plugin do +1 to the count parameter 83 it('Should run filter:api.video-playlist.videos.list.params', async function () {
88 expect(data).to.have.lengthOf(3) 84 const { data } = await servers[0].playlists.listVideos({
89 }) 85 count: 2,
86 playlistId: videoPlaylistUUID
87 })
90 88
91 it('Should run filter:api.video-playlist.videos.list.result', async function () { 89 // 1 plugin do +1 to the count parameter
92 const { total } = await servers[0].playlists.listVideos({ 90 expect(data).to.have.lengthOf(3)
93 count: 0,
94 playlistId: videoPlaylistUUID
95 }) 91 })
96 92
97 // Plugin do +1 to the total result 93 it('Should run filter:api.video-playlist.videos.list.result', async function () {
98 expect(total).to.equal(11) 94 const { total } = await servers[0].playlists.listVideos({
99 }) 95 count: 0,
96 playlistId: videoPlaylistUUID
97 })
100 98
101 it('Should run filter:api.accounts.videos.list.params', async function () { 99 // Plugin do +1 to the total result
102 const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) 100 expect(total).to.equal(11)
101 })
103 102
104 // 1 plugin do +1 to the count parameter 103 it('Should run filter:api.accounts.videos.list.params', async function () {
105 expect(data).to.have.lengthOf(3) 104 const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 })
106 })
107 105
108 it('Should run filter:api.accounts.videos.list.result', async function () { 106 // 1 plugin do +1 to the count parameter
109 const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) 107 expect(data).to.have.lengthOf(3)
108 })
110 109
111 // Plugin do +2 to the total result 110 it('Should run filter:api.accounts.videos.list.result', async function () {
112 expect(total).to.equal(12) 111 const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 })
113 })
114 112
115 it('Should run filter:api.video-channels.videos.list.params', async function () { 113 // Plugin do +2 to the total result
116 const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) 114 expect(total).to.equal(12)
115 })
117 116
118 // 1 plugin do +3 to the count parameter 117 it('Should run filter:api.video-channels.videos.list.params', async function () {
119 expect(data).to.have.lengthOf(5) 118 const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 })
120 })
121 119
122 it('Should run filter:api.video-channels.videos.list.result', async function () { 120 // 1 plugin do +3 to the count parameter
123 const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) 121 expect(data).to.have.lengthOf(5)
122 })
124 123
125 // Plugin do +3 to the total result 124 it('Should run filter:api.video-channels.videos.list.result', async function () {
126 expect(total).to.equal(13) 125 const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 })
127 })
128 126
129 it('Should run filter:api.user.me.videos.list.params', async function () { 127 // Plugin do +3 to the total result
130 const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) 128 expect(total).to.equal(13)
129 })
131 130
132 // 1 plugin do +4 to the count parameter 131 it('Should run filter:api.user.me.videos.list.params', async function () {
133 expect(data).to.have.lengthOf(6) 132 const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 })
134 })
135 133
136 it('Should run filter:api.user.me.videos.list.result', async function () { 134 // 1 plugin do +4 to the count parameter
137 const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) 135 expect(data).to.have.lengthOf(6)
136 })
138 137
139 // Plugin do +4 to the total result 138 it('Should run filter:api.user.me.videos.list.result', async function () {
140 expect(total).to.equal(14) 139 const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 })
141 })
142 140
143 it('Should run filter:api.video.get.result', async function () { 141 // Plugin do +4 to the total result
144 const video = await servers[0].videos.get({ id: videoUUID }) 142 expect(total).to.equal(14)
145 expect(video.name).to.contain('<3') 143 })
146 })
147 144
148 it('Should run filter:api.video.upload.accept.result', async function () { 145 it('Should run filter:api.video.get.result', async function () {
149 await servers[0].videos.upload({ attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 146 const video = await servers[0].videos.get({ id: videoUUID })
147 expect(video.name).to.contain('<3')
148 })
150 }) 149 })
151 150
152 it('Should run filter:api.live-video.create.accept.result', async function () { 151 describe('Video/live/import accept', function () {
153 const attributes = {
154 name: 'video with bad word',
155 privacy: VideoPrivacy.PUBLIC,
156 channelId: servers[0].store.channel.id
157 }
158 152
159 await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 153 it('Should run filter:api.video.upload.accept.result', async function () {
160 }) 154 await servers[0].videos.upload({ attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
161 155 })
162 it('Should run filter:api.video.pre-import-url.accept.result', async function () {
163 const attributes = {
164 name: 'normal title',
165 privacy: VideoPrivacy.PUBLIC,
166 channelId: servers[0].store.channel.id,
167 targetUrl: FIXTURE_URLS.goodVideo + 'bad'
168 }
169 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
170 })
171 156
172 it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { 157 it('Should run filter:api.live-video.create.accept.result', async function () {
173 const attributes = { 158 const attributes = {
174 name: 'bad torrent', 159 name: 'video with bad word',
175 privacy: VideoPrivacy.PUBLIC, 160 privacy: VideoPrivacy.PUBLIC,
176 channelId: servers[0].store.channel.id, 161 channelId: servers[0].store.channel.id
177 torrentfile: 'video-720p.torrent' as any 162 }
178 }
179 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
180 })
181 163
182 it('Should run filter:api.video.post-import-url.accept.result', async function () { 164 await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
183 this.timeout(60000) 165 })
184 166
185 let videoImportId: number 167 it('Should run filter:api.video.pre-import-url.accept.result', async function () {
168 const attributes = {
169 name: 'normal title',
170 privacy: VideoPrivacy.PUBLIC,
171 channelId: servers[0].store.channel.id,
172 targetUrl: FIXTURE_URLS.goodVideo + 'bad'
173 }
174 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
175 })
186 176
187 { 177 it('Should run filter:api.video.pre-import-torrent.accept.result', async function () {
188 const attributes = { 178 const attributes = {
189 name: 'title with bad word', 179 name: 'bad torrent',
190 privacy: VideoPrivacy.PUBLIC, 180 privacy: VideoPrivacy.PUBLIC,
191 channelId: servers[0].store.channel.id, 181 channelId: servers[0].store.channel.id,
192 targetUrl: FIXTURE_URLS.goodVideo 182 torrentfile: 'video-720p.torrent' as any
193 } 183 }
194 const body = await servers[0].imports.importVideo({ attributes }) 184 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
195 videoImportId = body.id 185 })
196 }
197 186
198 await waitJobs(servers) 187 it('Should run filter:api.video.post-import-url.accept.result', async function () {
188 this.timeout(60000)
199 189
200 { 190 let videoImportId: number
201 const body = await servers[0].imports.getMyVideoImports()
202 const videoImports = body.data
203 191
204 const videoImport = videoImports.find(i => i.id === videoImportId) 192 {
193 const attributes = {
194 name: 'title with bad word',
195 privacy: VideoPrivacy.PUBLIC,
196 channelId: servers[0].store.channel.id,
197 targetUrl: FIXTURE_URLS.goodVideo
198 }
199 const body = await servers[0].imports.importVideo({ attributes })
200 videoImportId = body.id
201 }
205 202
206 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) 203 await waitJobs(servers)
207 expect(videoImport.state.label).to.equal('Rejected')
208 }
209 })
210 204
211 it('Should run filter:api.video.post-import-torrent.accept.result', async function () { 205 {
212 this.timeout(60000) 206 const body = await servers[0].imports.getMyVideoImports()
207 const videoImports = body.data
213 208
214 let videoImportId: number 209 const videoImport = videoImports.find(i => i.id === videoImportId)
215 210
216 { 211 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
217 const attributes = { 212 expect(videoImport.state.label).to.equal('Rejected')
218 name: 'title with bad word',
219 privacy: VideoPrivacy.PUBLIC,
220 channelId: servers[0].store.channel.id,
221 torrentfile: 'video-720p.torrent' as any
222 } 213 }
223 const body = await servers[0].imports.importVideo({ attributes }) 214 })
224 videoImportId = body.id
225 }
226 215
227 await waitJobs(servers) 216 it('Should run filter:api.video.post-import-torrent.accept.result', async function () {
217 this.timeout(60000)
228 218
229 { 219 let videoImportId: number
230 const { data: videoImports } = await servers[0].imports.getMyVideoImports()
231 220
232 const videoImport = videoImports.find(i => i.id === videoImportId) 221 {
222 const attributes = {
223 name: 'title with bad word',
224 privacy: VideoPrivacy.PUBLIC,
225 channelId: servers[0].store.channel.id,
226 torrentfile: 'video-720p.torrent' as any
227 }
228 const body = await servers[0].imports.importVideo({ attributes })
229 videoImportId = body.id
230 }
233 231
234 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) 232 await waitJobs(servers)
235 expect(videoImport.state.label).to.equal('Rejected') 233
236 } 234 {
237 }) 235 const { data: videoImports } = await servers[0].imports.getMyVideoImports()
236
237 const videoImport = videoImports.find(i => i.id === videoImportId)
238 238
239 it('Should run filter:api.video-thread.create.accept.result', async function () { 239 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
240 await servers[0].comments.createThread({ 240 expect(videoImport.state.label).to.equal('Rejected')
241 videoId: videoUUID, 241 }
242 text: 'comment with bad word',
243 expectedStatus: HttpStatusCode.FORBIDDEN_403
244 }) 242 })
245 }) 243 })
246 244
247 it('Should run filter:api.video-comment-reply.create.accept.result', async function () { 245 describe('Video comments accept', function () {
248 const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' })
249 threadId = created.id
250 246
251 await servers[0].comments.addReply({ 247 it('Should run filter:api.video-thread.create.accept.result', async function () {
252 videoId: videoUUID, 248 await servers[0].comments.createThread({
253 toCommentId: threadId, 249 videoId: videoUUID,
254 text: 'comment with bad word', 250 text: 'comment with bad word',
255 expectedStatus: HttpStatusCode.FORBIDDEN_403 251 expectedStatus: HttpStatusCode.FORBIDDEN_403
252 })
256 }) 253 })
257 await servers[0].comments.addReply({ 254
258 videoId: videoUUID, 255 it('Should run filter:api.video-comment-reply.create.accept.result', async function () {
259 toCommentId: threadId, 256 const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' })
260 text: 'comment with good word', 257 threadId = created.id
261 expectedStatus: HttpStatusCode.OK_200 258
259 await servers[0].comments.addReply({
260 videoId: videoUUID,
261 toCommentId: threadId,
262 text: 'comment with bad word',
263 expectedStatus: HttpStatusCode.FORBIDDEN_403
264 })
265 await servers[0].comments.addReply({
266 videoId: videoUUID,
267 toCommentId: threadId,
268 text: 'comment with good word',
269 expectedStatus: HttpStatusCode.OK_200
270 })
262 }) 271 })
263 })
264 272
265 it('Should run filter:api.video-threads.list.params', async function () { 273 it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a thread creation', async function () {
266 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) 274 this.timeout(30000)
267 275
268 // our plugin do +1 to the count parameter 276 await servers[1].comments.createThread({ videoId: videoUUID, text: 'comment with bad word' })
269 expect(data).to.have.lengthOf(1)
270 })
271 277
272 it('Should run filter:api.video-threads.list.result', async function () { 278 await waitJobs(servers)
273 const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 })
274 279
275 // Plugin do +1 to the total result 280 {
276 expect(total).to.equal(2) 281 const thread = await servers[0].comments.listThreads({ videoId: videoUUID })
277 }) 282 expect(thread.data).to.have.lengthOf(1)
283 expect(thread.data[0].text).to.not.include(' bad ')
284 }
285
286 {
287 const thread = await servers[1].comments.listThreads({ videoId: videoUUID })
288 expect(thread.data).to.have.lengthOf(2)
289 }
290 })
278 291
279 it('Should run filter:api.video-thread-comments.list.params') 292 it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a reply creation', async function () {
293 this.timeout(30000)
280 294
281 it('Should run filter:api.video-thread-comments.list.result', async function () { 295 const { data } = await servers[1].comments.listThreads({ videoId: videoUUID })
282 const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) 296 const threadIdServer2 = data.find(t => t.text === 'thread').id
283 297
284 expect(thread.comment.text.endsWith(' <3')).to.be.true 298 await servers[1].comments.addReply({
299 videoId: videoUUID,
300 toCommentId: threadIdServer2,
301 text: 'comment with bad word'
302 })
303
304 await waitJobs(servers)
305
306 {
307 const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId })
308 expect(tree.children).to.have.lengthOf(1)
309 expect(tree.children[0].comment.text).to.not.include(' bad ')
310 }
311
312 {
313 const tree = await servers[1].comments.getThread({ videoId: videoUUID, threadId: threadIdServer2 })
314 expect(tree.children).to.have.lengthOf(2)
315 }
316 })
285 }) 317 })
286 318
287 it('Should run filter:api.overviews.videos.list.{params,result}', async function () { 319 describe('Video comments', function () {
288 await servers[0].overviews.getVideos({ page: 1 }) 320
321 it('Should run filter:api.video-threads.list.params', async function () {
322 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 })
323
324 // our plugin do +1 to the count parameter
325 expect(data).to.have.lengthOf(1)
326 })
289 327
290 // 3 because we get 3 samples per page 328 it('Should run filter:api.video-threads.list.result', async function () {
291 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3) 329 const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 })
292 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3) 330
331 // Plugin do +1 to the total result
332 expect(total).to.equal(2)
333 })
334
335 it('Should run filter:api.video-thread-comments.list.params')
336
337 it('Should run filter:api.video-thread-comments.list.result', async function () {
338 const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId })
339
340 expect(thread.comment.text.endsWith(' <3')).to.be.true
341 })
342
343 it('Should run filter:api.overviews.videos.list.{params,result}', async function () {
344 await servers[0].overviews.getVideos({ page: 1 })
345
346 // 3 because we get 3 samples per page
347 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3)
348 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3)
349 })
293 }) 350 })
294 351
295 describe('filter:video.auto-blacklist.result', function () { 352 describe('filter:video.auto-blacklist.result', function () {
diff --git a/server/tests/plugins/index.ts b/server/tests/plugins/index.ts
index 4534120fd..210af7236 100644
--- a/server/tests/plugins/index.ts
+++ b/server/tests/plugins/index.ts
@@ -8,5 +8,6 @@ import './plugin-router'
8import './plugin-storage' 8import './plugin-storage'
9import './plugin-transcoding' 9import './plugin-transcoding'
10import './plugin-unloading' 10import './plugin-unloading'
11import './plugin-websocket'
11import './translations' 12import './translations'
12import './video-constants' 13import './video-constants'
diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts
index 955d7ddfd..31c18350a 100644
--- a/server/tests/plugins/plugin-helpers.ts
+++ b/server/tests/plugins/plugin-helpers.ts
@@ -83,6 +83,33 @@ describe('Test plugin helpers', function () {
83 }) 83 })
84 }) 84 })
85 85
86 describe('Socket', function () {
87
88 it('Should sendNotification without any exceptions', async () => {
89 const user = await servers[0].users.create({ username: 'notis_redding', password: 'secret1234?' })
90 await makePostBodyRequest({
91 url: servers[0].url,
92 path: '/plugins/test-four/router/send-notification',
93 fields: {
94 userId: user.id
95 },
96 expectedStatus: HttpStatusCode.CREATED_201
97 })
98 })
99
100 it('Should sendVideoLiveNewState without any exceptions', async () => {
101 const res = await servers[0].videos.quickUpload({ name: 'video server 1' })
102
103 await makePostBodyRequest({
104 url: servers[0].url,
105 path: '/plugins/test-four/router/send-video-live-new-state/' + res.uuid,
106 expectedStatus: HttpStatusCode.CREATED_201
107 })
108
109 await servers[0].videos.remove({ id: res.uuid })
110 })
111 })
112
86 describe('Plugin', function () { 113 describe('Plugin', function () {
87 114
88 it('Should get the base static route', async function () { 115 it('Should get the base static route', async function () {
diff --git a/server/tests/plugins/plugin-websocket.ts b/server/tests/plugins/plugin-websocket.ts
new file mode 100644
index 000000000..adaa28b1d
--- /dev/null
+++ b/server/tests/plugins/plugin-websocket.ts
@@ -0,0 +1,70 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import WebSocket from 'ws'
4import { cleanupTests, createSingleServer, PeerTubeServer, PluginsCommand, setAccessTokensToServers } from '@shared/server-commands'
5
6function buildWebSocket (server: PeerTubeServer, path: string) {
7 return new WebSocket('ws://' + server.host + path)
8}
9
10function expectErrorOrTimeout (server: PeerTubeServer, path: string, expectedTimeout: number) {
11 return new Promise<void>((res, rej) => {
12 const ws = buildWebSocket(server, path)
13 ws.on('error', () => res())
14
15 const timeout = setTimeout(() => res(), expectedTimeout)
16
17 ws.on('open', () => {
18 clearTimeout(timeout)
19
20 return rej(new Error('Connect did not timeout'))
21 })
22 })
23}
24
25describe('Test plugin websocket', function () {
26 let server: PeerTubeServer
27 const basePaths = [
28 '/plugins/test-websocket/ws/',
29 '/plugins/test-websocket/0.0.1/ws/'
30 ]
31
32 before(async function () {
33 this.timeout(30000)
34
35 server = await createSingleServer(1)
36 await setAccessTokensToServers([ server ])
37
38 await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-websocket') })
39 })
40
41 it('Should not connect to the websocket without the appropriate path', async function () {
42 const paths = [
43 '/plugins/unknown/ws/',
44 '/plugins/unknown/0.0.1/ws/'
45 ]
46
47 for (const path of paths) {
48 await expectErrorOrTimeout(server, path, 1000)
49 }
50 })
51
52 it('Should not connect to the websocket without the appropriate sub path', async function () {
53 for (const path of basePaths) {
54 await expectErrorOrTimeout(server, path + '/unknown', 1000)
55 }
56 })
57
58 it('Should connect to the websocket and receive pong', function (done) {
59 const ws = buildWebSocket(server, basePaths[0])
60
61 ws.on('open', () => ws.send('ping'))
62 ws.on('message', data => {
63 if (data.toString() === 'pong') return done()
64 })
65 })
66
67 after(async function () {
68 await cleanupTests([ server ])
69 })
70})
diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts
index f8f4a5137..41fd72e89 100644
--- a/server/tests/shared/actors.ts
+++ b/server/tests/shared/actors.ts
@@ -2,8 +2,6 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir } from 'fs-extra' 4import { pathExists, readdir } from 'fs-extra'
5import { join } from 'path'
6import { root } from '@shared/core-utils'
7import { Account, VideoChannel } from '@shared/models' 5import { Account, VideoChannel } from '@shared/models'
8import { PeerTubeServer } from '@shared/server-commands' 6import { PeerTubeServer } from '@shared/server-commands'
9 7
@@ -31,11 +29,9 @@ async function expectAccountFollows (options: {
31 return expectActorFollow({ ...options, data }) 29 return expectActorFollow({ ...options, data })
32} 30}
33 31
34async function checkActorFilesWereRemoved (filename: string, serverNumber: number) { 32async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) {
35 const testDirectory = 'test' + serverNumber
36
37 for (const directory of [ 'avatars' ]) { 33 for (const directory of [ 'avatars' ]) {
38 const directoryPath = join(root(), testDirectory, directory) 34 const directoryPath = server.getDirectoryPath(directory)
39 35
40 const directoryExists = await pathExists(directoryPath) 36 const directoryExists = await pathExists(directoryPath)
41 expect(directoryExists).to.be.true 37 expect(directoryExists).to.be.true
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index c7065a767..90d534a06 100644
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -2,22 +2,18 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir } from 'fs-extra' 4import { pathExists, readdir } from 'fs-extra'
5import { join } from 'path'
6import { root } from '@shared/core-utils'
7import { PeerTubeServer } from '@shared/server-commands' 5import { PeerTubeServer } from '@shared/server-commands'
8 6
9async function checkTmpIsEmpty (server: PeerTubeServer) { 7async function checkTmpIsEmpty (server: PeerTubeServer) {
10 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) 8 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
11 9
12 if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { 10 if (await pathExists(server.getDirectoryPath('tmp/hls'))) {
13 await checkDirectoryIsEmpty(server, 'tmp/hls') 11 await checkDirectoryIsEmpty(server, 'tmp/hls')
14 } 12 }
15} 13}
16 14
17async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { 15async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
18 const testDirectory = 'test' + server.internalServerNumber 16 const directoryPath = server.getDirectoryPath(directory)
19
20 const directoryPath = join(root(), testDirectory, directory)
21 17
22 const directoryExists = await pathExists(directoryPath) 18 const directoryExists = await pathExists(directoryPath)
23 expect(directoryExists).to.be.true 19 expect(directoryExists).to.be.true
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts
index 4bd4786fc..f165832fe 100644
--- a/server/tests/shared/live.ts
+++ b/server/tests/shared/live.ts
@@ -3,39 +3,95 @@
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir } from 'fs-extra' 4import { pathExists, readdir } from 'fs-extra'
5import { join } from 'path' 5import { join } from 'path'
6import { LiveVideo } from '@shared/models' 6import { wait } from '@shared/core-utils'
7import { PeerTubeServer } from '@shared/server-commands' 7import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models'
8import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands'
9import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists'
8 10
9async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) { 11async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) {
10 let live: LiveVideo
11
12 try {
13 live = await server.live.get({ videoId: videoUUID })
14 } catch {}
15
16 const basePath = server.servers.buildDirectory('streaming-playlists') 12 const basePath = server.servers.buildDirectory('streaming-playlists')
17 const hlsPath = join(basePath, 'hls', videoUUID) 13 const hlsPath = join(basePath, 'hls', videoUUID)
18 14
19 if (savedResolutions.length === 0) { 15 if (savedResolutions.length === 0) {
16 return checkUnsavedLiveCleanup(server, videoUUID, hlsPath)
17 }
18
19 return checkSavedLiveCleanup(hlsPath, savedResolutions)
20}
21
22// ---------------------------------------------------------------------------
20 23
21 if (live?.permanentLive) { 24async function testVideoResolutions (options: {
22 expect(await pathExists(hlsPath)).to.be.true 25 originServer: PeerTubeServer
26 servers: PeerTubeServer[]
27 liveVideoId: string
28 resolutions: number[]
29 transcoded: boolean
30 objectStorage: boolean
31}) {
32 const { originServer, servers, liveVideoId, resolutions, transcoded, objectStorage } = options
23 33
24 const hlsFiles = await readdir(hlsPath) 34 for (const server of servers) {
25 expect(hlsFiles).to.have.lengthOf(1) // Only replays directory 35 const { data } = await server.videos.list()
36 expect(data.find(v => v.uuid === liveVideoId)).to.exist
26 37
27 const replayDir = join(hlsPath, 'replay') 38 const video = await server.videos.get({ id: liveVideoId })
28 expect(await pathExists(replayDir)).to.be.true 39 expect(video.streamingPlaylists).to.have.lengthOf(1)
29 40
30 const replayFiles = await readdir(join(hlsPath, 'replay')) 41 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
31 expect(replayFiles).to.have.lengthOf(0) 42 expect(hlsPlaylist).to.exist
32 } else { 43 expect(hlsPlaylist.files).to.have.lengthOf(0) // Only fragmented mp4 files are displayed
33 expect(await pathExists(hlsPath)).to.be.false 44
45 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions, transcoded })
46
47 if (objectStorage) {
48 expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getPlaylistBaseUrl())
34 } 49 }
35 50
36 return 51 for (let i = 0; i < resolutions.length; i++) {
52 const segmentNum = 3
53 const segmentName = `${i}-00000${segmentNum}.ts`
54 await originServer.live.waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum })
55
56 const baseUrl = objectStorage
57 ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls'
58 : originServer.url + '/static/streaming-playlists/hls'
59
60 if (objectStorage) {
61 await originServer.live.waitUntilSegmentUpload({ playlistNumber: i, segment: segmentNum })
62 await wait(1000)
63
64 expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getPlaylistBaseUrl())
65 }
66
67 const subPlaylist = await originServer.streamingPlaylists.get({
68 url: `${baseUrl}/${video.uuid}/${i}.m3u8`,
69 withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
70 })
71
72 expect(subPlaylist).to.contain(segmentName)
73
74 await checkLiveSegmentHash({
75 server,
76 baseUrlSegment: baseUrl,
77 videoUUID: video.uuid,
78 segmentName,
79 hlsPlaylist
80 })
81 }
37 } 82 }
83}
84
85// ---------------------------------------------------------------------------
86
87export {
88 checkLiveCleanup,
89 testVideoResolutions
90}
38 91
92// ---------------------------------------------------------------------------
93
94async function checkSavedLiveCleanup (hlsPath: string, savedResolutions: number[] = []) {
39 const files = await readdir(hlsPath) 95 const files = await readdir(hlsPath)
40 96
41 // fragmented file and playlist per resolution + master playlist + segments sha256 json file 97 // fragmented file and playlist per resolution + master playlist + segments sha256 json file
@@ -56,6 +112,27 @@ async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, save
56 expect(shaFile).to.exist 112 expect(shaFile).to.exist
57} 113}
58 114
59export { 115async function checkUnsavedLiveCleanup (server: PeerTubeServer, videoUUID: string, hlsPath: string) {
60 checkLiveCleanup 116 let live: LiveVideo
117
118 try {
119 live = await server.live.get({ videoId: videoUUID })
120 } catch {}
121
122 if (live?.permanentLive) {
123 expect(await pathExists(hlsPath)).to.be.true
124
125 const hlsFiles = await readdir(hlsPath)
126 expect(hlsFiles).to.have.lengthOf(1) // Only replays directory
127
128 const replayDir = join(hlsPath, 'replay')
129 expect(await pathExists(replayDir)).to.be.true
130
131 const replayFiles = await readdir(join(hlsPath, 'replay'))
132 expect(replayFiles).to.have.lengthOf(0)
133
134 return
135 }
136
137 expect(await pathExists(hlsPath)).to.be.false
61} 138}
diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts
index fdd541d20..8db303fd8 100644
--- a/server/tests/shared/playlists.ts
+++ b/server/tests/shared/playlists.ts
@@ -1,17 +1,14 @@
1import { expect } from 'chai' 1import { expect } from 'chai'
2import { readdir } from 'fs-extra' 2import { readdir } from 'fs-extra'
3import { join } from 'path' 3import { PeerTubeServer } from '@shared/server-commands'
4import { root } from '@shared/core-utils'
5 4
6async function checkPlaylistFilesWereRemoved ( 5async function checkPlaylistFilesWereRemoved (
7 playlistUUID: string, 6 playlistUUID: string,
8 internalServerNumber: number, 7 server: PeerTubeServer,
9 directories = [ 'thumbnails' ] 8 directories = [ 'thumbnails' ]
10) { 9) {
11 const testDirectory = 'test' + internalServerNumber
12
13 for (const directory of directories) { 10 for (const directory of directories) {
14 const directoryPath = join(root(), testDirectory, directory) 11 const directoryPath = server.getDirectoryPath(directory)
15 12
16 const files = await readdir(directoryPath) 13 const files = await readdir(directoryPath)
17 for (const file of files) { 14 for (const file of files) {
diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts
index 4d82b3654..eff34944b 100644
--- a/server/tests/shared/streaming-playlists.ts
+++ b/server/tests/shared/streaming-playlists.ts
@@ -26,7 +26,7 @@ async function checkSegmentHash (options: {
26 const offset = parseInt(matches[2], 10) 26 const offset = parseInt(matches[2], 10)
27 const range = `${offset}-${offset + length - 1}` 27 const range = `${offset}-${offset + length - 1}`
28 28
29 const segmentBody = await command.getSegment({ 29 const segmentBody = await command.getFragmentedSegment({
30 url: `${baseUrlSegment}/${videoName}`, 30 url: `${baseUrlSegment}/${videoName}`,
31 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, 31 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206,
32 range: `bytes=${range}` 32 range: `bytes=${range}`
@@ -46,7 +46,7 @@ async function checkLiveSegmentHash (options: {
46 const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options 46 const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options
47 const command = server.streamingPlaylists 47 const command = server.streamingPlaylists
48 48
49 const segmentBody = await command.getSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) 49 const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` })
50 const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) 50 const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
51 51
52 expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) 52 expect(sha256(segmentBody)).to.equal(shaBody[segmentName])
@@ -56,15 +56,16 @@ async function checkResolutionsInMasterPlaylist (options: {
56 server: PeerTubeServer 56 server: PeerTubeServer
57 playlistUrl: string 57 playlistUrl: string
58 resolutions: number[] 58 resolutions: number[]
59 transcoded?: boolean // default true
59}) { 60}) {
60 const { server, playlistUrl, resolutions } = options 61 const { server, playlistUrl, resolutions, transcoded = true } = options
61 62
62 const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl }) 63 const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl })
63 64
64 for (const resolution of resolutions) { 65 for (const resolution of resolutions) {
65 const reg = new RegExp( 66 const reg = transcoded
66 '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"' 67 ? new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"')
67 ) 68 : new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + '')
68 69
69 expect(masterPlaylist).to.match(reg) 70 expect(masterPlaylist).to.match(reg)
70 } 71 }
diff --git a/server/types/plugins/index.ts b/server/types/plugins/index.ts
index de30ff2ab..bf9c35d49 100644
--- a/server/types/plugins/index.ts
+++ b/server/types/plugins/index.ts
@@ -1,3 +1,4 @@
1export * from './plugin-library.model' 1export * from './plugin-library.model'
2export * from './register-server-auth.model' 2export * from './register-server-auth.model'
3export * from './register-server-option.model' 3export * from './register-server-option.model'
4export * from './register-server-websocket-route.model'
diff --git a/server/types/plugins/register-server-option.model.ts b/server/types/plugins/register-server-option.model.ts
index fb4f12a4c..1e2bd830e 100644
--- a/server/types/plugins/register-server-option.model.ts
+++ b/server/types/plugins/register-server-option.model.ts
@@ -1,4 +1,5 @@
1import { Response, Router } from 'express' 1import { Response, Router } from 'express'
2import { Server } from 'http'
2import { Logger } from 'winston' 3import { Logger } from 'winston'
3import { ActorModel } from '@server/models/actor/actor' 4import { ActorModel } from '@server/models/actor/actor'
4import { 5import {
@@ -16,12 +17,13 @@ import {
16 ThumbnailType, 17 ThumbnailType,
17 VideoBlacklistCreate 18 VideoBlacklistCreate
18} from '@shared/models' 19} from '@shared/models'
19import { MUserDefault, MVideoThumbnail } from '../models' 20import { MUserDefault, MVideo, MVideoThumbnail, UserNotificationModelForApi } from '../models'
20import { 21import {
21 RegisterServerAuthExternalOptions, 22 RegisterServerAuthExternalOptions,
22 RegisterServerAuthExternalResult, 23 RegisterServerAuthExternalResult,
23 RegisterServerAuthPassOptions 24 RegisterServerAuthPassOptions
24} from './register-server-auth.model' 25} from './register-server-auth.model'
26import { RegisterServerWebSocketRouteOptions } from './register-server-websocket-route.model'
25 27
26export type PeerTubeHelpers = { 28export type PeerTubeHelpers = {
27 logger: Logger 29 logger: Logger
@@ -83,15 +85,25 @@ export type PeerTubeHelpers = {
83 } 85 }
84 86
85 server: { 87 server: {
88 // PeerTube >= 5.0
89 getHTTPServer: () => Server
90
86 getServerActor: () => Promise<ActorModel> 91 getServerActor: () => Promise<ActorModel>
87 } 92 }
88 93
94 socket: {
95 sendNotification: (userId: number, notification: UserNotificationModelForApi) => void
96 sendVideoLiveNewState: (video: MVideo) => void
97 }
98
89 plugin: { 99 plugin: {
90 // PeerTube >= 3.2 100 // PeerTube >= 3.2
91 getBaseStaticRoute: () => string 101 getBaseStaticRoute: () => string
92 102
93 // PeerTube >= 3.2 103 // PeerTube >= 3.2
94 getBaseRouterRoute: () => string 104 getBaseRouterRoute: () => string
105 // PeerTube >= 5.0
106 getBaseWebSocketRoute: () => string
95 107
96 // PeerTube >= 3.2 108 // PeerTube >= 3.2
97 getDataDirectoryPath: () => string 109 getDataDirectoryPath: () => string
@@ -135,5 +147,12 @@ export type RegisterServerOptions = {
135 // * /plugins/:pluginName/router/... 147 // * /plugins/:pluginName/router/...
136 getRouter(): Router 148 getRouter(): Router
137 149
150 // PeerTube >= 5.0
151 // Register WebSocket route
152 // Base routes of the WebSocket router are
153 // * /plugins/:pluginName/:pluginVersion/ws/...
154 // * /plugins/:pluginName/ws/...
155 registerWebSocketRoute: (options: RegisterServerWebSocketRouteOptions) => void
156
138 peertubeHelpers: PeerTubeHelpers 157 peertubeHelpers: PeerTubeHelpers
139} 158}
diff --git a/server/types/plugins/register-server-websocket-route.model.ts b/server/types/plugins/register-server-websocket-route.model.ts
new file mode 100644
index 000000000..edf64f66b
--- /dev/null
+++ b/server/types/plugins/register-server-websocket-route.model.ts
@@ -0,0 +1,8 @@
1import { IncomingMessage } from 'http'
2import { Duplex } from 'stream'
3
4export type RegisterServerWebSocketRouteOptions = {
5 route: string
6
7 handler: (request: IncomingMessage, socket: Duplex, head: Buffer) => any
8}