diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/index.ts | 4 | ||||
-rw-r--r-- | server/controllers/api/users/index.ts | 12 | ||||
-rw-r--r-- | server/controllers/api/users/token.ts | 5 | ||||
-rw-r--r-- | server/controllers/feeds.ts | 10 | ||||
-rw-r--r-- | server/helpers/audit-logger.ts | 12 | ||||
-rw-r--r-- | server/helpers/upload.ts | 9 | ||||
-rw-r--r-- | server/initializers/checker-before-init.ts | 5 | ||||
-rw-r--r-- | server/initializers/config.ts | 9 | ||||
-rw-r--r-- | server/initializers/constants.ts | 11 | ||||
-rw-r--r-- | server/lib/client-html.ts | 6 | ||||
-rw-r--r-- | server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts | 2 | ||||
-rw-r--r-- | server/lib/schedulers/geo-ip-update-scheduler.ts | 2 | ||||
-rw-r--r-- | server/middlewares/index.ts | 1 | ||||
-rw-r--r-- | server/middlewares/rate-limiter.ts | 31 | ||||
-rw-r--r-- | server/middlewares/validators/sort.ts | 4 | ||||
-rw-r--r-- | server/models/user/user.ts | 6 | ||||
-rw-r--r-- | server/models/utils.ts | 22 | ||||
-rw-r--r-- | server/models/video/video-channel.ts | 13 | ||||
-rw-r--r-- | server/tests/api/live/live.ts | 2 | ||||
-rw-r--r-- | server/tests/api/server/reverse-proxy.ts | 11 | ||||
-rw-r--r-- | server/tests/api/videos/video-channels.ts | 19 |
21 files changed, 143 insertions, 53 deletions
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index 5f49336b1..d1d4ef765 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import cors from 'cors' | 1 | import cors from 'cors' |
2 | import express from 'express' | 2 | import express from 'express' |
3 | import RateLimit from 'express-rate-limit' | 3 | import { buildRateLimiter } from '@server/middlewares' |
4 | import { HttpStatusCode } from '../../../shared/models' | 4 | import { HttpStatusCode } from '../../../shared/models' |
5 | import { badRequest } from '../../helpers/express-utils' | 5 | import { badRequest } from '../../helpers/express-utils' |
6 | import { CONFIG } from '../../initializers/config' | 6 | import { CONFIG } from '../../initializers/config' |
@@ -29,7 +29,7 @@ apiRouter.use(cors({ | |||
29 | credentials: true | 29 | credentials: true |
30 | })) | 30 | })) |
31 | 31 | ||
32 | const apiRateLimiter = RateLimit({ | 32 | const apiRateLimiter = buildRateLimiter({ |
33 | windowMs: CONFIG.RATES_LIMIT.API.WINDOW_MS, | 33 | windowMs: CONFIG.RATES_LIMIT.API.WINDOW_MS, |
34 | max: CONFIG.RATES_LIMIT.API.MAX | 34 | max: CONFIG.RATES_LIMIT.API.MAX |
35 | }) | 35 | }) |
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 8a06bfe93..46e80d56d 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import RateLimit from 'express-rate-limit' | ||
3 | import { tokensRouter } from '@server/controllers/api/users/token' | 2 | import { tokensRouter } from '@server/controllers/api/users/token' |
4 | import { Hooks } from '@server/lib/plugins/hooks' | 3 | import { Hooks } from '@server/lib/plugins/hooks' |
5 | import { OAuthTokenModel } from '@server/models/oauth/oauth-token' | 4 | import { OAuthTokenModel } from '@server/models/oauth/oauth-token' |
@@ -17,9 +16,11 @@ import { Notifier } from '../../../lib/notifier' | |||
17 | import { Redis } from '../../../lib/redis' | 16 | import { Redis } from '../../../lib/redis' |
18 | import { buildUser, createUserAccountAndChannelAndPlaylist, sendVerifyUserEmail } from '../../../lib/user' | 17 | import { buildUser, createUserAccountAndChannelAndPlaylist, sendVerifyUserEmail } from '../../../lib/user' |
19 | import { | 18 | import { |
19 | adminUsersSortValidator, | ||
20 | asyncMiddleware, | 20 | asyncMiddleware, |
21 | asyncRetryTransactionMiddleware, | 21 | asyncRetryTransactionMiddleware, |
22 | authenticate, | 22 | authenticate, |
23 | buildRateLimiter, | ||
23 | ensureUserHasRight, | 24 | ensureUserHasRight, |
24 | ensureUserRegistrationAllowed, | 25 | ensureUserRegistrationAllowed, |
25 | ensureUserRegistrationAllowedForIP, | 26 | ensureUserRegistrationAllowedForIP, |
@@ -32,7 +33,6 @@ import { | |||
32 | usersListValidator, | 33 | usersListValidator, |
33 | usersRegisterValidator, | 34 | usersRegisterValidator, |
34 | usersRemoveValidator, | 35 | usersRemoveValidator, |
35 | usersSortValidator, | ||
36 | usersUpdateValidator | 36 | usersUpdateValidator |
37 | } from '../../../middlewares' | 37 | } from '../../../middlewares' |
38 | import { | 38 | import { |
@@ -54,13 +54,13 @@ import { myVideoPlaylistsRouter } from './my-video-playlists' | |||
54 | 54 | ||
55 | const auditLogger = auditLoggerFactory('users') | 55 | const auditLogger = auditLoggerFactory('users') |
56 | 56 | ||
57 | const signupRateLimiter = RateLimit({ | 57 | const signupRateLimiter = buildRateLimiter({ |
58 | windowMs: CONFIG.RATES_LIMIT.SIGNUP.WINDOW_MS, | 58 | windowMs: CONFIG.RATES_LIMIT.SIGNUP.WINDOW_MS, |
59 | max: CONFIG.RATES_LIMIT.SIGNUP.MAX, | 59 | max: CONFIG.RATES_LIMIT.SIGNUP.MAX, |
60 | skipFailedRequests: true | 60 | skipFailedRequests: true |
61 | }) | 61 | }) |
62 | 62 | ||
63 | const askSendEmailLimiter = RateLimit({ | 63 | const askSendEmailLimiter = buildRateLimiter({ |
64 | windowMs: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.WINDOW_MS, | 64 | windowMs: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.WINDOW_MS, |
65 | max: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.MAX | 65 | max: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.MAX |
66 | }) | 66 | }) |
@@ -84,7 +84,7 @@ usersRouter.get('/', | |||
84 | authenticate, | 84 | authenticate, |
85 | ensureUserHasRight(UserRight.MANAGE_USERS), | 85 | ensureUserHasRight(UserRight.MANAGE_USERS), |
86 | paginationValidator, | 86 | paginationValidator, |
87 | usersSortValidator, | 87 | adminUsersSortValidator, |
88 | setDefaultSort, | 88 | setDefaultSort, |
89 | setDefaultPagination, | 89 | setDefaultPagination, |
90 | usersListValidator, | 90 | usersListValidator, |
@@ -277,7 +277,7 @@ async function autocompleteUsers (req: express.Request, res: express.Response) { | |||
277 | } | 277 | } |
278 | 278 | ||
279 | async function listUsers (req: express.Request, res: express.Response) { | 279 | async function listUsers (req: express.Request, res: express.Response) { |
280 | const resultList = await UserModel.listForApi({ | 280 | const resultList = await UserModel.listForAdminApi({ |
281 | start: req.query.start, | 281 | start: req.query.start, |
282 | count: req.query.count, | 282 | count: req.query.count, |
283 | sort: req.query.sort, | 283 | sort: req.query.sort, |
diff --git a/server/controllers/api/users/token.ts b/server/controllers/api/users/token.ts index 258b50fe9..012a49791 100644 --- a/server/controllers/api/users/token.ts +++ b/server/controllers/api/users/token.ts | |||
@@ -1,18 +1,17 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import RateLimit from 'express-rate-limit' | ||
3 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
4 | import { CONFIG } from '@server/initializers/config' | 3 | import { CONFIG } from '@server/initializers/config' |
5 | import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' | 4 | import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' |
6 | import { handleOAuthToken } from '@server/lib/auth/oauth' | 5 | import { handleOAuthToken } from '@server/lib/auth/oauth' |
7 | import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' | 6 | import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' |
8 | import { Hooks } from '@server/lib/plugins/hooks' | 7 | import { Hooks } from '@server/lib/plugins/hooks' |
9 | import { asyncMiddleware, authenticate, openapiOperationDoc } from '@server/middlewares' | 8 | import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares' |
10 | import { buildUUID } from '@shared/extra-utils' | 9 | import { buildUUID } from '@shared/extra-utils' |
11 | import { ScopedToken } from '@shared/models/users/user-scoped-token' | 10 | import { ScopedToken } from '@shared/models/users/user-scoped-token' |
12 | 11 | ||
13 | const tokensRouter = express.Router() | 12 | const tokensRouter = express.Router() |
14 | 13 | ||
15 | const loginRateLimiter = RateLimit({ | 14 | const loginRateLimiter = buildRateLimiter({ |
16 | windowMs: CONFIG.RATES_LIMIT.LOGIN.WINDOW_MS, | 15 | windowMs: CONFIG.RATES_LIMIT.LOGIN.WINDOW_MS, |
17 | max: CONFIG.RATES_LIMIT.LOGIN.MAX | 16 | max: CONFIG.RATES_LIMIT.LOGIN.MAX |
18 | }) | 17 | }) |
diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts index c929a6726..9eb31ed93 100644 --- a/server/controllers/feeds.ts +++ b/server/controllers/feeds.ts | |||
@@ -1,13 +1,13 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { Feed } from '@peertube/feed' | ||
3 | import { extname } from 'path' | 2 | import { extname } from 'path' |
3 | import { Feed } from '@peertube/feed' | ||
4 | import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' | 4 | import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' |
5 | import { getServerActor } from '@server/models/application/application' | 5 | import { getServerActor } from '@server/models/application/application' |
6 | import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils' | 6 | import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils' |
7 | import { VideoInclude } from '@shared/models' | 7 | import { VideoInclude } from '@shared/models' |
8 | import { buildNSFWFilter } from '../helpers/express-utils' | 8 | import { buildNSFWFilter } from '../helpers/express-utils' |
9 | import { CONFIG } from '../initializers/config' | 9 | import { CONFIG } from '../initializers/config' |
10 | import { FEEDS, MIMETYPES, PREVIEWS_SIZE, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' | 10 | import { MIMETYPES, PREVIEWS_SIZE, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' |
11 | import { | 11 | import { |
12 | asyncMiddleware, | 12 | asyncMiddleware, |
13 | commonVideosFiltersValidator, | 13 | commonVideosFiltersValidator, |
@@ -76,7 +76,7 @@ async function generateVideoCommentsFeed (req: express.Request, res: express.Res | |||
76 | 76 | ||
77 | const comments = await VideoCommentModel.listForFeed({ | 77 | const comments = await VideoCommentModel.listForFeed({ |
78 | start, | 78 | start, |
79 | count: FEEDS.COUNT, | 79 | count: CONFIG.FEEDS.COMMENTS.COUNT, |
80 | videoId: video ? video.id : undefined, | 80 | videoId: video ? video.id : undefined, |
81 | accountId: account ? account.id : undefined, | 81 | accountId: account ? account.id : undefined, |
82 | videoChannelId: videoChannel ? videoChannel.id : undefined | 82 | videoChannelId: videoChannel ? videoChannel.id : undefined |
@@ -166,7 +166,7 @@ async function generateVideoFeed (req: express.Request, res: express.Response) { | |||
166 | const server = await getServerActor() | 166 | const server = await getServerActor() |
167 | const { data } = await VideoModel.listForApi({ | 167 | const { data } = await VideoModel.listForApi({ |
168 | start, | 168 | start, |
169 | count: FEEDS.COUNT, | 169 | count: CONFIG.FEEDS.VIDEOS.COUNT, |
170 | sort: req.query.sort, | 170 | sort: req.query.sort, |
171 | displayOnlyForFollower: { | 171 | displayOnlyForFollower: { |
172 | actorId: server.id, | 172 | actorId: server.id, |
@@ -202,7 +202,7 @@ async function generateVideoFeedForSubscriptions (req: express.Request, res: exp | |||
202 | 202 | ||
203 | const { data } = await VideoModel.listForApi({ | 203 | const { data } = await VideoModel.listForApi({ |
204 | start, | 204 | start, |
205 | count: FEEDS.COUNT, | 205 | count: CONFIG.FEEDS.VIDEOS.COUNT, |
206 | sort: req.query.sort, | 206 | sort: req.query.sort, |
207 | nsfw, | 207 | nsfw, |
208 | 208 | ||
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index 79ef44be1..076b7f11d 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts | |||
@@ -120,7 +120,7 @@ const videoKeysToKeep = [ | |||
120 | 'downloadEnabled' | 120 | 'downloadEnabled' |
121 | ] | 121 | ] |
122 | class VideoAuditView extends EntityAuditView { | 122 | class VideoAuditView extends EntityAuditView { |
123 | constructor (private readonly video: VideoDetails) { | 123 | constructor (video: VideoDetails) { |
124 | super(videoKeysToKeep, 'video', video) | 124 | super(videoKeysToKeep, 'video', video) |
125 | } | 125 | } |
126 | } | 126 | } |
@@ -131,7 +131,7 @@ const videoImportKeysToKeep = [ | |||
131 | 'video-name' | 131 | 'video-name' |
132 | ] | 132 | ] |
133 | class VideoImportAuditView extends EntityAuditView { | 133 | class VideoImportAuditView extends EntityAuditView { |
134 | constructor (private readonly videoImport: VideoImport) { | 134 | constructor (videoImport: VideoImport) { |
135 | super(videoImportKeysToKeep, 'video-import', videoImport) | 135 | super(videoImportKeysToKeep, 'video-import', videoImport) |
136 | } | 136 | } |
137 | } | 137 | } |
@@ -150,7 +150,7 @@ const commentKeysToKeep = [ | |||
150 | 'account-name' | 150 | 'account-name' |
151 | ] | 151 | ] |
152 | class CommentAuditView extends EntityAuditView { | 152 | class CommentAuditView extends EntityAuditView { |
153 | constructor (private readonly comment: VideoComment) { | 153 | constructor (comment: VideoComment) { |
154 | super(commentKeysToKeep, 'comment', comment) | 154 | super(commentKeysToKeep, 'comment', comment) |
155 | } | 155 | } |
156 | } | 156 | } |
@@ -179,7 +179,7 @@ const userKeysToKeep = [ | |||
179 | 'videoChannels' | 179 | 'videoChannels' |
180 | ] | 180 | ] |
181 | class UserAuditView extends EntityAuditView { | 181 | class UserAuditView extends EntityAuditView { |
182 | constructor (private readonly user: User) { | 182 | constructor (user: User) { |
183 | super(userKeysToKeep, 'user', user) | 183 | super(userKeysToKeep, 'user', user) |
184 | } | 184 | } |
185 | } | 185 | } |
@@ -205,7 +205,7 @@ const channelKeysToKeep = [ | |||
205 | 'ownerAccount-displayedName' | 205 | 'ownerAccount-displayedName' |
206 | ] | 206 | ] |
207 | class VideoChannelAuditView extends EntityAuditView { | 207 | class VideoChannelAuditView extends EntityAuditView { |
208 | constructor (private readonly channel: VideoChannel) { | 208 | constructor (channel: VideoChannel) { |
209 | super(channelKeysToKeep, 'channel', channel) | 209 | super(channelKeysToKeep, 'channel', channel) |
210 | } | 210 | } |
211 | } | 211 | } |
@@ -217,7 +217,7 @@ const abuseKeysToKeep = [ | |||
217 | 'createdAt' | 217 | 'createdAt' |
218 | ] | 218 | ] |
219 | class AbuseAuditView extends EntityAuditView { | 219 | class AbuseAuditView extends EntityAuditView { |
220 | constructor (private readonly abuse: AdminAbuse) { | 220 | constructor (abuse: AdminAbuse) { |
221 | super(abuseKeysToKeep, 'abuse', abuse) | 221 | super(abuseKeysToKeep, 'abuse', abuse) |
222 | } | 222 | } |
223 | } | 223 | } |
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts index c94c7ab82..3cb17edd0 100644 --- a/server/helpers/upload.ts +++ b/server/helpers/upload.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { JobQueue } from '@server/lib/job-queue' | ||
3 | import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' | 2 | import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' |
4 | 3 | ||
5 | function getResumableUploadPath (filename?: string) { | 4 | function getResumableUploadPath (filename?: string) { |
@@ -8,14 +7,8 @@ function getResumableUploadPath (filename?: string) { | |||
8 | return RESUMABLE_UPLOAD_DIRECTORY | 7 | return RESUMABLE_UPLOAD_DIRECTORY |
9 | } | 8 | } |
10 | 9 | ||
11 | function scheduleDeleteResumableUploadMetaFile (filepath: string) { | ||
12 | const payload = { filepath } | ||
13 | JobQueue.Instance.createJob({ type: 'delete-resumable-upload-meta-file', payload }, { delay: 900 * 1000 }) // executed in 15 min | ||
14 | } | ||
15 | |||
16 | // --------------------------------------------------------------------------- | 10 | // --------------------------------------------------------------------------- |
17 | 11 | ||
18 | export { | 12 | export { |
19 | getResumableUploadPath, | 13 | getResumableUploadPath |
20 | scheduleDeleteResumableUploadMetaFile | ||
21 | } | 14 | } |
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 794303743..359f0c31d 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -31,8 +31,8 @@ function checkMissedConfig () { | |||
31 | 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', | 31 | 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', |
32 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', | 32 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', |
33 | 'transcoding.resolutions.2160p', 'video_studio.enabled', | 33 | 'transcoding.resolutions.2160p', 'video_studio.enabled', |
34 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'auto_blacklist.videos.of_users.enabled', | 34 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', |
35 | 'trending.videos.interval_days', | 35 | 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', |
36 | 'client.videos.miniature.display_author_avatar', | 36 | 'client.videos.miniature.display_author_avatar', |
37 | 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', | 37 | 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', |
38 | 'defaults.publish.download_enabled', 'defaults.publish.comments_enabled', 'defaults.publish.privacy', 'defaults.publish.licence', | 38 | 'defaults.publish.download_enabled', 'defaults.publish.comments_enabled', 'defaults.publish.privacy', 'defaults.publish.licence', |
@@ -44,6 +44,7 @@ function checkMissedConfig () { | |||
44 | 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', | 44 | 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', |
45 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', | 45 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', |
46 | 'theme.default', | 46 | 'theme.default', |
47 | 'feeds.videos.count', 'feeds.comments.count', | ||
47 | 'geo_ip.enabled', 'geo_ip.country.database_url', | 48 | 'geo_ip.enabled', 'geo_ip.country.database_url', |
48 | 'remote_redundancy.videos.accept_from', | 49 | 'remote_redundancy.videos.accept_from', |
49 | 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', | 50 | 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 59a65d6a5..c76a839bc 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -247,6 +247,14 @@ const CONFIG = { | |||
247 | } | 247 | } |
248 | } | 248 | } |
249 | }, | 249 | }, |
250 | FEEDS: { | ||
251 | VIDEOS: { | ||
252 | COUNT: config.get<number>('feeds.videos.count') | ||
253 | }, | ||
254 | COMMENTS: { | ||
255 | COUNT: config.get<number>('feeds.comments.count') | ||
256 | } | ||
257 | }, | ||
250 | ADMIN: { | 258 | ADMIN: { |
251 | get EMAIL () { return config.get<string>('admin.email') } | 259 | get EMAIL () { return config.get<string>('admin.email') } |
252 | }, | 260 | }, |
@@ -349,6 +357,7 @@ const CONFIG = { | |||
349 | IMPORT: { | 357 | IMPORT: { |
350 | VIDEOS: { | 358 | VIDEOS: { |
351 | get CONCURRENCY () { return config.get<number>('import.videos.concurrency') }, | 359 | get CONCURRENCY () { return config.get<number>('import.videos.concurrency') }, |
360 | get TIMEOUT () { return parseDurationToMs(config.get<string>('import.videos.timeout')) }, | ||
352 | 361 | ||
353 | HTTP: { | 362 | HTTP: { |
354 | get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }, | 363 | get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }, |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 909fffdb6..824a30bd2 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -58,7 +58,7 @@ const WEBSERVER = { | |||
58 | 58 | ||
59 | // Sortable columns per schema | 59 | // Sortable columns per schema |
60 | const SORTABLE_COLUMNS = { | 60 | const SORTABLE_COLUMNS = { |
61 | USERS: [ 'id', 'username', 'videoQuotaUsed', 'createdAt', 'lastLoginDate', 'role' ], | 61 | ADMIN_USERS: [ 'id', 'username', 'videoQuotaUsed', 'createdAt', 'lastLoginDate', 'role' ], |
62 | USER_SUBSCRIPTIONS: [ 'id', 'createdAt' ], | 62 | USER_SUBSCRIPTIONS: [ 'id', 'createdAt' ], |
63 | ACCOUNTS: [ 'createdAt' ], | 63 | ACCOUNTS: [ 'createdAt' ], |
64 | JOBS: [ 'createdAt' ], | 64 | JOBS: [ 'createdAt' ], |
@@ -183,7 +183,7 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
183 | 'video-file-import': 1000 * 3600, // 1 hour | 183 | 'video-file-import': 1000 * 3600, // 1 hour |
184 | 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long | 184 | 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long |
185 | 'video-studio-edition': 1000 * 3600 * 10, // 10 hours | 185 | 'video-studio-edition': 1000 * 3600 * 10, // 10 hours |
186 | 'video-import': 1000 * 3600 * 2, // 2 hours | 186 | 'video-import': CONFIG.IMPORT.VIDEOS.TIMEOUT, |
187 | 'email': 60000 * 10, // 10 minutes | 187 | 'email': 60000 * 10, // 10 minutes |
188 | 'actor-keys': 60000 * 20, // 20 minutes | 188 | 'actor-keys': 60000 * 20, // 20 minutes |
189 | 'videos-views-stats': undefined, // Unlimited | 189 | 'videos-views-stats': undefined, // Unlimited |
@@ -766,12 +766,6 @@ const CUSTOM_HTML_TAG_COMMENTS = { | |||
766 | SERVER_CONFIG: '<!-- server config -->' | 766 | SERVER_CONFIG: '<!-- server config -->' |
767 | } | 767 | } |
768 | 768 | ||
769 | // --------------------------------------------------------------------------- | ||
770 | |||
771 | const FEEDS = { | ||
772 | COUNT: 20 | ||
773 | } | ||
774 | |||
775 | const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 | 769 | const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 |
776 | const LOG_FILENAME = 'peertube.log' | 770 | const LOG_FILENAME = 'peertube.log' |
777 | const AUDIT_LOG_FILENAME = 'peertube-audit.log' | 771 | const AUDIT_LOG_FILENAME = 'peertube-audit.log' |
@@ -939,7 +933,6 @@ export { | |||
939 | ROUTE_CACHE_LIFETIME, | 933 | ROUTE_CACHE_LIFETIME, |
940 | SORTABLE_COLUMNS, | 934 | SORTABLE_COLUMNS, |
941 | HLS_STREAMING_PLAYLIST_DIRECTORY, | 935 | HLS_STREAMING_PLAYLIST_DIRECTORY, |
942 | FEEDS, | ||
943 | JOB_TTL, | 936 | JOB_TTL, |
944 | DEFAULT_THEME_NAME, | 937 | DEFAULT_THEME_NAME, |
945 | NSFW_POLICY_TYPES, | 938 | NSFW_POLICY_TYPES, |
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 337364ac9..1e8d03023 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -30,6 +30,7 @@ import { MAccountActor, MChannelActor } from '../types/models' | |||
30 | import { getActivityStreamDuration } from './activitypub/activity' | 30 | import { getActivityStreamDuration } from './activitypub/activity' |
31 | import { getBiggestActorImage } from './actor-image' | 31 | import { getBiggestActorImage } from './actor-image' |
32 | import { ServerConfigManager } from './server-config-manager' | 32 | import { ServerConfigManager } from './server-config-manager' |
33 | import { isTestInstance } from '@server/helpers/core-utils' | ||
33 | 34 | ||
34 | type Tags = { | 35 | type Tags = { |
35 | ogType: string | 36 | ogType: string |
@@ -232,7 +233,10 @@ class ClientHtml { | |||
232 | static async getEmbedHTML () { | 233 | static async getEmbedHTML () { |
233 | const path = ClientHtml.getEmbedPath() | 234 | const path = ClientHtml.getEmbedPath() |
234 | 235 | ||
235 | if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | 236 | // Disable HTML cache in dev mode because webpack can regenerate JS files |
237 | if (!isTestInstance() && ClientHtml.htmlCache[path]) { | ||
238 | return ClientHtml.htmlCache[path] | ||
239 | } | ||
236 | 240 | ||
237 | const buffer = await readFile(path) | 241 | const buffer = await readFile(path) |
238 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() | 242 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() |
diff --git a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts index daefa25bd..a7292de69 100644 --- a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts +++ b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts | |||
@@ -5,7 +5,7 @@ import { MAbuseFull, MAbuseMessage, MAccountDefault, MUserWithNotificationSettin | |||
5 | import { UserNotificationType } from '@shared/models' | 5 | import { UserNotificationType } from '@shared/models' |
6 | import { AbstractNotification } from '../common/abstract-notification' | 6 | import { AbstractNotification } from '../common/abstract-notification' |
7 | 7 | ||
8 | export type NewAbuseMessagePayload = { | 8 | type NewAbuseMessagePayload = { |
9 | abuse: MAbuseFull | 9 | abuse: MAbuseFull |
10 | message: MAbuseMessage | 10 | message: MAbuseMessage |
11 | } | 11 | } |
diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts index 9dda6d76c..b06f5a9b5 100644 --- a/server/lib/schedulers/geo-ip-update-scheduler.ts +++ b/server/lib/schedulers/geo-ip-update-scheduler.ts | |||
@@ -6,7 +6,7 @@ export class GeoIPUpdateScheduler extends AbstractScheduler { | |||
6 | 6 | ||
7 | private static instance: AbstractScheduler | 7 | private static instance: AbstractScheduler |
8 | 8 | ||
9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE | 9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE |
10 | 10 | ||
11 | private constructor () { | 11 | private constructor () { |
12 | super() | 12 | super() |
diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index d2ed079b6..b40f864ce 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts | |||
@@ -4,6 +4,7 @@ export * from './activitypub' | |||
4 | export * from './async' | 4 | export * from './async' |
5 | export * from './auth' | 5 | export * from './auth' |
6 | export * from './pagination' | 6 | export * from './pagination' |
7 | export * from './rate-limiter' | ||
7 | export * from './robots' | 8 | export * from './robots' |
8 | export * from './servers' | 9 | export * from './servers' |
9 | export * from './sort' | 10 | export * from './sort' |
diff --git a/server/middlewares/rate-limiter.ts b/server/middlewares/rate-limiter.ts new file mode 100644 index 000000000..bc9513969 --- /dev/null +++ b/server/middlewares/rate-limiter.ts | |||
@@ -0,0 +1,31 @@ | |||
1 | import { UserRole } from '@shared/models' | ||
2 | import RateLimit from 'express-rate-limit' | ||
3 | import { optionalAuthenticate } from './auth' | ||
4 | |||
5 | const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ]) | ||
6 | |||
7 | function buildRateLimiter (options: { | ||
8 | windowMs: number | ||
9 | max: number | ||
10 | skipFailedRequests?: boolean | ||
11 | }) { | ||
12 | return RateLimit({ | ||
13 | windowMs: options.windowMs, | ||
14 | max: options.max, | ||
15 | skipFailedRequests: options.skipFailedRequests, | ||
16 | |||
17 | handler: (req, res, next, options) => { | ||
18 | return optionalAuthenticate(req, res, () => { | ||
19 | if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) { | ||
20 | return next() | ||
21 | } | ||
22 | |||
23 | return res.status(options.statusCode).send(options.message) | ||
24 | }) | ||
25 | } | ||
26 | }) | ||
27 | } | ||
28 | |||
29 | export { | ||
30 | buildRateLimiter | ||
31 | } | ||
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index 3ba668460..c9978e3b4 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts | |||
@@ -28,7 +28,7 @@ function createSortableColumns (sortableColumns: string[]) { | |||
28 | return sortableColumns.concat(sortableColumnDesc) | 28 | return sortableColumns.concat(sortableColumnDesc) |
29 | } | 29 | } |
30 | 30 | ||
31 | const usersSortValidator = checkSortFactory(SORTABLE_COLUMNS.USERS) | 31 | const adminUsersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ADMIN_USERS) |
32 | const accountsSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNTS) | 32 | const accountsSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNTS) |
33 | const jobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.JOBS, [ 'jobs' ]) | 33 | const jobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.JOBS, [ 'jobs' ]) |
34 | const abusesSortValidator = checkSortFactory(SORTABLE_COLUMNS.ABUSES) | 34 | const abusesSortValidator = checkSortFactory(SORTABLE_COLUMNS.ABUSES) |
@@ -59,7 +59,7 @@ const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CH | |||
59 | // --------------------------------------------------------------------------- | 59 | // --------------------------------------------------------------------------- |
60 | 60 | ||
61 | export { | 61 | export { |
62 | usersSortValidator, | 62 | adminUsersSortValidator, |
63 | abusesSortValidator, | 63 | abusesSortValidator, |
64 | videoChannelsSortValidator, | 64 | videoChannelsSortValidator, |
65 | videoImportsSortValidator, | 65 | videoImportsSortValidator, |
diff --git a/server/models/user/user.ts b/server/models/user/user.ts index 326b2e789..20c2222a7 100644 --- a/server/models/user/user.ts +++ b/server/models/user/user.ts | |||
@@ -66,7 +66,7 @@ import { ActorModel } from '../actor/actor' | |||
66 | import { ActorFollowModel } from '../actor/actor-follow' | 66 | import { ActorFollowModel } from '../actor/actor-follow' |
67 | import { ActorImageModel } from '../actor/actor-image' | 67 | import { ActorImageModel } from '../actor/actor-image' |
68 | import { OAuthTokenModel } from '../oauth/oauth-token' | 68 | import { OAuthTokenModel } from '../oauth/oauth-token' |
69 | import { getSort, throwIfNotValid } from '../utils' | 69 | import { getAdminUsersSort, throwIfNotValid } from '../utils' |
70 | import { VideoModel } from '../video/video' | 70 | import { VideoModel } from '../video/video' |
71 | import { VideoChannelModel } from '../video/video-channel' | 71 | import { VideoChannelModel } from '../video/video-channel' |
72 | import { VideoImportModel } from '../video/video-import' | 72 | import { VideoImportModel } from '../video/video-import' |
@@ -461,7 +461,7 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> { | |||
461 | return this.count() | 461 | return this.count() |
462 | } | 462 | } |
463 | 463 | ||
464 | static listForApi (parameters: { | 464 | static listForAdminApi (parameters: { |
465 | start: number | 465 | start: number |
466 | count: number | 466 | count: number |
467 | sort: string | 467 | sort: string |
@@ -497,7 +497,7 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> { | |||
497 | const query: FindOptions = { | 497 | const query: FindOptions = { |
498 | offset: start, | 498 | offset: start, |
499 | limit: count, | 499 | limit: count, |
500 | order: getSort(sort), | 500 | order: getAdminUsersSort(sort), |
501 | where | 501 | where |
502 | } | 502 | } |
503 | 503 | ||
diff --git a/server/models/utils.ts b/server/models/utils.ts index b57290aff..88e31f22e 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts | |||
@@ -11,8 +11,6 @@ function getSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderIt | |||
11 | 11 | ||
12 | if (field.toLowerCase() === 'match') { // Search | 12 | if (field.toLowerCase() === 'match') { // Search |
13 | finalField = Sequelize.col('similarity') | 13 | finalField = Sequelize.col('similarity') |
14 | } else if (field === 'videoQuotaUsed') { // Users list | ||
15 | finalField = Sequelize.col('videoQuotaUsed') | ||
16 | } else { | 14 | } else { |
17 | finalField = field | 15 | finalField = field |
18 | } | 16 | } |
@@ -20,6 +18,25 @@ function getSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderIt | |||
20 | return [ [ finalField, direction ], lastSort ] | 18 | return [ [ finalField, direction ], lastSort ] |
21 | } | 19 | } |
22 | 20 | ||
21 | function getAdminUsersSort (value: string): OrderItem[] { | ||
22 | const { direction, field } = buildDirectionAndField(value) | ||
23 | |||
24 | let finalField: string | ReturnType<typeof Sequelize.col> | ||
25 | |||
26 | if (field === 'videoQuotaUsed') { // Users list | ||
27 | finalField = Sequelize.col('videoQuotaUsed') | ||
28 | } else { | ||
29 | finalField = field | ||
30 | } | ||
31 | |||
32 | const nullPolicy = direction === 'ASC' | ||
33 | ? 'NULLS FIRST' | ||
34 | : 'NULLS LAST' | ||
35 | |||
36 | // FIXME: typings | ||
37 | return [ [ finalField as any, direction, nullPolicy ], [ 'id', 'ASC' ] ] | ||
38 | } | ||
39 | |||
23 | function getPlaylistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { | 40 | function getPlaylistSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]): OrderItem[] { |
24 | const { direction, field } = buildDirectionAndField(value) | 41 | const { direction, field } = buildDirectionAndField(value) |
25 | 42 | ||
@@ -260,6 +277,7 @@ export { | |||
260 | buildLocalAccountIdsIn, | 277 | buildLocalAccountIdsIn, |
261 | getSort, | 278 | getSort, |
262 | getCommentSort, | 279 | getCommentSort, |
280 | getAdminUsersSort, | ||
263 | getVideoSort, | 281 | getVideoSort, |
264 | getBlacklistSort, | 282 | getBlacklistSort, |
265 | createSimilarityAttribute, | 283 | createSimilarityAttribute, |
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index d6dd1b8bb..91dafbcf1 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts | |||
@@ -311,6 +311,16 @@ export type SummaryOptions = { | |||
311 | ')' | 311 | ')' |
312 | ), | 312 | ), |
313 | 'viewsPerDay' | 313 | 'viewsPerDay' |
314 | ], | ||
315 | [ | ||
316 | literal( | ||
317 | '(' + | ||
318 | 'SELECT COALESCE(SUM("video".views), 0) AS totalViews ' + | ||
319 | 'FROM "video" ' + | ||
320 | 'WHERE "video"."channelId" = "VideoChannelModel"."id"' + | ||
321 | ')' | ||
322 | ), | ||
323 | 'totalViews' | ||
314 | ] | 324 | ] |
315 | ] | 325 | ] |
316 | } | 326 | } |
@@ -766,6 +776,8 @@ ON "Account->Actor"."serverId" = "Account->Actor->Server"."id"` | |||
766 | }) | 776 | }) |
767 | } | 777 | } |
768 | 778 | ||
779 | const totalViews = this.get('totalViews') as number | ||
780 | |||
769 | const actor = this.Actor.toFormattedJSON() | 781 | const actor = this.Actor.toFormattedJSON() |
770 | const videoChannel = { | 782 | const videoChannel = { |
771 | id: this.id, | 783 | id: this.id, |
@@ -779,6 +791,7 @@ ON "Account->Actor"."serverId" = "Account->Actor->Server"."id"` | |||
779 | 791 | ||
780 | videosCount, | 792 | videosCount, |
781 | viewsPerDay, | 793 | viewsPerDay, |
794 | totalViews, | ||
782 | 795 | ||
783 | avatars: actor.avatars, | 796 | avatars: actor.avatars, |
784 | 797 | ||
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index 9b8fbe3e2..c497f7840 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts | |||
@@ -654,7 +654,7 @@ describe('Test live', function () { | |||
654 | }) | 654 | }) |
655 | 655 | ||
656 | it('Should save a non permanent live replay', async function () { | 656 | it('Should save a non permanent live replay', async function () { |
657 | this.timeout(120000) | 657 | this.timeout(240000) |
658 | 658 | ||
659 | await commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) | 659 | await commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) |
660 | 660 | ||
diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts index fa2063536..0a1565faf 100644 --- a/server/tests/api/server/reverse-proxy.ts +++ b/server/tests/api/server/reverse-proxy.ts | |||
@@ -7,6 +7,7 @@ import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServ | |||
7 | 7 | ||
8 | describe('Test application behind a reverse proxy', function () { | 8 | describe('Test application behind a reverse proxy', function () { |
9 | let server: PeerTubeServer | 9 | let server: PeerTubeServer |
10 | let userAccessToken: string | ||
10 | let videoId: string | 11 | let videoId: string |
11 | 12 | ||
12 | before(async function () { | 13 | before(async function () { |
@@ -34,6 +35,8 @@ describe('Test application behind a reverse proxy', function () { | |||
34 | server = await createSingleServer(1, config) | 35 | server = await createSingleServer(1, config) |
35 | await setAccessTokensToServers([ server ]) | 36 | await setAccessTokensToServers([ server ]) |
36 | 37 | ||
38 | userAccessToken = await server.users.generateUserAndToken('user') | ||
39 | |||
37 | const { uuid } = await server.videos.upload() | 40 | const { uuid } = await server.videos.upload() |
38 | videoId = uuid | 41 | videoId = uuid |
39 | }) | 42 | }) |
@@ -93,7 +96,7 @@ describe('Test application behind a reverse proxy', function () { | |||
93 | it('Should rate limit logins', async function () { | 96 | it('Should rate limit logins', async function () { |
94 | const user = { username: 'root', password: 'fail' } | 97 | const user = { username: 'root', password: 'fail' } |
95 | 98 | ||
96 | for (let i = 0; i < 19; i++) { | 99 | for (let i = 0; i < 18; i++) { |
97 | await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 100 | await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) |
98 | } | 101 | } |
99 | 102 | ||
@@ -141,6 +144,12 @@ describe('Test application behind a reverse proxy', function () { | |||
141 | await server.videos.get({ id: videoId, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) | 144 | await server.videos.get({ id: videoId, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) |
142 | }) | 145 | }) |
143 | 146 | ||
147 | it('Should rate limit API calls with a user but not with an admin', async function () { | ||
148 | await server.videos.get({ id: videoId, token: userAccessToken, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) | ||
149 | |||
150 | await server.videos.get({ id: videoId, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
151 | }) | ||
152 | |||
144 | after(async function () { | 153 | after(async function () { |
145 | await cleanupTests([ server ]) | 154 | await cleanupTests([ server ]) |
146 | }) | 155 | }) |
diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts index 6f495c42d..42e0cf431 100644 --- a/server/tests/api/videos/video-channels.ts +++ b/server/tests/api/videos/video-channels.ts | |||
@@ -478,6 +478,25 @@ describe('Test video channels', function () { | |||
478 | } | 478 | } |
479 | }) | 479 | }) |
480 | 480 | ||
481 | it('Should report correct total views count', async function () { | ||
482 | // check if there's the property | ||
483 | { | ||
484 | const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) | ||
485 | |||
486 | for (const channel of data) { | ||
487 | expect(channel).to.haveOwnProperty('totalViews') | ||
488 | expect(channel.totalViews).to.be.a('number') | ||
489 | } | ||
490 | } | ||
491 | |||
492 | // Check if the totalViews count can be updated | ||
493 | { | ||
494 | const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) | ||
495 | const channelWithView = data.find(channel => channel.id === servers[0].store.channel.id) | ||
496 | expect(channelWithView.totalViews).to.equal(2) | ||
497 | } | ||
498 | }) | ||
499 | |||
481 | it('Should report correct videos count', async function () { | 500 | it('Should report correct videos count', async function () { |
482 | const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) | 501 | const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) |
483 | 502 | ||