aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/server/debug.ts2
-rw-r--r--server/controllers/api/users/index.ts4
-rw-r--r--server/controllers/api/users/my-history.ts3
-rw-r--r--server/controllers/api/users/my-video-playlists.ts8
-rw-r--r--server/controllers/api/users/token.ts7
-rw-r--r--server/controllers/api/users/two-factor.ts95
-rw-r--r--server/controllers/api/video-playlist.ts11
-rw-r--r--server/controllers/api/videos/import.ts2
-rw-r--r--server/controllers/api/videos/index.ts2
-rw-r--r--server/controllers/api/videos/token.ts33
-rw-r--r--server/controllers/api/videos/transcoding.ts8
-rw-r--r--server/controllers/api/videos/update.ts79
-rw-r--r--server/controllers/download.ts28
-rw-r--r--server/controllers/feeds.ts78
-rw-r--r--server/controllers/index.ts12
-rw-r--r--server/controllers/live.ts32
-rw-r--r--server/controllers/object-storage-proxy.ts86
-rw-r--r--server/controllers/services.ts5
-rw-r--r--server/controllers/static.ts43
-rw-r--r--server/controllers/well-known.ts7
-rw-r--r--server/helpers/core-utils.ts14
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts10
-rw-r--r--server/helpers/custom-validators/misc.ts2
-rw-r--r--server/helpers/custom-validators/plugins.ts22
-rw-r--r--server/helpers/custom-validators/servers.ts4
-rw-r--r--server/helpers/custom-validators/video-studio.ts3
-rw-r--r--server/helpers/custom-validators/videos.ts5
-rw-r--r--server/helpers/ffmpeg/ffmpeg-commons.ts2
-rw-r--r--server/helpers/ffmpeg/ffmpeg-vod.ts13
-rw-r--r--server/helpers/ffmpeg/ffprobe-utils.ts9
-rw-r--r--server/helpers/otp.ts58
-rw-r--r--server/helpers/peertube-crypto.ts49
-rw-r--r--server/helpers/upload.ts6
-rw-r--r--server/helpers/video.ts5
-rw-r--r--server/helpers/webtorrent.ts16
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts15
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts2
-rw-r--r--server/initializers/checker-after-init.ts15
-rw-r--r--server/initializers/checker-before-init.ts15
-rw-r--r--server/initializers/config.ts21
-rw-r--r--server/initializers/constants.ts70
-rw-r--r--server/initializers/installer.ts10
-rw-r--r--server/initializers/migrations/0745-user-otp.ts29
-rw-r--r--server/lib/activitypub/collection.ts3
-rw-r--r--server/lib/activitypub/process/process-create.ts2
-rw-r--r--server/lib/activitypub/share.ts2
-rw-r--r--server/lib/activitypub/video-comments.ts35
-rw-r--r--server/lib/auth/oauth.ts36
-rw-r--r--server/lib/hls.ts6
-rw-r--r--server/lib/job-queue/handlers/activitypub-cleaner.ts2
-rw-r--r--server/lib/job-queue/handlers/manage-video-torrent.ts37
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts22
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts23
-rw-r--r--server/lib/job-queue/handlers/video-import.ts228
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts49
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts100
-rw-r--r--server/lib/job-queue/job-queue.ts21
-rw-r--r--server/lib/live/live-manager.ts18
-rw-r--r--server/lib/live/live-segment-sha-store.ts82
-rw-r--r--server/lib/live/live-utils.ts69
-rw-r--r--server/lib/live/shared/muxing-session.ts115
-rw-r--r--server/lib/moderation.ts44
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts223
-rw-r--r--server/lib/object-storage/urls.ts29
-rw-r--r--server/lib/object-storage/videos.ts138
-rw-r--r--server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts2
-rw-r--r--server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts2
-rw-r--r--server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts2
-rw-r--r--server/lib/opentelemetry/metric-helpers/playback-metrics.ts2
-rw-r--r--server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts2
-rw-r--r--server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts2
-rw-r--r--server/lib/opentelemetry/metrics.ts7
-rw-r--r--server/lib/opentelemetry/tracing.ts4
-rw-r--r--server/lib/paths.ts17
-rw-r--r--server/lib/plugins/plugin-helpers-builder.ts28
-rw-r--r--server/lib/plugins/plugin-manager.ts31
-rw-r--r--server/lib/plugins/register-helpers.ts21
-rw-r--r--server/lib/plugins/yarn.ts4
-rw-r--r--server/lib/redis.ts105
-rw-r--r--server/lib/schedulers/plugins-check-scheduler.ts2
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts68
-rw-r--r--server/lib/schedulers/video-channel-sync-latest-scheduler.ts35
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts25
-rw-r--r--server/lib/sync-channel.ts88
-rw-r--r--server/lib/transcoding/transcoding.ts386
-rw-r--r--server/lib/uploadx.ts14
-rw-r--r--server/lib/video-path-manager.ts51
-rw-r--r--server/lib/video-pre-import.ts (renamed from server/lib/video-import.ts)0
-rw-r--r--server/lib/video-privacy.ts127
-rw-r--r--server/lib/video-tokens-manager.ts49
-rw-r--r--server/lib/video.ts61
-rw-r--r--server/middlewares/activitypub.ts2
-rw-r--r--server/middlewares/auth.ts8
-rw-r--r--server/middlewares/cache/shared/api-cache.ts6
-rw-r--r--server/middlewares/pagination.ts5
-rw-r--r--server/middlewares/validators/abuse.ts3
-rw-r--r--server/middlewares/validators/index.ts8
-rw-r--r--server/middlewares/validators/object-storage-proxy.ts20
-rw-r--r--server/middlewares/validators/plugins.ts13
-rw-r--r--server/middlewares/validators/redundancy.ts3
-rw-r--r--server/middlewares/validators/shared/abuses.ts3
-rw-r--r--server/middlewares/validators/shared/accounts.ts5
-rw-r--r--server/middlewares/validators/shared/index.ts1
-rw-r--r--server/middlewares/validators/shared/users.ts63
-rw-r--r--server/middlewares/validators/shared/video-comments.ts7
-rw-r--r--server/middlewares/validators/shared/video-ownerships.ts3
-rw-r--r--server/middlewares/validators/shared/videos.ts54
-rw-r--r--server/middlewares/validators/static.ts169
-rw-r--r--server/middlewares/validators/themes.ts4
-rw-r--r--server/middlewares/validators/two-factor.ts81
-rw-r--r--server/middlewares/validators/user-subscriptions.ts4
-rw-r--r--server/middlewares/validators/users.ts119
-rw-r--r--server/middlewares/validators/videos/video-comments.ts5
-rw-r--r--server/middlewares/validators/videos/video-imports.ts3
-rw-r--r--server/middlewares/validators/videos/video-playlists.ts3
-rw-r--r--server/middlewares/validators/videos/videos.ts33
-rw-r--r--server/models/abuse/abuse-query-builder.ts5
-rw-r--r--server/models/abuse/abuse.ts2
-rw-r--r--server/models/actor/actor.ts4
-rw-r--r--server/models/server/plugin.ts13
-rw-r--r--server/models/user/user-notification.ts3
-rw-r--r--server/models/user/user.ts30
-rw-r--r--server/models/utils.ts5
-rw-r--r--server/models/video/formatter/video-format-utils.ts26
-rw-r--r--server/models/video/sql/video/shared/abstract-video-query-builder.ts2
-rw-r--r--server/models/video/sql/video/videos-id-list-query-builder.ts5
-rw-r--r--server/models/video/video-channel.ts4
-rw-r--r--server/models/video/video-file.ts80
-rw-r--r--server/models/video/video-job-info.ts6
-rw-r--r--server/models/video/video-playlist-element.ts21
-rw-r--r--server/models/video/video-playlist.ts6
-rw-r--r--server/models/video/video-share.ts5
-rw-r--r--server/models/video/video-streaming-playlist.ts62
-rw-r--r--server/models/video/video.ts71
-rw-r--r--server/tests/api/check-params/index.ts6
-rw-r--r--server/tests/api/check-params/live.ts17
-rw-r--r--server/tests/api/check-params/two-factor.ts288
-rw-r--r--server/tests/api/check-params/video-files.ts217
-rw-r--r--server/tests/api/check-params/video-token.ts44
-rw-r--r--server/tests/api/live/live-constraints.ts2
-rw-r--r--server/tests/api/live/live-fast-restream.ts29
-rw-r--r--server/tests/api/live/live-permanent.ts12
-rw-r--r--server/tests/api/live/live-save-replay.ts18
-rw-r--r--server/tests/api/live/live.ts116
-rw-r--r--server/tests/api/notifications/admin-notifications.ts6
-rw-r--r--server/tests/api/notifications/moderation-notifications.ts4
-rw-r--r--server/tests/api/object-storage/index.ts1
-rw-r--r--server/tests/api/object-storage/live.ts191
-rw-r--r--server/tests/api/object-storage/video-imports.ts20
-rw-r--r--server/tests/api/object-storage/video-static-file-privacy.ts402
-rw-r--r--server/tests/api/object-storage/videos.ts60
-rw-r--r--server/tests/api/redundancy/redundancy.ts22
-rw-r--r--server/tests/api/server/follow-constraints.ts88
-rw-r--r--server/tests/api/server/open-telemetry.ts6
-rw-r--r--server/tests/api/server/proxy.ts16
-rw-r--r--server/tests/api/transcoding/audio-only.ts7
-rw-r--r--server/tests/api/transcoding/create-transcoding.ts26
-rw-r--r--server/tests/api/transcoding/hls.ts171
-rw-r--r--server/tests/api/transcoding/index.ts1
-rw-r--r--server/tests/api/transcoding/update-while-transcoding.ts151
-rw-r--r--server/tests/api/transcoding/video-studio.ts12
-rw-r--r--server/tests/api/users/index.ts1
-rw-r--r--server/tests/api/users/two-factor.ts200
-rw-r--r--server/tests/api/users/users-multiple-servers.ts2
-rw-r--r--server/tests/api/users/users.ts8
-rw-r--r--server/tests/api/videos/channel-import-videos.ts43
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/multiple-servers.ts2
-rw-r--r--server/tests/api/videos/video-channel-syncs.ts2
-rw-r--r--server/tests/api/videos/video-description.ts23
-rw-r--r--server/tests/api/videos/video-files.ts4
-rw-r--r--server/tests/api/videos/video-playlists.ts16
-rw-r--r--server/tests/api/videos/video-privacy.ts4
-rw-r--r--server/tests/api/videos/video-static-file-privacy.ts422
-rw-r--r--server/tests/api/videos/videos-common-filters.ts2
-rw-r--r--server/tests/api/views/video-views-counter.ts2
-rw-r--r--server/tests/api/views/video-views-overall-stats.ts4
-rw-r--r--server/tests/api/views/video-views-retention-stats.ts2
-rw-r--r--server/tests/api/views/video-views-timeserie-stats.ts4
-rw-r--r--server/tests/cli/create-import-video-file-job.ts12
-rw-r--r--server/tests/cli/create-move-video-storage-job.ts20
-rw-r--r--server/tests/cli/create-transcoding-job.ts14
-rw-r--r--server/tests/cli/peertube.ts21
-rw-r--r--server/tests/cli/prune-storage.ts41
-rw-r--r--server/tests/cli/regenerate-thumbnails.ts20
-rw-r--r--server/tests/external-plugins/akismet.ts160
-rw-r--r--server/tests/external-plugins/auth-ldap.ts8
-rw-r--r--server/tests/external-plugins/index.ts1
-rw-r--r--server/tests/feeds/feeds.ts10
-rw-r--r--server/tests/fixtures/peertube-plugin-test-four/main.js16
-rw-r--r--server/tests/fixtures/peertube-plugin-test-websocket/main.js36
-rw-r--r--server/tests/fixtures/peertube-plugin-test-websocket/package.json20
-rw-r--r--server/tests/fixtures/peertube-plugin-test/main.js9
-rw-r--r--server/tests/helpers/crypto.ts33
-rw-r--r--server/tests/helpers/index.ts6
-rw-r--r--server/tests/helpers/validator.ts32
-rw-r--r--server/tests/misc-endpoints.ts30
-rw-r--r--server/tests/plugins/action-hooks.ts2
-rw-r--r--server/tests/plugins/external-auth.ts6
-rw-r--r--server/tests/plugins/filter-hooks.ts419
-rw-r--r--server/tests/plugins/id-and-pass-auth.ts8
-rw-r--r--server/tests/plugins/index.ts1
-rw-r--r--server/tests/plugins/plugin-helpers.ts33
-rw-r--r--server/tests/plugins/plugin-websocket.ts70
-rw-r--r--server/tests/shared/actors.ts8
-rw-r--r--server/tests/shared/directories.ts8
-rw-r--r--server/tests/shared/live.ts146
-rw-r--r--server/tests/shared/mock-servers/mock-object-storage.ts2
-rw-r--r--server/tests/shared/playlists.ts9
-rw-r--r--server/tests/shared/streaming-playlists.ts138
-rw-r--r--server/tests/shared/videos.ts6
-rw-r--r--server/tools/cli.ts2
-rw-r--r--server/tools/peertube-plugins.ts2
-rw-r--r--server/tools/peertube-redundancy.ts4
-rw-r--r--server/types/express.d.ts2
-rw-r--r--server/types/models/video/video-playlist.ts12
-rw-r--r--server/types/plugins/index.ts1
-rw-r--r--server/types/plugins/register-server-option.model.ts21
-rw-r--r--server/types/plugins/register-server-websocket-route.model.ts8
219 files changed, 6484 insertions, 1922 deletions
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index 4e5333782..f3792bfc8 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -8,6 +8,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
8import { UserRight } from '../../../../shared/models/users' 8import { UserRight } from '../../../../shared/models/users'
9import { authenticate, ensureUserHasRight } from '../../../middlewares' 9import { authenticate, ensureUserHasRight } from '../../../middlewares'
10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' 10import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
11import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler'
11 12
12const debugRouter = express.Router() 13const debugRouter = express.Router()
13 14
@@ -45,6 +46,7 @@ async function runCommand (req: express.Request, res: express.Response) {
45 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), 46 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
46 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), 47 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
47 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), 48 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
49 'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),
48 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() 50 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
49 } 51 }
50 52
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 0b27d5277..a8677a1d3 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -51,6 +51,7 @@ import { myVideosHistoryRouter } from './my-history'
51import { myNotificationsRouter } from './my-notifications' 51import { myNotificationsRouter } from './my-notifications'
52import { mySubscriptionsRouter } from './my-subscriptions' 52import { mySubscriptionsRouter } from './my-subscriptions'
53import { myVideoPlaylistsRouter } from './my-video-playlists' 53import { myVideoPlaylistsRouter } from './my-video-playlists'
54import { twoFactorRouter } from './two-factor'
54 55
55const auditLogger = auditLoggerFactory('users') 56const auditLogger = auditLoggerFactory('users')
56 57
@@ -66,6 +67,7 @@ const askSendEmailLimiter = buildRateLimiter({
66}) 67})
67 68
68const usersRouter = express.Router() 69const usersRouter = express.Router()
70usersRouter.use('/', twoFactorRouter)
69usersRouter.use('/', tokensRouter) 71usersRouter.use('/', tokensRouter)
70usersRouter.use('/', myNotificationsRouter) 72usersRouter.use('/', myNotificationsRouter)
71usersRouter.use('/', mySubscriptionsRouter) 73usersRouter.use('/', mySubscriptionsRouter)
@@ -343,7 +345,7 @@ async function askResetUserPassword (req: express.Request, res: express.Response
343 345
344 const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) 346 const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id)
345 const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString 347 const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString
346 await Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url) 348 Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url)
347 349
348 return res.status(HttpStatusCode.NO_CONTENT_204).end() 350 return res.status(HttpStatusCode.NO_CONTENT_204).end()
349} 351}
diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts
index bc5b40f59..e6d3e86ac 100644
--- a/server/controllers/api/users/my-history.ts
+++ b/server/controllers/api/users/my-history.ts
@@ -1,3 +1,4 @@
1import { forceNumber } from '@shared/core-utils'
1import express from 'express' 2import express from 'express'
2import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' 3import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
3import { getFormattedObjects } from '../../../helpers/utils' 4import { getFormattedObjects } from '../../../helpers/utils'
@@ -55,7 +56,7 @@ async function listMyVideosHistory (req: express.Request, res: express.Response)
55async function removeUserHistoryElement (req: express.Request, res: express.Response) { 56async function removeUserHistoryElement (req: express.Request, res: express.Response) {
56 const user = res.locals.oauth.token.User 57 const user = res.locals.oauth.token.User
57 58
58 await UserVideoHistoryModel.removeUserHistoryElement(user, parseInt(req.params.videoId + '')) 59 await UserVideoHistoryModel.removeUserHistoryElement(user, forceNumber(req.params.videoId))
59 60
60 return res.sendStatus(HttpStatusCode.NO_CONTENT_204) 61 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
61} 62}
diff --git a/server/controllers/api/users/my-video-playlists.ts b/server/controllers/api/users/my-video-playlists.ts
index f55ea2ec4..fbdbb7e50 100644
--- a/server/controllers/api/users/my-video-playlists.ts
+++ b/server/controllers/api/users/my-video-playlists.ts
@@ -1,4 +1,6 @@
1import express from 'express' 1import express from 'express'
2import { forceNumber } from '@shared/core-utils'
3import { uuidToShort } from '@shared/extra-utils'
2import { VideosExistInPlaylists } from '../../../../shared/models/videos/playlist/video-exist-in-playlist.model' 4import { VideosExistInPlaylists } from '../../../../shared/models/videos/playlist/video-exist-in-playlist.model'
3import { asyncMiddleware, authenticate } from '../../../middlewares' 5import { asyncMiddleware, authenticate } from '../../../middlewares'
4import { doVideosInPlaylistExistValidator } from '../../../middlewares/validators/videos/video-playlists' 6import { doVideosInPlaylistExistValidator } from '../../../middlewares/validators/videos/video-playlists'
@@ -21,10 +23,10 @@ export {
21// --------------------------------------------------------------------------- 23// ---------------------------------------------------------------------------
22 24
23async function doVideosInPlaylistExist (req: express.Request, res: express.Response) { 25async function doVideosInPlaylistExist (req: express.Request, res: express.Response) {
24 const videoIds = req.query.videoIds.map(i => parseInt(i + '', 10)) 26 const videoIds = req.query.videoIds.map(i => forceNumber(i))
25 const user = res.locals.oauth.token.User 27 const user = res.locals.oauth.token.User
26 28
27 const results = await VideoPlaylistModel.listPlaylistIdsOf(user.Account.id, videoIds) 29 const results = await VideoPlaylistModel.listPlaylistSummariesOf(user.Account.id, videoIds)
28 30
29 const existObject: VideosExistInPlaylists = {} 31 const existObject: VideosExistInPlaylists = {}
30 32
@@ -37,6 +39,8 @@ async function doVideosInPlaylistExist (req: express.Request, res: express.Respo
37 existObject[element.videoId].push({ 39 existObject[element.videoId].push({
38 playlistElementId: element.id, 40 playlistElementId: element.id,
39 playlistId: result.id, 41 playlistId: result.id,
42 playlistDisplayName: result.name,
43 playlistShortUUID: uuidToShort(result.uuid),
40 startTimestamp: element.startTimestamp, 44 startTimestamp: element.startTimestamp,
41 stopTimestamp: element.stopTimestamp 45 stopTimestamp: element.stopTimestamp
42 }) 46 })
diff --git a/server/controllers/api/users/token.ts b/server/controllers/api/users/token.ts
index 012a49791..c6afea67c 100644
--- a/server/controllers/api/users/token.ts
+++ b/server/controllers/api/users/token.ts
@@ -1,8 +1,9 @@
1import express from 'express' 1import express from 'express'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { OTP } from '@server/initializers/constants'
4import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' 5import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth'
5import { handleOAuthToken } from '@server/lib/auth/oauth' 6import { handleOAuthToken, MissingTwoFactorError } from '@server/lib/auth/oauth'
6import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' 7import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model'
7import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
8import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares' 9import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares'
@@ -79,6 +80,10 @@ async function handleToken (req: express.Request, res: express.Response, next: e
79 } catch (err) { 80 } catch (err) {
80 logger.warn('Login error', { err }) 81 logger.warn('Login error', { err })
81 82
83 if (err instanceof MissingTwoFactorError) {
84 res.set(OTP.HEADER_NAME, OTP.HEADER_REQUIRED_VALUE)
85 }
86
82 return res.fail({ 87 return res.fail({
83 status: err.code, 88 status: err.code,
84 message: err.message, 89 message: err.message,
diff --git a/server/controllers/api/users/two-factor.ts b/server/controllers/api/users/two-factor.ts
new file mode 100644
index 000000000..e6ae9e4dd
--- /dev/null
+++ b/server/controllers/api/users/two-factor.ts
@@ -0,0 +1,95 @@
1import express from 'express'
2import { generateOTPSecret, isOTPValid } from '@server/helpers/otp'
3import { encrypt } from '@server/helpers/peertube-crypto'
4import { CONFIG } from '@server/initializers/config'
5import { Redis } from '@server/lib/redis'
6import { asyncMiddleware, authenticate, usersCheckCurrentPasswordFactory } from '@server/middlewares'
7import {
8 confirmTwoFactorValidator,
9 disableTwoFactorValidator,
10 requestOrConfirmTwoFactorValidator
11} from '@server/middlewares/validators/two-factor'
12import { HttpStatusCode, TwoFactorEnableResult } from '@shared/models'
13
14const twoFactorRouter = express.Router()
15
16twoFactorRouter.post('/:id/two-factor/request',
17 authenticate,
18 asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)),
19 asyncMiddleware(requestOrConfirmTwoFactorValidator),
20 asyncMiddleware(requestTwoFactor)
21)
22
23twoFactorRouter.post('/:id/two-factor/confirm-request',
24 authenticate,
25 asyncMiddleware(requestOrConfirmTwoFactorValidator),
26 confirmTwoFactorValidator,
27 asyncMiddleware(confirmRequestTwoFactor)
28)
29
30twoFactorRouter.post('/:id/two-factor/disable',
31 authenticate,
32 asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)),
33 asyncMiddleware(disableTwoFactorValidator),
34 asyncMiddleware(disableTwoFactor)
35)
36
37// ---------------------------------------------------------------------------
38
39export {
40 twoFactorRouter
41}
42
43// ---------------------------------------------------------------------------
44
45async function requestTwoFactor (req: express.Request, res: express.Response) {
46 const user = res.locals.user
47
48 const { secret, uri } = generateOTPSecret(user.email)
49
50 const encryptedSecret = await encrypt(secret, CONFIG.SECRETS.PEERTUBE)
51 const requestToken = await Redis.Instance.setTwoFactorRequest(user.id, encryptedSecret)
52
53 return res.json({
54 otpRequest: {
55 requestToken,
56 secret,
57 uri
58 }
59 } as TwoFactorEnableResult)
60}
61
62async function confirmRequestTwoFactor (req: express.Request, res: express.Response) {
63 const requestToken = req.body.requestToken
64 const otpToken = req.body.otpToken
65 const user = res.locals.user
66
67 const encryptedSecret = await Redis.Instance.getTwoFactorRequestToken(user.id, requestToken)
68 if (!encryptedSecret) {
69 return res.fail({
70 message: 'Invalid request token',
71 status: HttpStatusCode.FORBIDDEN_403
72 })
73 }
74
75 if (await isOTPValid({ encryptedSecret, token: otpToken }) !== true) {
76 return res.fail({
77 message: 'Invalid OTP token',
78 status: HttpStatusCode.FORBIDDEN_403
79 })
80 }
81
82 user.otpSecret = encryptedSecret
83 await user.save()
84
85 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
86}
87
88async function disableTwoFactor (req: express.Request, res: express.Response) {
89 const user = res.locals.user
90
91 user.otpSecret = null
92 await user.save()
93
94 return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
95}
diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts
index 1255d14c6..f8a607170 100644
--- a/server/controllers/api/video-playlist.ts
+++ b/server/controllers/api/video-playlist.ts
@@ -4,6 +4,7 @@ import { scheduleRefreshIfNeeded } from '@server/lib/activitypub/playlists'
4import { Hooks } from '@server/lib/plugins/hooks' 4import { Hooks } from '@server/lib/plugins/hooks'
5import { getServerActor } from '@server/models/application/application' 5import { getServerActor } from '@server/models/application/application'
6import { MVideoPlaylistFull, MVideoPlaylistThumbnail, MVideoThumbnail } from '@server/types/models' 6import { MVideoPlaylistFull, MVideoPlaylistThumbnail, MVideoThumbnail } from '@server/types/models'
7import { forceNumber } from '@shared/core-utils'
7import { uuidToShort } from '@shared/extra-utils' 8import { uuidToShort } from '@shared/extra-utils'
8import { VideoPlaylistCreateResult, VideoPlaylistElementCreateResult } from '@shared/models' 9import { VideoPlaylistCreateResult, VideoPlaylistElementCreateResult } from '@shared/models'
9import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' 10import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
@@ -245,7 +246,7 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response)
245 if (videoPlaylistInfoToUpdate.description !== undefined) videoPlaylistInstance.description = videoPlaylistInfoToUpdate.description 246 if (videoPlaylistInfoToUpdate.description !== undefined) videoPlaylistInstance.description = videoPlaylistInfoToUpdate.description
246 247
247 if (videoPlaylistInfoToUpdate.privacy !== undefined) { 248 if (videoPlaylistInfoToUpdate.privacy !== undefined) {
248 videoPlaylistInstance.privacy = parseInt(videoPlaylistInfoToUpdate.privacy.toString(), 10) 249 videoPlaylistInstance.privacy = forceNumber(videoPlaylistInfoToUpdate.privacy)
249 250
250 if (wasNotPrivatePlaylist === true && videoPlaylistInstance.privacy === VideoPlaylistPrivacy.PRIVATE) { 251 if (wasNotPrivatePlaylist === true && videoPlaylistInstance.privacy === VideoPlaylistPrivacy.PRIVATE) {
251 await sendDeleteVideoPlaylist(videoPlaylistInstance, t) 252 await sendDeleteVideoPlaylist(videoPlaylistInstance, t)
@@ -424,7 +425,13 @@ async function reorderVideosPlaylist (req: express.Request, res: express.Respons
424 425
425 const endOldPosition = oldPosition + reorderLength - 1 426 const endOldPosition = oldPosition + reorderLength - 1
426 // Insert our reordered elements in their place (update) 427 // Insert our reordered elements in their place (update)
427 await VideoPlaylistElementModel.reassignPositionOf(videoPlaylist.id, oldPosition, endOldPosition, newPosition, t) 428 await VideoPlaylistElementModel.reassignPositionOf({
429 videoPlaylistId: videoPlaylist.id,
430 firstPosition: oldPosition,
431 endPosition: endOldPosition,
432 newPosition,
433 transaction: t
434 })
428 435
429 // Decrease positions of elements after the old position of our ordered elements (decrease) 436 // Decrease positions of elements after the old position of our ordered elements (decrease)
430 await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, oldPosition, -reorderLength, t) 437 await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, oldPosition, -reorderLength, t)
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 9d7b0260b..6a50aaf4e 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -3,7 +3,7 @@ import { move, readFile } from 'fs-extra'
3import { decode } from 'magnet-uri' 3import { decode } from 'magnet-uri'
4import parseTorrent, { Instance } from 'parse-torrent' 4import parseTorrent, { Instance } from 'parse-torrent'
5import { join } from 'path' 5import { join } from 'path'
6import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import' 6import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import'
7import { MThumbnail, MVideoThumbnail } from '@server/types/models' 7import { MThumbnail, MVideoThumbnail } from '@server/types/models'
8import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models' 8import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
9import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' 9import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index b301515df..ea081e5ab 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -41,6 +41,7 @@ import { ownershipVideoRouter } from './ownership'
41import { rateVideoRouter } from './rate' 41import { rateVideoRouter } from './rate'
42import { statsRouter } from './stats' 42import { statsRouter } from './stats'
43import { studioRouter } from './studio' 43import { studioRouter } from './studio'
44import { tokenRouter } from './token'
44import { transcodingRouter } from './transcoding' 45import { transcodingRouter } from './transcoding'
45import { updateRouter } from './update' 46import { updateRouter } from './update'
46import { uploadRouter } from './upload' 47import { uploadRouter } from './upload'
@@ -63,6 +64,7 @@ videosRouter.use('/', uploadRouter)
63videosRouter.use('/', updateRouter) 64videosRouter.use('/', updateRouter)
64videosRouter.use('/', filesRouter) 65videosRouter.use('/', filesRouter)
65videosRouter.use('/', transcodingRouter) 66videosRouter.use('/', transcodingRouter)
67videosRouter.use('/', tokenRouter)
66 68
67videosRouter.get('/categories', 69videosRouter.get('/categories',
68 openapiOperationDoc({ operationId: 'getCategories' }), 70 openapiOperationDoc({ operationId: 'getCategories' }),
diff --git a/server/controllers/api/videos/token.ts b/server/controllers/api/videos/token.ts
new file mode 100644
index 000000000..009b6dfb6
--- /dev/null
+++ b/server/controllers/api/videos/token.ts
@@ -0,0 +1,33 @@
1import express from 'express'
2import { VideoTokensManager } from '@server/lib/video-tokens-manager'
3import { VideoToken } from '@shared/models'
4import { asyncMiddleware, authenticate, videosCustomGetValidator } from '../../../middlewares'
5
6const tokenRouter = express.Router()
7
8tokenRouter.post('/:id/token',
9 authenticate,
10 asyncMiddleware(videosCustomGetValidator('only-video')),
11 generateToken
12)
13
14// ---------------------------------------------------------------------------
15
16export {
17 tokenRouter
18}
19
20// ---------------------------------------------------------------------------
21
22function generateToken (req: express.Request, res: express.Response) {
23 const video = res.locals.onlyVideo
24
25 const { token, expires } = VideoTokensManager.Instance.create(video.uuid)
26
27 return res.json({
28 files: {
29 token,
30 expires
31 }
32 } as VideoToken)
33}
diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts
index 9aca761c1..8c9a5322b 100644
--- a/server/controllers/api/videos/transcoding.ts
+++ b/server/controllers/api/videos/transcoding.ts
@@ -32,9 +32,10 @@ async function createTranscoding (req: express.Request, res: express.Response) {
32 32
33 const body: VideoTranscodingCreate = req.body 33 const body: VideoTranscodingCreate = req.body
34 34
35 const { resolution: maxResolution, audioStream } = await video.probeMaxQualityFile() 35 const { resolution: maxResolution, hasAudio } = await video.probeMaxQualityFile()
36
36 const resolutions = await Hooks.wrapObject( 37 const resolutions = await Hooks.wrapObject(
37 computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false }), 38 computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false, hasAudio }),
38 'filter:transcoding.manual.resolutions-to-transcode.result', 39 'filter:transcoding.manual.resolutions-to-transcode.result',
39 body 40 body
40 ) 41 )
@@ -46,9 +47,10 @@ async function createTranscoding (req: express.Request, res: express.Response) {
46 video.state = VideoState.TO_TRANSCODE 47 video.state = VideoState.TO_TRANSCODE
47 await video.save() 48 await video.save()
48 49
49 const hasAudio = !!audioStream
50 const childrenResolutions = resolutions.filter(r => r !== maxResolution) 50 const childrenResolutions = resolutions.filter(r => r !== maxResolution)
51 51
52 logger.info('Manually creating transcoding jobs for %s.', body.transcodingType, { childrenResolutions, maxResolution })
53
52 const children = await Bluebird.mapSeries(childrenResolutions, resolution => { 54 const children = await Bluebird.mapSeries(childrenResolutions, resolution => {
53 if (body.transcodingType === 'hls') { 55 if (body.transcodingType === 'hls') {
54 return buildHLSJobOption({ 56 return buildHLSJobOption({
diff --git a/server/controllers/api/videos/update.ts b/server/controllers/api/videos/update.ts
index ab1a23d9a..260dee2b9 100644
--- a/server/controllers/api/videos/update.ts
+++ b/server/controllers/api/videos/update.ts
@@ -1,12 +1,12 @@
1import express from 'express' 1import express from 'express'
2import { Transaction } from 'sequelize/types' 2import { Transaction } from 'sequelize/types'
3import { changeVideoChannelShare } from '@server/lib/activitypub/share' 3import { changeVideoChannelShare } from '@server/lib/activitypub/share'
4import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' 4import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
5import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' 5import { setVideoPrivacy } from '@server/lib/video-privacy'
6import { openapiOperationDoc } from '@server/middlewares/doc' 6import { openapiOperationDoc } from '@server/middlewares/doc'
7import { FilteredModelAttributes } from '@server/types' 7import { FilteredModelAttributes } from '@server/types'
8import { MVideoFullLight } from '@server/types/models' 8import { MVideoFullLight } from '@server/types/models'
9import { HttpStatusCode, ManageVideoTorrentPayload, VideoUpdate } from '@shared/models' 9import { HttpStatusCode, VideoUpdate } from '@shared/models'
10import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' 10import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
11import { resetSequelizeInstance } from '../../../helpers/database-utils' 11import { resetSequelizeInstance } from '../../../helpers/database-utils'
12import { createReqFiles } from '../../../helpers/express-utils' 12import { createReqFiles } from '../../../helpers/express-utils'
@@ -18,6 +18,8 @@ import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
18import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares' 18import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares'
19import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' 19import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
20import { VideoModel } from '../../../models/video/video' 20import { VideoModel } from '../../../models/video/video'
21import { VideoPathManager } from '@server/lib/video-path-manager'
22import { forceNumber } from '@shared/core-utils'
21 23
22const lTags = loggerTagsFactory('api', 'video') 24const lTags = loggerTagsFactory('api', 'video')
23const auditLogger = auditLoggerFactory('videos') 25const auditLogger = auditLoggerFactory('videos')
@@ -47,8 +49,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
47 const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON()) 49 const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON())
48 const videoInfoToUpdate: VideoUpdate = req.body 50 const videoInfoToUpdate: VideoUpdate = req.body
49 51
50 const wasConfidentialVideo = videoFromReq.isConfidential()
51 const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation() 52 const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation()
53 const oldPrivacy = videoFromReq.privacy
52 54
53 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ 55 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
54 video: videoFromReq, 56 video: videoFromReq,
@@ -57,12 +59,13 @@ async function updateVideo (req: express.Request, res: express.Response) {
57 automaticallyGenerated: false 59 automaticallyGenerated: false
58 }) 60 })
59 61
62 const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid)
63
60 try { 64 try {
61 const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => { 65 const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => {
62 // Refresh video since thumbnails to prevent concurrent updates 66 // Refresh video since thumbnails to prevent concurrent updates
63 const video = await VideoModel.loadFull(videoFromReq.id, t) 67 const video = await VideoModel.loadFull(videoFromReq.id, t)
64 68
65 const sequelizeOptions = { transaction: t }
66 const oldVideoChannel = video.VideoChannel 69 const oldVideoChannel = video.VideoChannel
67 70
68 const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [ 71 const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [
@@ -97,7 +100,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
97 await video.setAsRefreshed(t) 100 await video.setAsRefreshed(t)
98 } 101 }
99 102
100 const videoInstanceUpdated = await video.save(sequelizeOptions) as MVideoFullLight 103 const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight
101 104
102 // Thumbnail & preview updates? 105 // Thumbnail & preview updates?
103 if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) 106 if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t)
@@ -113,7 +116,9 @@ async function updateVideo (req: express.Request, res: express.Response) {
113 await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t }) 116 await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t })
114 videoInstanceUpdated.VideoChannel = res.locals.videoChannel 117 videoInstanceUpdated.VideoChannel = res.locals.videoChannel
115 118
116 if (hadPrivacyForFederation === true) await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t) 119 if (hadPrivacyForFederation === true) {
120 await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t)
121 }
117 } 122 }
118 123
119 // Schedule an update in the future? 124 // Schedule an update in the future?
@@ -139,7 +144,12 @@ async function updateVideo (req: express.Request, res: express.Response) {
139 144
140 Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res }) 145 Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res })
141 146
142 await addVideoJobsAfterUpdate({ video: videoInstanceUpdated, videoInfoToUpdate, wasConfidentialVideo, isNewVideo }) 147 await addVideoJobsAfterUpdate({
148 video: videoInstanceUpdated,
149 nameChanged: !!videoInfoToUpdate.name,
150 oldPrivacy,
151 isNewVideo
152 })
143 } catch (err) { 153 } catch (err) {
144 // Force fields we want to update 154 // Force fields we want to update
145 // If the transaction is retried, sequelize will think the object has not changed 155 // If the transaction is retried, sequelize will think the object has not changed
@@ -147,6 +157,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
147 resetSequelizeInstance(videoFromReq, videoFieldsSave) 157 resetSequelizeInstance(videoFromReq, videoFieldsSave)
148 158
149 throw err 159 throw err
160 } finally {
161 videoFileLockReleaser()
150 } 162 }
151 163
152 return res.type('json') 164 return res.type('json')
@@ -163,8 +175,8 @@ async function updateVideoPrivacy (options: {
163 const { videoInstance, videoInfoToUpdate, hadPrivacyForFederation, transaction } = options 175 const { videoInstance, videoInfoToUpdate, hadPrivacyForFederation, transaction } = options
164 const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy) 176 const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy)
165 177
166 const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10) 178 const newPrivacy = forceNumber(videoInfoToUpdate.privacy)
167 videoInstance.setPrivacy(newPrivacy) 179 setVideoPrivacy(videoInstance, newPrivacy)
168 180
169 // Unfederate the video if the new privacy is not compatible with federation 181 // Unfederate the video if the new privacy is not compatible with federation
170 if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) { 182 if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) {
@@ -185,50 +197,3 @@ function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: Vide
185 return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction) 197 return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction)
186 } 198 }
187} 199}
188
189async function addVideoJobsAfterUpdate (options: {
190 video: MVideoFullLight
191 videoInfoToUpdate: VideoUpdate
192 wasConfidentialVideo: boolean
193 isNewVideo: boolean
194}) {
195 const { video, videoInfoToUpdate, wasConfidentialVideo, isNewVideo } = options
196 const jobs: CreateJobArgument[] = []
197
198 if (!video.isLive && videoInfoToUpdate.name) {
199
200 for (const file of (video.VideoFiles || [])) {
201 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
202
203 jobs.push({ type: 'manage-video-torrent', payload })
204 }
205
206 const hls = video.getHLSPlaylist()
207
208 for (const file of (hls?.VideoFiles || [])) {
209 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
210
211 jobs.push({ type: 'manage-video-torrent', payload })
212 }
213 }
214
215 jobs.push({
216 type: 'federate-video',
217 payload: {
218 videoUUID: video.uuid,
219 isNewVideo
220 }
221 })
222
223 if (wasConfidentialVideo) {
224 jobs.push({
225 type: 'notify',
226 payload: {
227 action: 'new-video',
228 videoUUID: video.uuid
229 }
230 })
231 }
232
233 return JobQueue.Instance.createSequentialJobFlow(...jobs)
234}
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index a270180c0..65b9a1d1b 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -5,9 +5,10 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache
5import { Hooks } from '@server/lib/plugins/hooks' 5import { Hooks } from '@server/lib/plugins/hooks'
6import { VideoPathManager } from '@server/lib/video-path-manager' 6import { VideoPathManager } from '@server/lib/video-path-manager'
7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
8import { addQueryParams, forceNumber } from '@shared/core-utils'
8import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' 9import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' 10import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
10import { asyncMiddleware, videosDownloadValidator } from '../middlewares' 11import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares'
11 12
12const downloadRouter = express.Router() 13const downloadRouter = express.Router()
13 14
@@ -20,12 +21,14 @@ downloadRouter.use(
20 21
21downloadRouter.use( 22downloadRouter.use(
22 STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', 23 STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
24 optionalAuthenticate,
23 asyncMiddleware(videosDownloadValidator), 25 asyncMiddleware(videosDownloadValidator),
24 asyncMiddleware(downloadVideoFile) 26 asyncMiddleware(downloadVideoFile)
25) 27)
26 28
27downloadRouter.use( 29downloadRouter.use(
28 STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', 30 STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension',
31 optionalAuthenticate,
29 asyncMiddleware(videosDownloadValidator), 32 asyncMiddleware(videosDownloadValidator),
30 asyncMiddleware(downloadHLSVideoFile) 33 asyncMiddleware(downloadHLSVideoFile)
31) 34)
@@ -82,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
82 if (!checkAllowResult(res, allowParameters, allowedResult)) return 85 if (!checkAllowResult(res, allowParameters, allowedResult)) return
83 86
84 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { 87 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
85 return res.redirect(videoFile.getObjectStorageUrl()) 88 return redirectToObjectStorage({ req, res, video, file: videoFile })
86 } 89 }
87 90
88 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { 91 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
@@ -118,7 +121,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
118 if (!checkAllowResult(res, allowParameters, allowedResult)) return 121 if (!checkAllowResult(res, allowParameters, allowedResult)) return
119 122
120 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { 123 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
121 return res.redirect(videoFile.getObjectStorageUrl()) 124 return redirectToObjectStorage({ req, res, video, file: videoFile })
122 } 125 }
123 126
124 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { 127 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
@@ -129,7 +132,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
129} 132}
130 133
131function getVideoFile (req: express.Request, files: MVideoFile[]) { 134function getVideoFile (req: express.Request, files: MVideoFile[]) {
132 const resolution = parseInt(req.params.resolution, 10) 135 const resolution = forceNumber(req.params.resolution)
133 return files.find(f => f.resolution === resolution) 136 return files.find(f => f.resolution === resolution)
134} 137}
135 138
@@ -172,3 +175,20 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
172 175
173 return true 176 return true
174} 177}
178
179function redirectToObjectStorage (options: {
180 req: express.Request
181 res: express.Response
182 video: MVideo
183 file: MVideoFile
184}) {
185 const { req, res, video, file } = options
186
187 const baseUrl = file.getObjectStorageUrl(video)
188
189 const url = video.hasPrivateStaticPath() && req.query.videoFileToken
190 ? addQueryParams(baseUrl, { videoFileToken: req.query.videoFileToken })
191 : baseUrl
192
193 return res.redirect(url)
194}
diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts
index 241715fb9..772fe734d 100644
--- a/server/controllers/feeds.ts
+++ b/server/controllers/feeds.ts
@@ -4,7 +4,8 @@ import { Feed } from '@peertube/feed'
4import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' 4import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown'
5import { getServerActor } from '@server/models/application/application' 5import { getServerActor } from '@server/models/application/application'
6import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils' 6import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils'
7import { VideoInclude } from '@shared/models' 7import { MAccountDefault, MChannelBannerAccountDefault, MVideoFullLight } from '@server/types/models'
8import { ActorImageType, VideoInclude } from '@shared/models'
8import { buildNSFWFilter } from '../helpers/express-utils' 9import { buildNSFWFilter } from '../helpers/express-utils'
9import { CONFIG } from '../initializers/config' 10import { CONFIG } from '../initializers/config'
10import { MIMETYPES, PREVIEWS_SIZE, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' 11import { MIMETYPES, PREVIEWS_SIZE, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
@@ -82,22 +83,12 @@ async function generateVideoCommentsFeed (req: express.Request, res: express.Res
82 videoChannelId: videoChannel ? videoChannel.id : undefined 83 videoChannelId: videoChannel ? videoChannel.id : undefined
83 }) 84 })
84 85
85 let name: string 86 const { name, description, imageUrl } = buildFeedMetadata({ video, account, videoChannel })
86 let description: string
87 87
88 if (videoChannel) {
89 name = videoChannel.getDisplayName()
90 description = videoChannel.description
91 } else if (account) {
92 name = account.getDisplayName()
93 description = account.description
94 } else {
95 name = video ? video.name : CONFIG.INSTANCE.NAME
96 description = video ? video.description : CONFIG.INSTANCE.DESCRIPTION
97 }
98 const feed = initFeed({ 88 const feed = initFeed({
99 name, 89 name,
100 description, 90 description,
91 imageUrl,
101 resourceType: 'video-comments', 92 resourceType: 'video-comments',
102 queryString: new URL(WEBSERVER.URL + req.originalUrl).search 93 queryString: new URL(WEBSERVER.URL + req.originalUrl).search
103 }) 94 })
@@ -137,23 +128,12 @@ async function generateVideoFeed (req: express.Request, res: express.Response) {
137 const videoChannel = res.locals.videoChannel 128 const videoChannel = res.locals.videoChannel
138 const nsfw = buildNSFWFilter(res, req.query.nsfw) 129 const nsfw = buildNSFWFilter(res, req.query.nsfw)
139 130
140 let name: string 131 const { name, description, imageUrl } = buildFeedMetadata({ videoChannel, account })
141 let description: string
142
143 if (videoChannel) {
144 name = videoChannel.getDisplayName()
145 description = videoChannel.description
146 } else if (account) {
147 name = account.getDisplayName()
148 description = account.description
149 } else {
150 name = CONFIG.INSTANCE.NAME
151 description = CONFIG.INSTANCE.DESCRIPTION
152 }
153 132
154 const feed = initFeed({ 133 const feed = initFeed({
155 name, 134 name,
156 description, 135 description,
136 imageUrl,
157 resourceType: 'videos', 137 resourceType: 'videos',
158 queryString: new URL(WEBSERVER.URL + req.url).search 138 queryString: new URL(WEBSERVER.URL + req.url).search
159 }) 139 })
@@ -190,12 +170,13 @@ async function generateVideoFeedForSubscriptions (req: express.Request, res: exp
190 const start = 0 170 const start = 0
191 const account = res.locals.account 171 const account = res.locals.account
192 const nsfw = buildNSFWFilter(res, req.query.nsfw) 172 const nsfw = buildNSFWFilter(res, req.query.nsfw)
193 const name = account.getDisplayName() 173
194 const description = account.description 174 const { name, description, imageUrl } = buildFeedMetadata({ account })
195 175
196 const feed = initFeed({ 176 const feed = initFeed({
197 name, 177 name,
198 description, 178 description,
179 imageUrl,
199 resourceType: 'videos', 180 resourceType: 'videos',
200 queryString: new URL(WEBSERVER.URL + req.url).search 181 queryString: new URL(WEBSERVER.URL + req.url).search
201 }) 182 })
@@ -229,11 +210,12 @@ async function generateVideoFeedForSubscriptions (req: express.Request, res: exp
229function initFeed (parameters: { 210function initFeed (parameters: {
230 name: string 211 name: string
231 description: string 212 description: string
213 imageUrl: string
232 resourceType?: 'videos' | 'video-comments' 214 resourceType?: 'videos' | 'video-comments'
233 queryString?: string 215 queryString?: string
234}) { 216}) {
235 const webserverUrl = WEBSERVER.URL 217 const webserverUrl = WEBSERVER.URL
236 const { name, description, resourceType, queryString } = parameters 218 const { name, description, resourceType, queryString, imageUrl } = parameters
237 219
238 return new Feed({ 220 return new Feed({
239 title: name, 221 title: name,
@@ -241,7 +223,7 @@ function initFeed (parameters: {
241 // updated: TODO: somehowGetLatestUpdate, // optional, default = today 223 // updated: TODO: somehowGetLatestUpdate, // optional, default = today
242 id: webserverUrl, 224 id: webserverUrl,
243 link: webserverUrl, 225 link: webserverUrl,
244 image: webserverUrl + '/client/assets/images/icons/icon-96x96.png', 226 image: imageUrl,
245 favicon: webserverUrl + '/client/assets/images/favicon.png', 227 favicon: webserverUrl + '/client/assets/images/favicon.png',
246 copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + 228 copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` +
247 ` and potential licenses granted by each content's rightholder.`, 229 ` and potential licenses granted by each content's rightholder.`,
@@ -369,3 +351,39 @@ function sendFeed (feed: Feed, req: express.Request, res: express.Response) {
369 351
370 return res.send(feed.rss2()).end() 352 return res.send(feed.rss2()).end()
371} 353}
354
355function buildFeedMetadata (options: {
356 videoChannel?: MChannelBannerAccountDefault
357 account?: MAccountDefault
358 video?: MVideoFullLight
359}) {
360 const { video, videoChannel, account } = options
361
362 let imageUrl = WEBSERVER.URL + '/client/assets/images/icons/icon-96x96.png'
363 let name: string
364 let description: string
365
366 if (videoChannel) {
367 name = videoChannel.getDisplayName()
368 description = videoChannel.description
369
370 if (videoChannel.Actor.hasImage(ActorImageType.AVATAR)) {
371 imageUrl = WEBSERVER.URL + videoChannel.Actor.Avatars[0].getStaticPath()
372 }
373 } else if (account) {
374 name = account.getDisplayName()
375 description = account.description
376
377 if (account.Actor.hasImage(ActorImageType.AVATAR)) {
378 imageUrl = WEBSERVER.URL + account.Actor.Avatars[0].getStaticPath()
379 }
380 } else if (video) {
381 name = video.name
382 description = video.description
383 } else {
384 name = CONFIG.INSTANCE.NAME
385 description = CONFIG.INSTANCE.DESCRIPTION
386 }
387
388 return { name, description, imageUrl }
389}
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index e8833d58c..eaa2dd7c8 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -1,15 +1,15 @@
1export * from './activitypub' 1export * from './activitypub'
2export * from './api' 2export * from './api'
3export * from './bots'
3export * from './client' 4export * from './client'
4export * from './download' 5export * from './download'
5export * from './feeds' 6export * from './feeds'
6export * from './services'
7export * from './static'
8export * from './lazy-static' 7export * from './lazy-static'
9export * from './live'
10export * from './misc' 8export * from './misc'
11export * from './webfinger' 9export * from './object-storage-proxy'
12export * from './tracker'
13export * from './bots'
14export * from './plugins' 10export * from './plugins'
11export * from './services'
12export * from './static'
13export * from './tracker'
14export * from './webfinger'
15export * from './well-known' 15export * from './well-known'
diff --git a/server/controllers/live.ts b/server/controllers/live.ts
deleted file mode 100644
index 81008f120..000000000
--- a/server/controllers/live.ts
+++ /dev/null
@@ -1,32 +0,0 @@
1import cors from 'cors'
2import express from 'express'
3import { mapToJSON } from '@server/helpers/core-utils'
4import { LiveSegmentShaStore } from '@server/lib/live'
5import { HttpStatusCode } from '@shared/models'
6
7const liveRouter = express.Router()
8
9liveRouter.use('/segments-sha256/:videoUUID',
10 cors(),
11 getSegmentsSha256
12)
13
14// ---------------------------------------------------------------------------
15
16export {
17 liveRouter
18}
19
20// ---------------------------------------------------------------------------
21
22function getSegmentsSha256 (req: express.Request, res: express.Response) {
23 const videoUUID = req.params.videoUUID
24
25 const result = LiveSegmentShaStore.Instance.getSegmentsSha256(videoUUID)
26
27 if (!result) {
28 return res.status(HttpStatusCode.NOT_FOUND_404).end()
29 }
30
31 return res.json(mapToJSON(result))
32}
diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts
new file mode 100644
index 000000000..3ce279671
--- /dev/null
+++ b/server/controllers/object-storage-proxy.ts
@@ -0,0 +1,86 @@
1import cors from 'cors'
2import express from 'express'
3import { logger } from '@server/helpers/logger'
4import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
5import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
6import {
7 asyncMiddleware,
8 ensureCanAccessPrivateVideoHLSFiles,
9 ensureCanAccessVideoPrivateWebTorrentFiles,
10 ensurePrivateObjectStorageProxyIsEnabled,
11 optionalAuthenticate
12} from '@server/middlewares'
13import { HttpStatusCode } from '@shared/models'
14
15const objectStorageProxyRouter = express.Router()
16
17objectStorageProxyRouter.use(cors())
18
19objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename',
20 ensurePrivateObjectStorageProxyIsEnabled,
21 optionalAuthenticate,
22 asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
23 asyncMiddleware(proxifyWebTorrent)
24)
25
26objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
27 ensurePrivateObjectStorageProxyIsEnabled,
28 optionalAuthenticate,
29 asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
30 asyncMiddleware(proxifyHLS)
31)
32
33// ---------------------------------------------------------------------------
34
35export {
36 objectStorageProxyRouter
37}
38
39async function proxifyWebTorrent (req: express.Request, res: express.Response) {
40 const filename = req.params.filename
41
42 logger.debug('Proxifying WebTorrent file %s from object storage.', filename)
43
44 try {
45 const stream = await getWebTorrentFileReadStream({
46 filename,
47 rangeHeader: req.header('range')
48 })
49
50 return stream.pipe(res)
51 } catch (err) {
52 return handleObjectStorageFailure(res, err)
53 }
54}
55
56async function proxifyHLS (req: express.Request, res: express.Response) {
57 const playlist = res.locals.videoStreamingPlaylist
58 const video = res.locals.onlyVideo
59 const filename = req.params.filename
60
61 logger.debug('Proxifying HLS file %s from object storage.', filename)
62
63 try {
64 const stream = await getHLSFileReadStream({
65 playlist: playlist.withVideo(video),
66 filename,
67 rangeHeader: req.header('range')
68 })
69
70 return stream.pipe(res)
71 } catch (err) {
72 return handleObjectStorageFailure(res, err)
73 }
74}
75
76function handleObjectStorageFailure (res: express.Response, err: Error) {
77 if (err.name === 'NoSuchKey') {
78 return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
79 }
80
81 return res.fail({
82 status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
83 message: err.message,
84 type: err.name
85 })
86}
diff --git a/server/controllers/services.ts b/server/controllers/services.ts
index cabcbc00b..7c7ca1ff3 100644
--- a/server/controllers/services.ts
+++ b/server/controllers/services.ts
@@ -4,6 +4,7 @@ import { escapeHTML } from '@shared/core-utils/renderer'
4import { EMBED_SIZE, PREVIEWS_SIZE, THUMBNAILS_SIZE, WEBSERVER } from '../initializers/constants' 4import { EMBED_SIZE, PREVIEWS_SIZE, THUMBNAILS_SIZE, WEBSERVER } from '../initializers/constants'
5import { asyncMiddleware, oembedValidator } from '../middlewares' 5import { asyncMiddleware, oembedValidator } from '../middlewares'
6import { accountNameWithHostGetValidator } from '../middlewares/validators' 6import { accountNameWithHostGetValidator } from '../middlewares/validators'
7import { forceNumber } from '@shared/core-utils'
7 8
8const servicesRouter = express.Router() 9const servicesRouter = express.Router()
9 10
@@ -108,8 +109,8 @@ function buildOEmbed (options: {
108 const { req, previewSize, previewPath, title, channel, embedPath } = options 109 const { req, previewSize, previewPath, title, channel, embedPath } = options
109 110
110 const webserverUrl = WEBSERVER.URL 111 const webserverUrl = WEBSERVER.URL
111 const maxHeight = parseInt(req.query.maxheight, 10) 112 const maxHeight = forceNumber(req.query.maxheight)
112 const maxWidth = parseInt(req.query.maxwidth, 10) 113 const maxWidth = forceNumber(req.query.maxwidth)
113 114
114 const embedUrl = webserverUrl + embedPath 115 const embedUrl = webserverUrl + embedPath
115 const embedTitle = escapeHTML(title) 116 const embedTitle = escapeHTML(title)
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 33c429eb1..6ef9154b9 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,30 +1,63 @@
1import cors from 'cors' 1import cors from 'cors'
2import express from 'express' 2import express from 'express'
3import { handleStaticError } from '@server/middlewares' 3import {
4 asyncMiddleware,
5 ensureCanAccessPrivateVideoHLSFiles,
6 ensureCanAccessVideoPrivateWebTorrentFiles,
7 handleStaticError,
8 optionalAuthenticate
9} from '@server/middlewares'
4import { CONFIG } from '../initializers/config' 10import { CONFIG } from '../initializers/config'
5import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' 11import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
6 12
7const staticRouter = express.Router() 13const staticRouter = express.Router()
8 14
9// Cors is very important to let other servers access torrent and video files 15// Cors is very important to let other servers access torrent and video files
10staticRouter.use(cors()) 16staticRouter.use(cors())
11 17
12// Videos path for webseed 18// ---------------------------------------------------------------------------
19// WebTorrent/Classic videos
20// ---------------------------------------------------------------------------
21
22const privateWebTorrentStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true
23 ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles) ]
24 : []
25
26staticRouter.use(
27 STATIC_PATHS.PRIVATE_WEBSEED,
28 ...privateWebTorrentStaticMiddlewares,
29 express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
30 handleStaticError
31)
13staticRouter.use( 32staticRouter.use(
14 STATIC_PATHS.WEBSEED, 33 STATIC_PATHS.WEBSEED,
15 express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }), 34 express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }),
16 handleStaticError 35 handleStaticError
17) 36)
37
18staticRouter.use( 38staticRouter.use(
19 STATIC_PATHS.REDUNDANCY, 39 STATIC_PATHS.REDUNDANCY,
20 express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }), 40 express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }),
21 handleStaticError 41 handleStaticError
22) 42)
23 43
44// ---------------------------------------------------------------------------
24// HLS 45// HLS
46// ---------------------------------------------------------------------------
47
48const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true
49 ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ]
50 : []
51
52staticRouter.use(
53 STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
54 ...privateHLSStaticMiddlewares,
55 express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
56 handleStaticError
57)
25staticRouter.use( 58staticRouter.use(
26 STATIC_PATHS.STREAMING_PLAYLISTS.HLS, 59 STATIC_PATHS.STREAMING_PLAYLISTS.HLS,
27 express.static(HLS_STREAMING_PLAYLIST_DIRECTORY, { fallthrough: false }), 60 express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }),
28 handleStaticError 61 handleStaticError
29) 62)
30 63
diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts
index f467bd629..ce5883571 100644
--- a/server/controllers/well-known.ts
+++ b/server/controllers/well-known.ts
@@ -5,6 +5,7 @@ import { root } from '@shared/core-utils'
5import { CONFIG } from '../initializers/config' 5import { CONFIG } from '../initializers/config'
6import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' 6import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
7import { cacheRoute } from '../middlewares/cache/cache' 7import { cacheRoute } from '../middlewares/cache/cache'
8import { handleStaticError } from '@server/middlewares'
8 9
9const wellKnownRouter = express.Router() 10const wellKnownRouter = express.Router()
10 11
@@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta',
69 } 70 }
70) 71)
71 72
73wellKnownRouter.use('/.well-known/',
74 cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN),
75 express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }),
76 handleStaticError
77)
78
72// --------------------------------------------------------------------------- 79// ---------------------------------------------------------------------------
73 80
74export { 81export {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index c762f6a29..73bd994c1 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -6,7 +6,7 @@
6*/ 6*/
7 7
8import { exec, ExecOptions } from 'child_process' 8import { exec, ExecOptions } from 'child_process'
9import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions } from 'crypto' 9import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
10import { truncate } from 'lodash' 10import { truncate } from 'lodash'
11import { pipeline } from 'stream' 11import { pipeline } from 'stream'
12import { URL } from 'url' 12import { URL } from 'url'
@@ -311,7 +311,17 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
311 } 311 }
312} 312}
313 313
314// eslint-disable-next-line max-len
315function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> {
316 return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> {
317 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
318 func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ])
319 })
320 }
321}
322
314const randomBytesPromise = promisify1<number, Buffer>(randomBytes) 323const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
324const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
315const execPromise2 = promisify2<string, any, string>(exec) 325const execPromise2 = promisify2<string, any, string>(exec)
316const execPromise = promisify1<string, string>(exec) 326const execPromise = promisify1<string, string>(exec)
317const pipelinePromise = promisify(pipeline) 327const pipelinePromise = promisify(pipeline)
@@ -339,6 +349,8 @@ export {
339 promisify1, 349 promisify1,
340 promisify2, 350 promisify2,
341 351
352 scryptPromise,
353
342 randomBytesPromise, 354 randomBytesPromise,
343 355
344 generateRSAKeyPairPromise, 356 generateRSAKeyPairPromise,
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 2a2f008b9..97b3577af 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -7,11 +7,11 @@ import { peertubeTruncate } from '../../core-utils'
7import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' 7import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
8import { isLiveLatencyModeValid } from '../video-lives' 8import { isLiveLatencyModeValid } from '../video-lives'
9import { 9import {
10 isVideoDescriptionValid,
10 isVideoDurationValid, 11 isVideoDurationValid,
11 isVideoNameValid, 12 isVideoNameValid,
12 isVideoStateValid, 13 isVideoStateValid,
13 isVideoTagValid, 14 isVideoTagValid,
14 isVideoTruncatedDescriptionValid,
15 isVideoViewsValid 15 isVideoViewsValid
16} from '../videos' 16} from '../videos'
17import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' 17import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc'
@@ -32,7 +32,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
32 logger.debug('Video has invalid urls', { video }) 32 logger.debug('Video has invalid urls', { video })
33 return false 33 return false
34 } 34 }
35 if (!setRemoteVideoTruncatedContent(video)) { 35 if (!setRemoteVideoContent(video)) {
36 logger.debug('Video has invalid content', { video }) 36 logger.debug('Video has invalid content', { video })
37 return false 37 return false
38 } 38 }
@@ -168,7 +168,7 @@ function isRemoteStringIdentifierValid (data: any) {
168} 168}
169 169
170function isRemoteVideoContentValid (mediaType: string, content: string) { 170function isRemoteVideoContentValid (mediaType: string, content: string) {
171 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) 171 return mediaType === 'text/markdown' && isVideoDescriptionValid(content)
172} 172}
173 173
174function setValidRemoteIcon (video: any) { 174function setValidRemoteIcon (video: any) {
@@ -194,9 +194,9 @@ function setValidRemoteVideoUrls (video: any) {
194 return true 194 return true
195} 195}
196 196
197function setRemoteVideoTruncatedContent (video: any) { 197function setRemoteVideoContent (video: any) {
198 if (video.content) { 198 if (video.content) {
199 video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max }) 199 video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max })
200 } 200 }
201 201
202 return true 202 return true
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index 17750379d..3dc5504e3 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -86,7 +86,7 @@ function isFileValid (options: {
86 86
87 // The file exists 87 // The file exists
88 const file = fileArray[0] 88 const file = fileArray[0]
89 if (!file || !file.originalname) return false 89 if (!file?.originalname) return false
90 90
91 // Check size 91 // Check size
92 if ((maxSize !== null) && file.size > maxSize) return false 92 if ((maxSize !== null) && file.size > maxSize) return false
diff --git a/server/helpers/custom-validators/plugins.ts b/server/helpers/custom-validators/plugins.ts
index 60b29dc89..a20de0c4a 100644
--- a/server/helpers/custom-validators/plugins.ts
+++ b/server/helpers/custom-validators/plugins.ts
@@ -1,9 +1,9 @@
1import { exists, isArray, isSafePath } from './misc'
2import validator from 'validator' 1import validator from 'validator'
2import { PluginPackageJSON } from '../../../shared/models/plugins/plugin-package-json.model'
3import { PluginType } from '../../../shared/models/plugins/plugin.type' 3import { PluginType } from '../../../shared/models/plugins/plugin.type'
4import { CONSTRAINTS_FIELDS } from '../../initializers/constants' 4import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
5import { PluginPackageJSON } from '../../../shared/models/plugins/plugin-package-json.model'
6import { isUrlValid } from './activitypub/misc' 5import { isUrlValid } from './activitypub/misc'
6import { exists, isArray, isSafePath } from './misc'
7 7
8const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS 8const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS
9 9
@@ -29,7 +29,7 @@ function isPluginDescriptionValid (value: string) {
29 return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION) 29 return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION)
30} 30}
31 31
32function isPluginVersionValid (value: string) { 32function isPluginStableVersionValid (value: string) {
33 if (!exists(value)) return false 33 if (!exists(value)) return false
34 34
35 const parts = (value + '').split('.') 35 const parts = (value + '').split('.')
@@ -37,6 +37,19 @@ function isPluginVersionValid (value: string) {
37 return parts.length === 3 && parts.every(p => validator.isInt(p)) 37 return parts.length === 3 && parts.every(p => validator.isInt(p))
38} 38}
39 39
40function isPluginStableOrUnstableVersionValid (value: string) {
41 if (!exists(value)) return false
42
43 // suffix is beta.x or alpha.x
44 const [ stable, suffix ] = value.split('-')
45 if (!isPluginStableVersionValid(stable)) return false
46
47 const suffixRegex = /^(rc|alpha|beta)\.\d+$/
48 if (suffix && !suffixRegex.test(suffix)) return false
49
50 return true
51}
52
40function isPluginEngineValid (engine: any) { 53function isPluginEngineValid (engine: any) {
41 return exists(engine) && exists(engine.peertube) 54 return exists(engine) && exists(engine.peertube)
42} 55}
@@ -156,7 +169,8 @@ export {
156 isPackageJSONValid, 169 isPackageJSONValid,
157 isThemeNameValid, 170 isThemeNameValid,
158 isPluginHomepage, 171 isPluginHomepage,
159 isPluginVersionValid, 172 isPluginStableVersionValid,
173 isPluginStableOrUnstableVersionValid,
160 isPluginNameValid, 174 isPluginNameValid,
161 isPluginDescriptionValid, 175 isPluginDescriptionValid,
162 isLibraryCodeValid, 176 isLibraryCodeValid,
diff --git a/server/helpers/custom-validators/servers.ts b/server/helpers/custom-validators/servers.ts
index b9f45c282..94fda05aa 100644
--- a/server/helpers/custom-validators/servers.ts
+++ b/server/helpers/custom-validators/servers.ts
@@ -1,6 +1,6 @@
1import validator from 'validator' 1import validator from 'validator'
2import { CONFIG } from '@server/initializers/config'
2import { CONSTRAINTS_FIELDS } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
3import { isTestOrDevInstance } from '../core-utils'
4import { exists, isArray } from './misc' 4import { exists, isArray } from './misc'
5 5
6function isHostValid (host: string) { 6function isHostValid (host: string) {
@@ -10,7 +10,7 @@ function isHostValid (host: string) {
10 } 10 }
11 11
12 // We validate 'localhost', so we don't have the top level domain 12 // We validate 'localhost', so we don't have the top level domain
13 if (isTestOrDevInstance()) { 13 if (CONFIG.WEBSERVER.HOSTNAME === 'localhost') {
14 isURLOptions.require_tld = false 14 isURLOptions.require_tld = false
15 } 15 }
16 16
diff --git a/server/helpers/custom-validators/video-studio.ts b/server/helpers/custom-validators/video-studio.ts
index 19e7906d5..68dfec8dd 100644
--- a/server/helpers/custom-validators/video-studio.ts
+++ b/server/helpers/custom-validators/video-studio.ts
@@ -4,6 +4,7 @@ import { buildTaskFileFieldname } from '@server/lib/video-studio'
4import { VideoStudioTask } from '@shared/models' 4import { VideoStudioTask } from '@shared/models'
5import { isArray } from './misc' 5import { isArray } from './misc'
6import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos' 6import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos'
7import { forceNumber } from '@shared/core-utils'
7 8
8function isValidStudioTasksArray (tasks: any) { 9function isValidStudioTasksArray (tasks: any) {
9 if (!isArray(tasks)) return false 10 if (!isArray(tasks)) return false
@@ -24,7 +25,7 @@ function isStudioCutTaskValid (task: VideoStudioTask) {
24 25
25 if (!start || !end) return true 26 if (!start || !end) return true
26 27
27 return parseInt(start + '') < parseInt(end + '') 28 return forceNumber(start) < forceNumber(end)
28} 29}
29 30
30function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { 31function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 3ebfe2937..9e8177f77 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -45,10 +45,6 @@ function isVideoDurationValid (value: string) {
45 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) 45 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION)
46} 46}
47 47
48function isVideoTruncatedDescriptionValid (value: string) {
49 return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.TRUNCATED_DESCRIPTION)
50}
51
52function isVideoDescriptionValid (value: string) { 48function isVideoDescriptionValid (value: string) {
53 return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)) 49 return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION))
54} 50}
@@ -151,7 +147,6 @@ export {
151 isVideoCategoryValid, 147 isVideoCategoryValid,
152 isVideoLicenceValid, 148 isVideoLicenceValid,
153 isVideoLanguageValid, 149 isVideoLanguageValid,
154 isVideoTruncatedDescriptionValid,
155 isVideoDescriptionValid, 150 isVideoDescriptionValid,
156 isVideoFileInfoHashValid, 151 isVideoFileInfoHashValid,
157 isVideoNameValid, 152 isVideoNameValid,
diff --git a/server/helpers/ffmpeg/ffmpeg-commons.ts b/server/helpers/ffmpeg/ffmpeg-commons.ts
index b01989899..3906a2089 100644
--- a/server/helpers/ffmpeg/ffmpeg-commons.ts
+++ b/server/helpers/ffmpeg/ffmpeg-commons.ts
@@ -38,7 +38,7 @@ function getFFmpegVersion () {
38 return execPromise(`${ffmpegPath} -version`) 38 return execPromise(`${ffmpegPath} -version`)
39 .then(stdout => { 39 .then(stdout => {
40 const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) 40 const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/)
41 if (!parsed || !parsed[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) 41 if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`))
42 42
43 // Fix ffmpeg version that does not include patch version (4.4 for example) 43 // Fix ffmpeg version that does not include patch version (4.4 for example)
44 let version = parsed[1] 44 let version = parsed[1]
diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts
index 7a81a1313..d84703eb9 100644
--- a/server/helpers/ffmpeg/ffmpeg-vod.ts
+++ b/server/helpers/ffmpeg/ffmpeg-vod.ts
@@ -1,14 +1,15 @@
1import { MutexInterface } from 'async-mutex'
1import { Job } from 'bullmq' 2import { Job } from 'bullmq'
2import { FfmpegCommand } from 'fluent-ffmpeg' 3import { FfmpegCommand } from 'fluent-ffmpeg'
3import { readFile, writeFile } from 'fs-extra' 4import { readFile, writeFile } from 'fs-extra'
4import { dirname } from 'path' 5import { dirname } from 'path'
6import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
5import { pick } from '@shared/core-utils' 7import { pick } from '@shared/core-utils'
6import { AvailableEncoders, VideoResolution } from '@shared/models' 8import { AvailableEncoders, VideoResolution } from '@shared/models'
7import { logger, loggerTagsFactory } from '../logger' 9import { logger, loggerTagsFactory } from '../logger'
8import { getFFmpeg, runCommand } from './ffmpeg-commons' 10import { getFFmpeg, runCommand } from './ffmpeg-commons'
9import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' 11import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
10import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils' 12import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
11import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
12 13
13const lTags = loggerTagsFactory('ffmpeg') 14const lTags = loggerTagsFactory('ffmpeg')
14 15
@@ -22,6 +23,10 @@ interface BaseTranscodeVODOptions {
22 inputPath: string 23 inputPath: string
23 outputPath: string 24 outputPath: string
24 25
26 // Will be released after the ffmpeg started
27 // To prevent a bug where the input file does not exist anymore when running ffmpeg
28 inputFileMutexReleaser: MutexInterface.Releaser
29
25 availableEncoders: AvailableEncoders 30 availableEncoders: AvailableEncoders
26 profile: string 31 profile: string
27 32
@@ -94,6 +99,12 @@ async function transcodeVOD (options: TranscodeVODOptions) {
94 99
95 command = await builders[options.type](command, options) 100 command = await builders[options.type](command, options)
96 101
102 command.on('start', () => {
103 setTimeout(() => {
104 options.inputFileMutexReleaser()
105 }, 1000)
106 })
107
97 await runCommand({ command, job: options.job }) 108 await runCommand({ command, job: options.job })
98 109
99 await fixHLSPlaylistIfNeeded(options) 110 await fixHLSPlaylistIfNeeded(options)
diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts
index 2c6253d44..fb270b3cb 100644
--- a/server/helpers/ffmpeg/ffprobe-utils.ts
+++ b/server/helpers/ffmpeg/ffprobe-utils.ts
@@ -15,6 +15,7 @@ import {
15import { VideoResolution, VideoTranscodingFPS } from '@shared/models' 15import { VideoResolution, VideoTranscodingFPS } from '@shared/models'
16import { CONFIG } from '../../initializers/config' 16import { CONFIG } from '../../initializers/config'
17import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' 17import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants'
18import { toEven } from '../core-utils'
18import { logger } from '../logger' 19import { logger } from '../logger'
19 20
20/** 21/**
@@ -96,8 +97,9 @@ function computeResolutionsToTranscode (options: {
96 type: 'vod' | 'live' 97 type: 'vod' | 'live'
97 includeInput: boolean 98 includeInput: boolean
98 strictLower: boolean 99 strictLower: boolean
100 hasAudio: boolean
99}) { 101}) {
100 const { input, type, includeInput, strictLower } = options 102 const { input, type, includeInput, strictLower, hasAudio } = options
101 103
102 const configResolutions = type === 'vod' 104 const configResolutions = type === 'vod'
103 ? CONFIG.TRANSCODING.RESOLUTIONS 105 ? CONFIG.TRANSCODING.RESOLUTIONS
@@ -125,12 +127,15 @@ function computeResolutionsToTranscode (options: {
125 if (input < resolution) continue 127 if (input < resolution) continue
126 // We only want lower resolutions than input file 128 // We only want lower resolutions than input file
127 if (strictLower && input === resolution) continue 129 if (strictLower && input === resolution) continue
130 // Audio resolutio but no audio in the video
131 if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue
128 132
129 resolutionsEnabled.add(resolution) 133 resolutionsEnabled.add(resolution)
130 } 134 }
131 135
132 if (includeInput) { 136 if (includeInput) {
133 resolutionsEnabled.add(input) 137 // Always use an even resolution to avoid issues with ffmpeg
138 resolutionsEnabled.add(toEven(input))
134 } 139 }
135 140
136 return Array.from(resolutionsEnabled) 141 return Array.from(resolutionsEnabled)
diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts
new file mode 100644
index 000000000..a32cc9621
--- /dev/null
+++ b/server/helpers/otp.ts
@@ -0,0 +1,58 @@
1import { Secret, TOTP } from 'otpauth'
2import { CONFIG } from '@server/initializers/config'
3import { WEBSERVER } from '@server/initializers/constants'
4import { decrypt } from './peertube-crypto'
5
6async function isOTPValid (options: {
7 encryptedSecret: string
8 token: string
9}) {
10 const { token, encryptedSecret } = options
11
12 const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE)
13
14 const totp = new TOTP({
15 ...baseOTPOptions(),
16
17 secret
18 })
19
20 const delta = totp.validate({
21 token,
22 window: 1
23 })
24
25 if (delta === null) return false
26
27 return true
28}
29
30function generateOTPSecret (email: string) {
31 const totp = new TOTP({
32 ...baseOTPOptions(),
33
34 label: email,
35 secret: new Secret()
36 })
37
38 return {
39 secret: totp.secret.base32,
40 uri: totp.toString()
41 }
42}
43
44export {
45 isOTPValid,
46 generateOTPSecret
47}
48
49// ---------------------------------------------------------------------------
50
51function baseOTPOptions () {
52 return {
53 issuer: WEBSERVER.HOST,
54 algorithm: 'SHA1',
55 digits: 6,
56 period: 30
57 }
58}
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index 8aca50900..ae7d11800 100644
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -1,11 +1,11 @@
1import { compare, genSalt, hash } from 'bcrypt' 1import { compare, genSalt, hash } from 'bcrypt'
2import { createSign, createVerify } from 'crypto' 2import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
3import { Request } from 'express' 3import { Request } from 'express'
4import { cloneDeep } from 'lodash' 4import { cloneDeep } from 'lodash'
5import { sha256 } from '@shared/extra-utils' 5import { sha256 } from '@shared/extra-utils'
6import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' 6import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
7import { MActor } from '../types/models' 7import { MActor } from '../types/models'
8import { generateRSAKeyPairPromise, promisify1, promisify2 } from './core-utils' 8import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils'
9import { jsonld } from './custom-jsonld-signature' 9import { jsonld } from './custom-jsonld-signature'
10import { logger } from './logger' 10import { logger } from './logger'
11 11
@@ -21,9 +21,13 @@ function createPrivateAndPublicKeys () {
21 return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) 21 return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE)
22} 22}
23 23
24// ---------------------------------------------------------------------------
24// User password checks 25// User password checks
26// ---------------------------------------------------------------------------
25 27
26function comparePassword (plainPassword: string, hashPassword: string) { 28function comparePassword (plainPassword: string, hashPassword: string) {
29 if (!plainPassword) return Promise.resolve(false)
30
27 return bcryptComparePromise(plainPassword, hashPassword) 31 return bcryptComparePromise(plainPassword, hashPassword)
28} 32}
29 33
@@ -33,7 +37,9 @@ async function cryptPassword (password: string) {
33 return bcryptHashPromise(password, salt) 37 return bcryptHashPromise(password, salt)
34} 38}
35 39
40// ---------------------------------------------------------------------------
36// HTTP Signature 41// HTTP Signature
42// ---------------------------------------------------------------------------
37 43
38function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { 44function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean {
39 if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { 45 if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) {
@@ -62,7 +68,9 @@ function parseHTTPSignature (req: Request, clockSkew?: number) {
62 return parsed 68 return parsed
63} 69}
64 70
71// ---------------------------------------------------------------------------
65// JSONLD 72// JSONLD
73// ---------------------------------------------------------------------------
66 74
67function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> { 75function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> {
68 if (signedDocument.signature.type === 'RsaSignature2017') { 76 if (signedDocument.signature.type === 'RsaSignature2017') {
@@ -112,6 +120,8 @@ async function signJsonLDObject <T> (byActor: MActor, data: T) {
112 return Object.assign(data, { signature }) 120 return Object.assign(data, { signature })
113} 121}
114 122
123// ---------------------------------------------------------------------------
124
115function buildDigest (body: any) { 125function buildDigest (body: any) {
116 const rawBody = typeof body === 'string' ? body : JSON.stringify(body) 126 const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
117 127
@@ -119,6 +129,34 @@ function buildDigest (body: any) {
119} 129}
120 130
121// --------------------------------------------------------------------------- 131// ---------------------------------------------------------------------------
132// Encryption
133// ---------------------------------------------------------------------------
134
135async function encrypt (str: string, secret: string) {
136 const iv = await randomBytesPromise(ENCRYPTION.IV)
137
138 const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
139 const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv)
140
141 let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':'
142 encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING)
143 encrypted += cipher.final(ENCRYPTION.ENCODING)
144
145 return encrypted
146}
147
148async function decrypt (encryptedArg: string, secret: string) {
149 const [ ivStr, encryptedStr ] = encryptedArg.split(':')
150
151 const iv = Buffer.from(ivStr, 'hex')
152 const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
153
154 const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv)
155
156 return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8')
157}
158
159// ---------------------------------------------------------------------------
122 160
123export { 161export {
124 isHTTPSignatureDigestValid, 162 isHTTPSignatureDigestValid,
@@ -129,7 +167,10 @@ export {
129 comparePassword, 167 comparePassword,
130 createPrivateAndPublicKeys, 168 createPrivateAndPublicKeys,
131 cryptPassword, 169 cryptPassword,
132 signJsonLDObject 170 signJsonLDObject,
171
172 encrypt,
173 decrypt
133} 174}
134 175
135// --------------------------------------------------------------------------- 176// ---------------------------------------------------------------------------
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
index 3cb17edd0..f5f476913 100644
--- a/server/helpers/upload.ts
+++ b/server/helpers/upload.ts
@@ -1,10 +1,10 @@
1import { join } from 'path' 1import { join } from 'path'
2import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' 2import { DIRECTORIES } from '@server/initializers/constants'
3 3
4function getResumableUploadPath (filename?: string) { 4function getResumableUploadPath (filename?: string) {
5 if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename) 5 if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
6 6
7 return RESUMABLE_UPLOAD_DIRECTORY 7 return DIRECTORIES.RESUMABLE_UPLOAD
8} 8}
9 9
10// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index f5f645d3e..c688ef1e3 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -2,6 +2,7 @@ import { Response } from 'express'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' 3import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
4import { VideoPrivacy, VideoState } from '@shared/models' 4import { VideoPrivacy, VideoState } from '@shared/models'
5import { forceNumber } from '@shared/core-utils'
5 6
6function getVideoWithAttributes (res: Response) { 7function getVideoWithAttributes (res: Response) {
7 return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo 8 return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo
@@ -14,14 +15,14 @@ function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
14} 15}
15 16
16function isPrivacyForFederation (privacy: VideoPrivacy) { 17function isPrivacyForFederation (privacy: VideoPrivacy) {
17 const castedPrivacy = parseInt(privacy + '', 10) 18 const castedPrivacy = forceNumber(privacy)
18 19
19 return castedPrivacy === VideoPrivacy.PUBLIC || 20 return castedPrivacy === VideoPrivacy.PUBLIC ||
20 (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true && castedPrivacy === VideoPrivacy.UNLISTED) 21 (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true && castedPrivacy === VideoPrivacy.UNLISTED)
21} 22}
22 23
23function isStateForFederation (state: VideoState) { 24function isStateForFederation (state: VideoState) {
24 const castedState = parseInt(state + '', 10) 25 const castedState = forceNumber(state)
25 26
26 return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED 27 return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED
27} 28}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 88bdb16b6..a3c93e6fe 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -1,6 +1,6 @@
1import { decode, encode } from 'bencode' 1import { decode, encode } from 'bencode'
2import createTorrent from 'create-torrent' 2import createTorrent from 'create-torrent'
3import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra' 3import { createWriteStream, ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra'
4import magnetUtil from 'magnet-uri' 4import magnetUtil from 'magnet-uri'
5import parseTorrent from 'parse-torrent' 5import parseTorrent from 'parse-torrent'
6import { dirname, join } from 'path' 6import { dirname, join } from 'path'
@@ -134,6 +134,11 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli
134 134
135 const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) 135 const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
136 136
137 if (!await pathExists(oldTorrentPath)) {
138 logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath)
139 return
140 }
141
137 const torrentContent = await readFile(oldTorrentPath) 142 const torrentContent = await readFile(oldTorrentPath)
138 const decoded = decode(torrentContent) 143 const decoded = decode(torrentContent)
139 144
@@ -151,7 +156,7 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli
151 logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath) 156 logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath)
152 157
153 await writeFile(newTorrentPath, encode(decoded)) 158 await writeFile(newTorrentPath, encode(decoded))
154 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) 159 await remove(oldTorrentPath)
155 160
156 videoFile.torrentFilename = newTorrentFilename 161 videoFile.torrentFilename = newTorrentFilename
157 videoFile.infoHash = sha1(encode(decoded.info)) 162 videoFile.infoHash = sha1(encode(decoded.info))
@@ -164,7 +169,10 @@ function generateMagnetUri (
164) { 169) {
165 const xs = videoFile.getTorrentUrl() 170 const xs = videoFile.getTorrentUrl()
166 const announce = trackerUrls 171 const announce = trackerUrls
167 let urlList = [ videoFile.getFileUrl(video) ] 172
173 let urlList = video.hasPrivateStaticPath()
174 ? []
175 : [ videoFile.getFileUrl(video) ]
168 176
169 const redundancies = videoFile.RedundancyVideos 177 const redundancies = videoFile.RedundancyVideos
170 if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) 178 if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
@@ -240,6 +248,8 @@ function buildAnnounceList () {
240} 248}
241 249
242function buildUrlList (video: MVideo, videoFile: MVideoFile) { 250function buildUrlList (video: MVideo, videoFile: MVideoFile) {
251 if (video.hasPrivateStaticPath()) return []
252
243 return [ videoFile.getFileUrl(video) ] 253 return [ videoFile.getFileUrl(video) ]
244} 254}
245 255
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index fc4c40787..a2f630953 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -128,14 +128,14 @@ export class YoutubeDLCLI {
128 const data = await this.run({ url, args: completeArgs, processOptions }) 128 const data = await this.run({ url, args: completeArgs, processOptions })
129 if (!data) return undefined 129 if (!data) return undefined
130 130
131 const info = data.map(this.parseInfo) 131 const info = data.map(d => JSON.parse(d))
132 132
133 return info.length === 1 133 return info.length === 1
134 ? info[0] 134 ? info[0]
135 : info 135 : info
136 } 136 }
137 137
138 getListInfo (options: { 138 async getListInfo (options: {
139 url: string 139 url: string
140 latestVideosCount?: number 140 latestVideosCount?: number
141 processOptions: execa.NodeOptions 141 processOptions: execa.NodeOptions
@@ -151,12 +151,17 @@ export class YoutubeDLCLI {
151 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) 151 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
152 } 152 }
153 153
154 return this.getInfo({ 154 const result = await this.getInfo({
155 url: options.url, 155 url: options.url,
156 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), 156 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
157 processOptions: options.processOptions, 157 processOptions: options.processOptions,
158 additionalYoutubeDLArgs 158 additionalYoutubeDLArgs
159 }) 159 })
160
161 if (!result) return result
162 if (!Array.isArray(result)) return [ result ]
163
164 return result
160 } 165 }
161 166
162 async getSubs (options: { 167 async getSubs (options: {
@@ -241,8 +246,4 @@ export class YoutubeDLCLI {
241 246
242 return args 247 return args
243 } 248 }
244
245 private parseInfo (data: string) {
246 return JSON.parse(data)
247 }
248} 249}
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 966b8df78..ac3cd190e 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -77,7 +77,7 @@ class YoutubeDLWrapper {
77 77
78 const subtitles = files.reduce((acc, filename) => { 78 const subtitles = files.reduce((acc, filename) => {
79 const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i) 79 const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
80 if (!matched || !matched[1]) return acc 80 if (!matched?.[1]) return acc
81 81
82 return [ 82 return [
83 ...acc, 83 ...acc,
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index 42839d1c9..09e878eee 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -42,6 +42,7 @@ function checkConfig () {
42 logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.') 42 logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.')
43 } 43 }
44 44
45 checkSecretsConfig()
45 checkEmailConfig() 46 checkEmailConfig()
46 checkNSFWPolicyConfig() 47 checkNSFWPolicyConfig()
47 checkLocalRedundancyConfig() 48 checkLocalRedundancyConfig()
@@ -103,6 +104,12 @@ export {
103 104
104// --------------------------------------------------------------------------- 105// ---------------------------------------------------------------------------
105 106
107function checkSecretsConfig () {
108 if (!CONFIG.SECRETS.PEERTUBE) {
109 throw new Error('secrets.peertube is missing in config. Generate one using `openssl rand -hex 32`')
110 }
111}
112
106function checkEmailConfig () { 113function checkEmailConfig () {
107 if (!isEmailEnabled()) { 114 if (!isEmailEnabled()) {
108 if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { 115 if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) {
@@ -271,6 +278,14 @@ function checkObjectStorageConfig () {
271 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' 278 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
272 ) 279 )
273 } 280 }
281
282 if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC) {
283 throw new Error('object_storage.upload_acl.public must be set')
284 }
285
286 if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE) {
287 throw new Error('object_storage.upload_acl.private must be set')
288 }
274 } 289 }
275} 290}
276 291
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 3188903be..39713a266 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -11,13 +11,17 @@ const config: IConfig = require('config')
11function checkMissedConfig () { 11function checkMissedConfig () {
12 const required = [ 'listen.port', 'listen.hostname', 12 const required = [ 'listen.port', 'listen.hostname',
13 'webserver.https', 'webserver.hostname', 'webserver.port', 13 'webserver.https', 'webserver.hostname', 'webserver.port',
14 'secrets.peertube',
14 'trust_proxy', 15 'trust_proxy',
15 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max', 16 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max',
16 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 17 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
17 'email.body.signature', 'email.subject.prefix', 18 'email.body.signature', 'email.subject.prefix',
18 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 19 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
19 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 20 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known',
20 'log.level', 21 'log.level', 'log.rotation.enabled', 'log.rotation.max_file_size', 'log.rotation.max_files', 'log.anonymize_ip',
22 'log.log_ping_requests', 'log.log_tracker_unknown_infohash', 'log.prettify_sql', 'log.accept_client_log',
23 'open_telemetry.metrics.enabled', 'open_telemetry.metrics.prometheus_exporter.hostname',
24 'open_telemetry.metrics.prometheus_exporter.port', 'open_telemetry.tracing.enabled', 'open_telemetry.tracing.jaeger_exporter.endpoint',
21 'user.video_quota', 'user.video_quota_daily', 25 'user.video_quota', 'user.video_quota_daily',
22 'video_channels.max_per_user', 26 'video_channels.max_per_user',
23 'csp.enabled', 'csp.report_only', 'csp.report_uri', 27 'csp.enabled', 'csp.report_only', 'csp.report_uri',
@@ -34,6 +38,7 @@ function checkMissedConfig () {
34 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 38 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
35 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', 39 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
36 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', 40 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
41 'import.video_channel_synchronization.full_sync_videos_limit',
37 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', 42 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
38 'client.videos.miniature.display_author_avatar', 43 'client.videos.miniature.display_author_avatar',
39 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', 44 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
@@ -45,6 +50,12 @@ function checkMissedConfig () {
45 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', 50 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
46 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', 51 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration',
47 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', 52 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
53 'static_files.private_files_require_auth',
54 'object_storage.enabled', 'object_storage.endpoint', 'object_storage.region', 'object_storage.upload_acl.public',
55 'object_storage.upload_acl.private', 'object_storage.proxy.proxify_private_files', 'object_storage.credentials.access_key_id',
56 'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name',
57 'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.videos.bucket_name',
58 'object_storage.videos.prefix', 'object_storage.videos.base_url',
48 'theme.default', 59 'theme.default',
49 'feeds.videos.count', 'feeds.comments.count', 60 'feeds.videos.count', 'feeds.comments.count',
50 'geo_ip.enabled', 'geo_ip.country.database_url', 61 'geo_ip.enabled', 'geo_ip.country.database_url',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 2c92bea22..c2f8b19fd 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -20,6 +20,9 @@ const CONFIG = {
20 PORT: config.get<number>('listen.port'), 20 PORT: config.get<number>('listen.port'),
21 HOSTNAME: config.get<string>('listen.hostname') 21 HOSTNAME: config.get<string>('listen.hostname')
22 }, 22 },
23 SECRETS: {
24 PEERTUBE: config.get<string>('secrets.peertube')
25 },
23 DATABASE: { 26 DATABASE: {
24 DBNAME: config.has('database.name') ? config.get<string>('database.name') : 'peertube' + config.get<string>('database.suffix'), 27 DBNAME: config.has('database.name') ? config.get<string>('database.name') : 'peertube' + config.get<string>('database.suffix'),
25 HOSTNAME: config.get<string>('database.hostname'), 28 HOSTNAME: config.get<string>('database.hostname'),
@@ -107,18 +110,28 @@ const CONFIG = {
107 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')), 110 TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
108 CACHE_DIR: buildPath(config.get<string>('storage.cache')), 111 CACHE_DIR: buildPath(config.get<string>('storage.cache')),
109 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), 112 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
110 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) 113 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
114 WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
115 },
116 STATIC_FILES: {
117 PRIVATE_FILES_REQUIRE_AUTH: config.get<boolean>('static_files.private_files_require_auth')
111 }, 118 },
112 OBJECT_STORAGE: { 119 OBJECT_STORAGE: {
113 ENABLED: config.get<boolean>('object_storage.enabled'), 120 ENABLED: config.get<boolean>('object_storage.enabled'),
114 MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), 121 MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
115 ENDPOINT: config.get<string>('object_storage.endpoint'), 122 ENDPOINT: config.get<string>('object_storage.endpoint'),
116 REGION: config.get<string>('object_storage.region'), 123 REGION: config.get<string>('object_storage.region'),
117 UPLOAD_ACL: config.get<string>('object_storage.upload_acl'), 124 UPLOAD_ACL: {
125 PUBLIC: config.get<string>('object_storage.upload_acl.public'),
126 PRIVATE: config.get<string>('object_storage.upload_acl.private')
127 },
118 CREDENTIALS: { 128 CREDENTIALS: {
119 ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), 129 ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
120 SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') 130 SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
121 }, 131 },
132 PROXY: {
133 PROXIFY_PRIVATE_FILES: config.get<boolean>('object_storage.proxy.proxify_private_files')
134 },
122 VIDEOS: { 135 VIDEOS: {
123 BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), 136 BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
124 PREFIX: config.get<string>('object_storage.videos.prefix'), 137 PREFIX: config.get<string>('object_storage.videos.prefix'),
@@ -177,6 +190,7 @@ const CONFIG = {
177 ENABLED: config.get<boolean>('open_telemetry.metrics.enabled'), 190 ENABLED: config.get<boolean>('open_telemetry.metrics.enabled'),
178 191
179 PROMETHEUS_EXPORTER: { 192 PROMETHEUS_EXPORTER: {
193 HOSTNAME: config.get<string>('open_telemetry.metrics.prometheus_exporter.hostname'),
180 PORT: config.get<number>('open_telemetry.metrics.prometheus_exporter.port') 194 PORT: config.get<number>('open_telemetry.metrics.prometheus_exporter.port')
181 } 195 }
182 }, 196 },
@@ -405,6 +419,9 @@ const CONFIG = {
405 get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) }, 419 get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
406 get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { 420 get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
407 return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization') 421 return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
422 },
423 get FULL_SYNC_VIDEOS_LIMIT () {
424 return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit')
408 } 425 }
409 } 426 }
410 }, 427 },
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index eb54781b6..3908bbf05 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,5 +1,5 @@
1import { RepeatOptions } from 'bullmq' 1import { RepeatOptions } from 'bullmq'
2import { randomBytes } from 'crypto' 2import { Encoding, randomBytes } from 'crypto'
3import { invert } from 'lodash' 3import { invert } from 'lodash'
4import { join } from 'path' 4import { join } from 'path'
5import { randomInt, root } from '@shared/core-utils' 5import { randomInt, root } from '@shared/core-utils'
@@ -25,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
25 25
26// --------------------------------------------------------------------------- 26// ---------------------------------------------------------------------------
27 27
28const LAST_MIGRATION_VERSION = 740 28const LAST_MIGRATION_VERSION = 745
29 29
30// --------------------------------------------------------------------------- 30// ---------------------------------------------------------------------------
31 31
@@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = {
116 ACTIVITY_PUB: { 116 ACTIVITY_PUB: {
117 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example 117 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
118 }, 118 },
119 STATS: '4 hours' 119 STATS: '4 hours',
120 WELL_KNOWN: '1 day'
120} 121}
121 122
122// --------------------------------------------------------------------------- 123// ---------------------------------------------------------------------------
@@ -636,9 +637,18 @@ let PRIVATE_RSA_KEY_SIZE = 2048
636// Password encryption 637// Password encryption
637const BCRYPT_SALT_SIZE = 10 638const BCRYPT_SALT_SIZE = 10
638 639
640const ENCRYPTION = {
641 ALGORITHM: 'aes-256-cbc',
642 IV: 16,
643 SALT: 'peertube',
644 ENCODING: 'hex' as Encoding
645}
646
639const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes 647const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes
640const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days 648const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days
641 649
650const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes
651
642const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes 652const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes
643 653
644const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { 654const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
@@ -652,10 +662,15 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
652// Express static paths (router) 662// Express static paths (router)
653const STATIC_PATHS = { 663const STATIC_PATHS = {
654 THUMBNAILS: '/static/thumbnails/', 664 THUMBNAILS: '/static/thumbnails/',
665
655 WEBSEED: '/static/webseed/', 666 WEBSEED: '/static/webseed/',
667 PRIVATE_WEBSEED: '/static/webseed/private/',
668
656 REDUNDANCY: '/static/redundancy/', 669 REDUNDANCY: '/static/redundancy/',
670
657 STREAMING_PLAYLISTS: { 671 STREAMING_PLAYLISTS: {
658 HLS: '/static/streaming-playlists/hls' 672 HLS: '/static/streaming-playlists/hls',
673 PRIVATE_HLS: '/static/streaming-playlists/hls/private/'
659 } 674 }
660} 675}
661const STATIC_DOWNLOAD_PATHS = { 676const STATIC_DOWNLOAD_PATHS = {
@@ -670,6 +685,13 @@ const LAZY_STATIC_PATHS = {
670 VIDEO_CAPTIONS: '/lazy-static/video-captions/', 685 VIDEO_CAPTIONS: '/lazy-static/video-captions/',
671 TORRENTS: '/lazy-static/torrents/' 686 TORRENTS: '/lazy-static/torrents/'
672} 687}
688const OBJECT_STORAGE_PROXY_PATHS = {
689 PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
690
691 STREAMING_PLAYLISTS: {
692 PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/'
693 }
694}
673 695
674// Cache control 696// Cache control
675const STATIC_MAX_AGE = { 697const STATIC_MAX_AGE = {
@@ -689,7 +711,7 @@ const PREVIEWS_SIZE = {
689 height: 480, 711 height: 480,
690 minWidth: 400 712 minWidth: 400
691} 713}
692const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: number }[]} = { 714const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: number }[] } = {
693 [ActorImageType.AVATAR]: [ 715 [ActorImageType.AVATAR]: [
694 { 716 {
695 width: 120, 717 width: 120,
@@ -735,12 +757,32 @@ const LRU_CACHE = {
735 }, 757 },
736 ACTOR_IMAGE_STATIC: { 758 ACTOR_IMAGE_STATIC: {
737 MAX_SIZE: 500 759 MAX_SIZE: 500
760 },
761 STATIC_VIDEO_FILES_RIGHTS_CHECK: {
762 MAX_SIZE: 5000,
763 TTL: parseDurationToMs('10 seconds')
764 },
765 VIDEO_TOKENS: {
766 MAX_SIZE: 100_000,
767 TTL: parseDurationToMs('8 hours')
738 } 768 }
739} 769}
740 770
741const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads') 771const DIRECTORIES = {
742const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') 772 RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'),
743const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') 773
774 HLS_STREAMING_PLAYLIST: {
775 PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'),
776 PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private')
777 },
778
779 VIDEOS: {
780 PUBLIC: CONFIG.STORAGE.VIDEOS_DIR,
781 PRIVATE: join(CONFIG.STORAGE.VIDEOS_DIR, 'private')
782 },
783
784 HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
785}
744 786
745const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS 787const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
746 788
@@ -796,6 +838,10 @@ const REDUNDANCY = {
796} 838}
797 839
798const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) 840const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
841const OTP = {
842 HEADER_NAME: 'x-peertube-otp',
843 HEADER_REQUIRED_VALUE: 'required; app'
844}
799 845
800const ASSETS_PATH = { 846const ASSETS_PATH = {
801 DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'), 847 DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'),
@@ -944,13 +990,14 @@ const VIDEO_FILTERS = {
944export { 990export {
945 WEBSERVER, 991 WEBSERVER,
946 API_VERSION, 992 API_VERSION,
993 ENCRYPTION,
947 VIDEO_LIVE, 994 VIDEO_LIVE,
948 PEERTUBE_VERSION, 995 PEERTUBE_VERSION,
949 LAZY_STATIC_PATHS, 996 LAZY_STATIC_PATHS,
997 OBJECT_STORAGE_PROXY_PATHS,
950 SEARCH_INDEX, 998 SEARCH_INDEX,
951 RESUMABLE_UPLOAD_DIRECTORY, 999 DIRECTORIES,
952 RESUMABLE_UPLOAD_SESSION_LIFETIME, 1000 RESUMABLE_UPLOAD_SESSION_LIFETIME,
953 HLS_REDUNDANCY_DIRECTORY,
954 P2P_MEDIA_LOADER_PEER_VERSION, 1001 P2P_MEDIA_LOADER_PEER_VERSION,
955 ACTOR_IMAGES_SIZE, 1002 ACTOR_IMAGES_SIZE,
956 ACCEPT_HEADERS, 1003 ACCEPT_HEADERS,
@@ -977,13 +1024,13 @@ export {
977 FOLLOW_STATES, 1024 FOLLOW_STATES,
978 DEFAULT_USER_THEME_NAME, 1025 DEFAULT_USER_THEME_NAME,
979 SERVER_ACTOR_NAME, 1026 SERVER_ACTOR_NAME,
1027 TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
980 PLUGIN_GLOBAL_CSS_FILE_NAME, 1028 PLUGIN_GLOBAL_CSS_FILE_NAME,
981 PLUGIN_GLOBAL_CSS_PATH, 1029 PLUGIN_GLOBAL_CSS_PATH,
982 PRIVATE_RSA_KEY_SIZE, 1030 PRIVATE_RSA_KEY_SIZE,
983 VIDEO_FILTERS, 1031 VIDEO_FILTERS,
984 ROUTE_CACHE_LIFETIME, 1032 ROUTE_CACHE_LIFETIME,
985 SORTABLE_COLUMNS, 1033 SORTABLE_COLUMNS,
986 HLS_STREAMING_PLAYLIST_DIRECTORY,
987 JOB_TTL, 1034 JOB_TTL,
988 DEFAULT_THEME_NAME, 1035 DEFAULT_THEME_NAME,
989 NSFW_POLICY_TYPES, 1036 NSFW_POLICY_TYPES,
@@ -1032,6 +1079,7 @@ export {
1032 PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME, 1079 PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME,
1033 ASSETS_PATH, 1080 ASSETS_PATH,
1034 FILES_CONTENT_HASH, 1081 FILES_CONTENT_HASH,
1082 OTP,
1035 loadLanguages, 1083 loadLanguages,
1036 buildLanguages, 1084 buildLanguages,
1037 generateContentHash 1085 generateContentHash
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index b02be9567..f5d8eedf1 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -10,7 +10,7 @@ import { ApplicationModel } from '../models/application/application'
10import { OAuthClientModel } from '../models/oauth/oauth-client' 10import { OAuthClientModel } from '../models/oauth/oauth-client'
11import { applicationExist, clientsExist, usersExist } from './checker-after-init' 11import { applicationExist, clientsExist, usersExist } from './checker-after-init'
12import { CONFIG } from './config' 12import { CONFIG } from './config'
13import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' 13import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants'
14import { sequelizeTypescript } from './database' 14import { sequelizeTypescript } from './database'
15 15
16async function installApplication () { 16async function installApplication () {
@@ -92,11 +92,13 @@ function createDirectoriesIfNotExist () {
92 tasks.push(ensureDir(dir)) 92 tasks.push(ensureDir(dir))
93 } 93 }
94 94
95 // Playlist directories 95 tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE))
96 tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY)) 96 tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC))
97 tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC))
98 tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE))
97 99
98 // Resumable upload directory 100 // Resumable upload directory
99 tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY)) 101 tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD))
100 102
101 return Promise.all(tasks) 103 return Promise.all(tasks)
102} 104}
diff --git a/server/initializers/migrations/0745-user-otp.ts b/server/initializers/migrations/0745-user-otp.ts
new file mode 100644
index 000000000..157308ea1
--- /dev/null
+++ b/server/initializers/migrations/0745-user-otp.ts
@@ -0,0 +1,29 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction
5 queryInterface: Sequelize.QueryInterface
6 sequelize: Sequelize.Sequelize
7 db: any
8}): Promise<void> {
9 const { transaction } = utils
10
11 const data = {
12 type: Sequelize.STRING,
13 defaultValue: null,
14 allowNull: true
15 }
16 await utils.queryInterface.addColumn('user', 'otpSecret', data, { transaction })
17
18}
19
20async function down (utils: {
21 queryInterface: Sequelize.QueryInterface
22 transaction: Sequelize.Transaction
23}) {
24}
25
26export {
27 up,
28 down
29}
diff --git a/server/lib/activitypub/collection.ts b/server/lib/activitypub/collection.ts
index f897141ea..a176cab51 100644
--- a/server/lib/activitypub/collection.ts
+++ b/server/lib/activitypub/collection.ts
@@ -3,6 +3,7 @@ import validator from 'validator'
3import { pageToStartAndCount } from '@server/helpers/core-utils' 3import { pageToStartAndCount } from '@server/helpers/core-utils'
4import { ACTIVITY_PUB } from '@server/initializers/constants' 4import { ACTIVITY_PUB } from '@server/initializers/constants'
5import { ResultList } from '@shared/models' 5import { ResultList } from '@shared/models'
6import { forceNumber } from '@shared/core-utils'
6 7
7type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>> 8type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
8 9
@@ -33,7 +34,7 @@ async function activityPubCollectionPagination (
33 let prev: string | undefined 34 let prev: string | undefined
34 35
35 // Assert page is a number 36 // Assert page is a number
36 page = parseInt(page, 10) 37 page = forceNumber(page)
37 38
38 // There are more results 39 // There are more results
39 if (result.total > page * size) { 40 if (result.total > page * size) {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 76ed37aae..1e6e8956c 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -109,8 +109,10 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc
109 let video: MVideoAccountLightBlacklistAllFiles 109 let video: MVideoAccountLightBlacklistAllFiles
110 let created: boolean 110 let created: boolean
111 let comment: MCommentOwnerVideo 111 let comment: MCommentOwnerVideo
112
112 try { 113 try {
113 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) 114 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
115 if (!resolveThreadResult) return // Comment not accepted
114 116
115 video = resolveThreadResult.video 117 video = resolveThreadResult.video
116 created = resolveThreadResult.commentCreated 118 created = resolveThreadResult.commentCreated
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 0fefcbbc5..af0dd510a 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -57,7 +57,7 @@ export {
57 57
58async function addVideoShare (shareUrl: string, video: MVideoId) { 58async function addVideoShare (shareUrl: string, video: MVideoId) {
59 const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true }) 59 const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
60 if (!body || !body.actor) throw new Error('Body or body actor is invalid') 60 if (!body?.actor) throw new Error('Body or body actor is invalid')
61 61
62 const actorUrl = getAPId(body.actor) 62 const actorUrl = getAPId(body.actor)
63 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { 63 if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 911c7cd30..b65baf0e9 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -4,7 +4,9 @@ import { logger } from '../../helpers/logger'
4import { doJSONRequest } from '../../helpers/requests' 4import { doJSONRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 7import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
8import { isRemoteVideoCommentAccepted } from '../moderation'
9import { Hooks } from '../plugins/hooks'
8import { getOrCreateAPActor } from './actors' 10import { getOrCreateAPActor } from './actors'
9import { checkUrlsSameHost } from './url' 11import { checkUrlsSameHost } from './url'
10import { getOrCreateAPVideo } from './videos' 12import { getOrCreateAPVideo } from './videos'
@@ -103,6 +105,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
103 firstReply.changed('updatedAt', true) 105 firstReply.changed('updatedAt', true)
104 firstReply.Video = video 106 firstReply.Video = video
105 107
108 if (await isRemoteCommentAccepted(firstReply) !== true) {
109 return undefined
110 }
111
106 comments[comments.length - 1] = await firstReply.save() 112 comments[comments.length - 1] = await firstReply.save()
107 113
108 for (let i = comments.length - 2; i >= 0; i--) { 114 for (let i = comments.length - 2; i >= 0; i--) {
@@ -113,6 +119,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
113 comment.changed('updatedAt', true) 119 comment.changed('updatedAt', true)
114 comment.Video = video 120 comment.Video = video
115 121
122 if (await isRemoteCommentAccepted(comment) !== true) {
123 return undefined
124 }
125
116 comments[i] = await comment.save() 126 comments[i] = await comment.save()
117 } 127 }
118 128
@@ -169,3 +179,26 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
169 commentCreated: true 179 commentCreated: true
170 }) 180 })
171} 181}
182
183async function isRemoteCommentAccepted (comment: MComment) {
184 // Already created
185 if (comment.id) return true
186
187 const acceptParameters = {
188 comment
189 }
190
191 const acceptedResult = await Hooks.wrapFun(
192 isRemoteVideoCommentAccepted,
193 acceptParameters,
194 'filter:activity-pub.remote-video-comment.create.accept.result'
195 )
196
197 if (!acceptedResult || acceptedResult.accepted !== true) {
198 logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters })
199
200 return false
201 }
202
203 return true
204}
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts
index fa1887315..bc0d4301f 100644
--- a/server/lib/auth/oauth.ts
+++ b/server/lib/auth/oauth.ts
@@ -9,11 +9,23 @@ import OAuth2Server, {
9 UnsupportedGrantTypeError 9 UnsupportedGrantTypeError
10} from '@node-oauth/oauth2-server' 10} from '@node-oauth/oauth2-server'
11import { randomBytesPromise } from '@server/helpers/core-utils' 11import { randomBytesPromise } from '@server/helpers/core-utils'
12import { isOTPValid } from '@server/helpers/otp'
12import { MOAuthClient } from '@server/types/models' 13import { MOAuthClient } from '@server/types/models'
13import { sha1 } from '@shared/extra-utils' 14import { sha1 } from '@shared/extra-utils'
14import { OAUTH_LIFETIME } from '../../initializers/constants' 15import { HttpStatusCode } from '@shared/models'
16import { OAUTH_LIFETIME, OTP } from '../../initializers/constants'
15import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' 17import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model'
16 18
19class MissingTwoFactorError extends Error {
20 code = HttpStatusCode.UNAUTHORIZED_401
21 name = 'missing_two_factor'
22}
23
24class InvalidTwoFactorError extends Error {
25 code = HttpStatusCode.BAD_REQUEST_400
26 name = 'invalid_two_factor'
27}
28
17/** 29/**
18 * 30 *
19 * Reimplement some functions of OAuth2Server to inject external auth methods 31 * Reimplement some functions of OAuth2Server to inject external auth methods
@@ -83,17 +95,15 @@ async function handleOAuthToken (req: express.Request, options: { refreshTokenAu
83 95
84function handleOAuthAuthenticate ( 96function handleOAuthAuthenticate (
85 req: express.Request, 97 req: express.Request,
86 res: express.Response, 98 res: express.Response
87 authenticateInQuery = false
88) { 99) {
89 const options = authenticateInQuery 100 return oAuthServer.authenticate(new Request(req), new Response(res))
90 ? { allowBearerTokensInQueryString: true }
91 : {}
92
93 return oAuthServer.authenticate(new Request(req), new Response(res), options)
94} 101}
95 102
96export { 103export {
104 MissingTwoFactorError,
105 InvalidTwoFactorError,
106
97 handleOAuthToken, 107 handleOAuthToken,
98 handleOAuthAuthenticate 108 handleOAuthAuthenticate
99} 109}
@@ -118,6 +128,16 @@ async function handlePasswordGrant (options: {
118 const user = await getUser(request.body.username, request.body.password, bypassLogin) 128 const user = await getUser(request.body.username, request.body.password, bypassLogin)
119 if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid') 129 if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid')
120 130
131 if (user.otpSecret) {
132 if (!request.headers[OTP.HEADER_NAME]) {
133 throw new MissingTwoFactorError('Missing two factor header')
134 }
135
136 if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) {
137 throw new InvalidTwoFactorError('Invalid two factor header')
138 }
139 }
140
121 const token = await buildToken() 141 const token = await buildToken()
122 142
123 return saveToken(token, client, user, { bypassLogin }) 143 return saveToken(token, client, user, { bypassLogin })
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index a0a5afc0f..a41f1ae48 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -15,7 +15,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
15import { sequelizeTypescript } from '../initializers/database' 15import { sequelizeTypescript } from '../initializers/database'
16import { VideoFileModel } from '../models/video/video-file' 16import { VideoFileModel } from '../models/video/video-file'
17import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 17import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
18import { storeHLSFile } from './object-storage' 18import { storeHLSFileFromFilename } from './object-storage'
19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' 19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
20import { VideoPathManager } from './video-path-manager' 20import { VideoPathManager } from './video-path-manager'
21 21
@@ -95,7 +95,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist
95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
96 96
97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) { 97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
98 playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) 98 playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
99 await remove(masterPlaylistPath) 99 await remove(masterPlaylistPath)
100 } 100 }
101 101
@@ -146,7 +146,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist
146 await outputJSON(outputPath, json) 146 await outputJSON(outputPath, json)
147 147
148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) { 148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
149 playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) 149 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
150 await remove(outputPath) 150 await remove(outputPath)
151 } 151 }
152 152
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts
index 84c0a2de2..a25f00b0a 100644
--- a/server/lib/job-queue/handlers/activitypub-cleaner.ts
+++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts
@@ -88,7 +88,7 @@ async function updateObjectIfNeeded <T> (options: {
88 const { body } = await doJSONRequest<any>(url, { activityPub: true }) 88 const { body } = await doJSONRequest<any>(url, { activityPub: true })
89 89
90 // If not same id, check same host and update 90 // If not same id, check same host and update
91 if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) 91 if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
92 92
93 if (body.type === 'Tombstone') { 93 if (body.type === 'Tombstone') {
94 return on404OrTombstone() 94 return on404OrTombstone()
diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts
index 03aa414c9..cef93afda 100644
--- a/server/lib/job-queue/handlers/manage-video-torrent.ts
+++ b/server/lib/job-queue/handlers/manage-video-torrent.ts
@@ -1,5 +1,7 @@
1import { Job } from 'bullmq' 1import { Job } from 'bullmq'
2import { extractVideo } from '@server/helpers/video'
2import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent' 3import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent'
4import { VideoPathManager } from '@server/lib/video-path-manager'
3import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
4import { VideoFileModel } from '@server/models/video/video-file' 6import { VideoFileModel } from '@server/models/video/video-file'
5import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 7import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
@@ -30,17 +32,23 @@ async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'c
30 32
31 if (!video || !file) return 33 if (!video || !file) return
32 34
33 await createTorrentAndSetInfoHash(video, file) 35 const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
34 36
35 // Refresh videoFile because the createTorrentAndSetInfoHash could be long 37 try {
36 const refreshedFile = await VideoFileModel.loadWithVideo(file.id) 38 await createTorrentAndSetInfoHash(video, file)
37 // File does not exist anymore, remove the generated torrent
38 if (!refreshedFile) return file.removeTorrent()
39 39
40 refreshedFile.infoHash = file.infoHash 40 // Refresh videoFile because the createTorrentAndSetInfoHash could be long
41 refreshedFile.torrentFilename = file.torrentFilename 41 const refreshedFile = await VideoFileModel.loadWithVideo(file.id)
42 // File does not exist anymore, remove the generated torrent
43 if (!refreshedFile) return file.removeTorrent()
42 44
43 return refreshedFile.save() 45 refreshedFile.infoHash = file.infoHash
46 refreshedFile.torrentFilename = file.torrentFilename
47
48 await refreshedFile.save()
49 } finally {
50 fileMutexReleaser()
51 }
44} 52}
45 53
46async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { 54async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) {
@@ -52,9 +60,16 @@ async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { ac
52 60
53 if ((!video && !streamingPlaylist) || !file) return 61 if ((!video && !streamingPlaylist) || !file) return
54 62
55 await updateTorrentMetadata(video || streamingPlaylist, file) 63 const extractedVideo = extractVideo(video || streamingPlaylist)
64 const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid)
56 65
57 await file.save() 66 try {
67 await updateTorrentMetadata(video || streamingPlaylist, file)
68
69 await file.save()
70 } finally {
71 fileMutexReleaser()
72 }
58} 73}
59 74
60async function loadVideoOrLog (videoId: number) { 75async function loadVideoOrLog (videoId: number) {
@@ -82,7 +97,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
82async function loadFileOrLog (videoFileId: number) { 97async function loadFileOrLog (videoFileId: number) {
83 if (!videoFileId) return undefined 98 if (!videoFileId) return undefined
84 99
85 const file = await VideoFileModel.loadWithVideo(videoFileId) 100 const file = await VideoFileModel.load(videoFileId)
86 101
87 if (!file) { 102 if (!file) {
88 logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) 103 logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 25bdebeea..a1530cc57 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -3,10 +3,10 @@ import { remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 6import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' 7import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { VideoPathManager } from '@server/lib/video-path-manager'
10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' 10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
12import { VideoJobInfoModel } from '@server/models/video/video-job-info' 12import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@@ -28,6 +28,8 @@ export async function processMoveToObjectStorage (job: Job) {
28 28
29 const lTags = lTagsBase(video.uuid, video.url) 29 const lTags = lTagsBase(video.uuid, video.url)
30 30
31 const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
32
31 try { 33 try {
32 if (video.VideoFiles) { 34 if (video.VideoFiles) {
33 logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags) 35 logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
@@ -49,6 +51,10 @@ export async function processMoveToObjectStorage (job: Job) {
49 } 51 }
50 } catch (err) { 52 } catch (err) {
51 await onMoveToObjectStorageFailure(job, err) 53 await onMoveToObjectStorageFailure(job, err)
54
55 throw err
56 } finally {
57 fileMutexReleaser()
52 } 58 }
53 59
54 return payload.videoUUID 60 return payload.videoUUID
@@ -72,9 +78,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
72 for (const file of video.VideoFiles) { 78 for (const file of video.VideoFiles) {
73 if (file.storage !== VideoStorage.FILE_SYSTEM) continue 79 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
74 80
75 const fileUrl = await storeWebTorrentFile(file.filename) 81 const fileUrl = await storeWebTorrentFile(video, file)
76 82
77 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) 83 const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
78 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) 84 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
79 } 85 }
80} 86}
@@ -88,10 +94,10 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
88 94
89 // Resolution playlist 95 // Resolution playlist
90 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 96 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
91 await storeHLSFile(playlistWithVideo, playlistFilename) 97 await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
92 98
93 // Resolution fragmented file 99 // Resolution fragmented file
94 const fileUrl = await storeHLSFile(playlistWithVideo, file.filename) 100 const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
95 101
96 const oldPath = join(getHLSDirectory(video), file.filename) 102 const oldPath = join(getHLSDirectory(video), file.filename)
97 103
@@ -113,9 +119,9 @@ async function doAfterLastJob (options: {
113 const playlistWithVideo = playlist.withVideo(video) 119 const playlistWithVideo = playlist.withVideo(video)
114 120
115 // Master playlist 121 // Master playlist
116 playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename) 122 playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
117 // Sha256 segments file 123 // Sha256 segments file
118 playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename) 124 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
119 125
120 playlist.storage = VideoStorage.OBJECT_STORAGE 126 playlist.storage = VideoStorage.OBJECT_STORAGE
121 127
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
index 600292844..035f88e96 100644
--- a/server/lib/job-queue/handlers/video-channel-import.ts
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel' 5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
7import { MChannelSync } from '@server/types/models' 7import { MChannelSync } from '@server/types/models'
8import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' 8import { VideoChannelImportPayload } from '@shared/models'
9 9
10export async function processVideoChannelImport (job: Job) { 10export async function processVideoChannelImport (job: Job) {
11 const payload = job.data as VideoChannelImportPayload 11 const payload = job.data as VideoChannelImportPayload
@@ -32,17 +32,12 @@ export async function processVideoChannelImport (job: Job) {
32 32
33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) 33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
34 34
35 try { 35 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
36 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) 36
37 37 await synchronizeChannel({
38 await synchronizeChannel({ 38 channel: videoChannel,
39 channel: videoChannel, 39 externalChannelUrl: payload.externalChannelUrl,
40 externalChannelUrl: payload.externalChannelUrl, 40 channelSync,
41 channelSync 41 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
42 }) 42 })
43 } catch (err) {
44 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
45 channelSync.state = VideoChannelSyncState.FAILED
46 await channelSync.save()
47 }
48} 43}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 9901b878c..4d361c7b9 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -12,7 +12,8 @@ import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@serv
12import { VideoPathManager } from '@server/lib/video-path-manager' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state' 13import { buildNextVideoState } from '@server/lib/video-state'
14import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
15import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' 15import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
16import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
16import { getLowercaseExtension } from '@shared/core-utils' 17import { getLowercaseExtension } from '@shared/core-utils'
17import { isAudioFile } from '@shared/extra-utils' 18import { isAudioFile } from '@shared/extra-utils'
18import { 19import {
@@ -36,7 +37,6 @@ import { sequelizeTypescript } from '../../../initializers/database'
36import { VideoModel } from '../../../models/video/video' 37import { VideoModel } from '../../../models/video/video'
37import { VideoFileModel } from '../../../models/video/video-file' 38import { VideoFileModel } from '../../../models/video/video-file'
38import { VideoImportModel } from '../../../models/video/video-import' 39import { VideoImportModel } from '../../../models/video/video-import'
39import { MThumbnail } from '../../../types/models/video/thumbnail'
40import { federateVideoIfNeeded } from '../../activitypub/videos' 40import { federateVideoIfNeeded } from '../../activitypub/videos'
41import { Notifier } from '../../notifier' 41import { Notifier } from '../../notifier'
42import { generateVideoMiniature } from '../../thumbnail' 42import { generateVideoMiniature } from '../../thumbnail'
@@ -107,7 +107,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul
107 107
108async function getVideoImportOrDie (payload: VideoImportPayload) { 108async function getVideoImportOrDie (payload: VideoImportPayload) {
109 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) 109 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
110 if (!videoImport || !videoImport.Video) { 110 if (!videoImport?.Video) {
111 throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`) 111 throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`)
112 } 112 }
113 113
@@ -178,125 +178,159 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
178 } 178 }
179 179
180 // Video is accepted, resuming preparation 180 // Video is accepted, resuming preparation
181 const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) 181 const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid)
182 // To clean files if the import fails
183 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
184
185 // Move file
186 const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
187 await move(tempVideoPath, videoDestFile)
188 tempVideoPath = null // This path is not used anymore
189
190 // Generate miniature if the import did not created it
191 let thumbnailModel: MThumbnail
192 let thumbnailSave: object
193 if (!videoImportWithFiles.Video.getMiniature()) {
194 thumbnailModel = await generateVideoMiniature({
195 video: videoImportWithFiles.Video,
196 videoFile,
197 type: ThumbnailType.MINIATURE
198 })
199 thumbnailSave = thumbnailModel.toJSON()
200 }
201 182
202 // Generate preview if the import did not created it 183 try {
203 let previewModel: MThumbnail 184 const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile)
204 let previewSave: object
205 if (!videoImportWithFiles.Video.getPreview()) {
206 previewModel = await generateVideoMiniature({
207 video: videoImportWithFiles.Video,
208 videoFile,
209 type: ThumbnailType.PREVIEW
210 })
211 previewSave = previewModel.toJSON()
212 }
213 185
214 // Create torrent 186 // Move file
215 await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) 187 const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
188 await move(tempVideoPath, videoDestFile)
216 189
217 const videoFileSave = videoFile.toJSON() 190 tempVideoPath = null // This path is not used anymore
218 191
219 const { videoImportUpdated, video } = await retryTransactionWrapper(() => { 192 let {
220 return sequelizeTypescript.transaction(async t => { 193 miniatureModel: thumbnailModel,
221 const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo 194 miniatureJSONSave: thumbnailSave
195 } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE)
222 196
223 // Refresh video 197 let {
224 const video = await VideoModel.load(videoImportToUpdate.videoId, t) 198 miniatureModel: previewModel,
225 if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.') 199 miniatureJSONSave: previewSave
200 } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW)
226 201
227 const videoFileCreated = await videoFile.save({ transaction: t }) 202 // Create torrent
203 await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
228 204
229 // Update video DB object 205 const videoFileSave = videoFile.toJSON()
230 video.duration = duration
231 video.state = buildNextVideoState(video.state)
232 await video.save({ transaction: t })
233 206
234 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) 207 const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
235 if (previewModel) await video.addAndSaveThumbnail(previewModel, t) 208 return sequelizeTypescript.transaction(async t => {
209 // Refresh video
210 const video = await VideoModel.load(videoImportWithFiles.videoId, t)
211 if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
236 212
237 // Now we can federate the video (reload from database, we need more attributes) 213 await videoFile.save({ transaction: t })
238 const videoForFederation = await VideoModel.loadFull(video.uuid, t)
239 await federateVideoIfNeeded(videoForFederation, true, t)
240 214
241 // Update video import object 215 // Update video DB object
242 videoImportToUpdate.state = VideoImportState.SUCCESS 216 video.duration = duration
243 const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo 217 video.state = buildNextVideoState(video.state)
244 videoImportUpdated.Video = video 218 await video.save({ transaction: t })
245 219
246 videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] }) 220 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
221 if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
247 222
248 logger.info('Video %s imported.', video.uuid) 223 // Now we can federate the video (reload from database, we need more attributes)
224 const videoForFederation = await VideoModel.loadFull(video.uuid, t)
225 await federateVideoIfNeeded(videoForFederation, true, t)
249 226
250 return { videoImportUpdated, video: videoForFederation } 227 // Update video import object
251 }).catch(err => { 228 videoImportWithFiles.state = VideoImportState.SUCCESS
252 // Reset fields 229 const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport
253 if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave)
254 if (previewModel) previewModel = new ThumbnailModel(previewSave)
255 230
256 videoFile = new VideoFileModel(videoFileSave) 231 logger.info('Video %s imported.', video.uuid)
257 232
258 throw err 233 return { videoImportUpdated, video: videoForFederation }
259 }) 234 }).catch(err => {
260 }) 235 // Reset fields
236 if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave)
237 if (previewModel) previewModel = new ThumbnailModel(previewSave)
261 238
262 Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: videoImportUpdated, success: true }) 239 videoFile = new VideoFileModel(videoFileSave)
263 240
264 if (video.isBlacklisted()) { 241 throw err
265 const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) 242 })
243 })
266 244
267 Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) 245 await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User })
268 } else { 246 } finally {
269 Notifier.Instance.notifyOnNewVideoIfNeeded(video) 247 videoFileLockReleaser()
270 } 248 }
249 } catch (err) {
250 await onImportError(err, tempVideoPath, videoImport)
271 251
272 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { 252 throw err
273 await JobQueue.Instance.createJob( 253 }
274 await buildMoveToObjectStorageJob({ video: videoImportUpdated.Video, previousVideoState: VideoState.TO_IMPORT }) 254}
275 )
276 }
277 255
278 // Create transcoding jobs? 256async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> {
279 if (video.state === VideoState.TO_TRANSCODE) { 257 // Refresh video, privacy may have changed
280 await JobQueue.Instance.createJob( 258 const video = await videoImport.Video.reload()
281 await buildOptimizeOrMergeAudioJob({ video: videoImportUpdated.Video, videoFile, user: videoImport.User }) 259 const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
282 )
283 }
284 260
285 } catch (err) { 261 return Object.assign(videoImport, { Video: videoWithFiles })
286 try { 262}
287 if (tempVideoPath) await remove(tempVideoPath)
288 } catch (errUnlink) {
289 logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
290 }
291 263
292 videoImport.error = err.message 264async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) {
293 if (videoImport.state !== VideoImportState.REJECTED) { 265 // Generate miniature if the import did not created it
294 videoImport.state = VideoImportState.FAILED 266 const needsMiniature = thumbnailType === ThumbnailType.MINIATURE
267 ? !videoImportWithFiles.Video.getMiniature()
268 : !videoImportWithFiles.Video.getPreview()
269
270 if (!needsMiniature) {
271 return {
272 miniatureModel: null,
273 miniatureJSONSave: null
295 } 274 }
296 await videoImport.save() 275 }
297 276
298 Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) 277 const miniatureModel = await generateVideoMiniature({
278 video: videoImportWithFiles.Video,
279 videoFile,
280 type: thumbnailType
281 })
282 const miniatureJSONSave = miniatureModel.toJSON()
299 283
300 throw err 284 return {
285 miniatureModel,
286 miniatureJSONSave
287 }
288}
289
290async function afterImportSuccess (options: {
291 videoImport: MVideoImport
292 video: MVideoFullLight
293 videoFile: MVideoFile
294 user: MUserId
295}) {
296 const { video, videoFile, videoImport, user } = options
297
298 Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
299
300 if (video.isBlacklisted()) {
301 const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
302
303 Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
304 } else {
305 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
301 } 306 }
307
308 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
309 await JobQueue.Instance.createJob(
310 await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
311 )
312 return
313 }
314
315 if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
316 await JobQueue.Instance.createJob(
317 await buildOptimizeOrMergeAudioJob({ video, videoFile, user })
318 )
319 }
320}
321
322async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) {
323 try {
324 if (tempVideoPath) await remove(tempVideoPath)
325 } catch (errUnlink) {
326 logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
327 }
328
329 videoImport.error = err.message
330 if (videoImport.state !== VideoImportState.REJECTED) {
331 videoImport.state = VideoImportState.FAILED
332 }
333 await videoImport.save()
334
335 Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
302} 336}
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 8a3ee09a2..c6263f55a 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -4,7 +4,7 @@ import { join } from 'path'
4import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' 4import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg'
5import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 5import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
7import { cleanupPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' 7import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
8import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' 8import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
9import { generateVideoMiniature } from '@server/lib/thumbnail' 9import { generateVideoMiniature } from '@server/lib/thumbnail'
10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' 10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding'
@@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
18import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' 18import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
19import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 19import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
20import { logger, loggerTagsFactory } from '../../../helpers/logger' 20import { logger, loggerTagsFactory } from '../../../helpers/logger'
21import { VideoPathManager } from '@server/lib/video-path-manager'
21 22
22const lTags = loggerTagsFactory('live', 'job') 23const lTags = loggerTagsFactory('live', 'job')
23 24
@@ -34,13 +35,13 @@ async function processVideoLiveEnding (job: Job) {
34 const live = await VideoLiveModel.loadByVideoId(payload.videoId) 35 const live = await VideoLiveModel.loadByVideoId(payload.videoId)
35 const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) 36 const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId)
36 37
37 const permanentLive = live.permanentLive
38
39 if (!video || !live || !liveSession) { 38 if (!video || !live || !liveSession) {
40 logError() 39 logError()
41 return 40 return
42 } 41 }
43 42
43 const permanentLive = live.permanentLive
44
44 liveSession.endingProcessed = true 45 liveSession.endingProcessed = true
45 await liveSession.save() 46 await liveSession.save()
46 47
@@ -141,23 +142,22 @@ async function replaceLiveByReplay (options: {
141}) { 142}) {
142 const { video, liveSession, live, permanentLive, replayDirectory } = options 143 const { video, liveSession, live, permanentLive, replayDirectory } = options
143 144
144 await cleanupTMPLiveFiles(video) 145 const videoWithFiles = await VideoModel.loadFull(video.id)
146 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
147
148 await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist)
145 149
146 await live.destroy() 150 await live.destroy()
147 151
148 video.isLive = false 152 videoWithFiles.isLive = false
149 video.waitTranscoding = true 153 videoWithFiles.waitTranscoding = true
150 video.state = VideoState.TO_TRANSCODE 154 videoWithFiles.state = VideoState.TO_TRANSCODE
151 155
152 await video.save() 156 await videoWithFiles.save()
153 157
154 liveSession.replayVideoId = video.id 158 liveSession.replayVideoId = videoWithFiles.id
155 await liveSession.save() 159 await liveSession.save()
156 160
157 // Remove old HLS playlist video files
158 const videoWithFiles = await VideoModel.loadFull(video.id)
159
160 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
161 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) 161 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
162 162
163 // Reset playlist 163 // Reset playlist
@@ -206,18 +206,27 @@ async function assignReplayFilesToVideo (options: {
206 const concatenatedTsFiles = await readdir(replayDirectory) 206 const concatenatedTsFiles = await readdir(replayDirectory)
207 207
208 for (const concatenatedTsFile of concatenatedTsFiles) { 208 for (const concatenatedTsFile of concatenatedTsFiles) {
209 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
210
209 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) 211 const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
210 212
211 const probe = await ffprobePromise(concatenatedTsFilePath) 213 const probe = await ffprobePromise(concatenatedTsFilePath)
212 const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) 214 const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
213 const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) 215 const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
214 216
215 await generateHlsPlaylistResolutionFromTS({ 217 try {
216 video, 218 await generateHlsPlaylistResolutionFromTS({
217 concatenatedTsFilePath, 219 video,
218 resolution, 220 inputFileMutexReleaser,
219 isAAC: audioStream?.codec_name === 'aac' 221 concatenatedTsFilePath,
220 }) 222 resolution,
223 isAAC: audioStream?.codec_name === 'aac'
224 })
225 } catch (err) {
226 logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
227 }
228
229 inputFileMutexReleaser()
221 } 230 }
222 231
223 return video 232 return video
@@ -234,7 +243,7 @@ async function cleanupLiveAndFederate (options: {
234 243
235 if (streamingPlaylist) { 244 if (streamingPlaylist) {
236 if (permanentLive) { 245 if (permanentLive) {
237 await cleanupPermanentLive(video, streamingPlaylist) 246 await cleanupAndDestroyPermanentLive(video, streamingPlaylist)
238 } else { 247 } else {
239 await cleanupUnsavedNormalLive(video, streamingPlaylist) 248 await cleanupUnsavedNormalLive(video, streamingPlaylist)
240 } 249 }
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index b0e92acf7..3e6d23363 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -13,7 +13,6 @@ import {
13 MergeAudioTranscodingPayload, 13 MergeAudioTranscodingPayload,
14 NewWebTorrentResolutionTranscodingPayload, 14 NewWebTorrentResolutionTranscodingPayload,
15 OptimizeTranscodingPayload, 15 OptimizeTranscodingPayload,
16 VideoResolution,
17 VideoTranscodingPayload 16 VideoTranscodingPayload
18} from '@shared/models' 17} from '@shared/models'
19import { retryTransactionWrapper } from '../../../helpers/database-utils' 18import { retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -94,15 +93,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
94 93
95 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 94 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
96 95
97 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { 96 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
98 return generateHlsPlaylistResolution({ 97
99 video, 98 try {
100 videoInputPath, 99 await videoFileInput.getVideo().reload()
101 resolution: payload.resolution, 100
102 copyCodecs: payload.copyCodecs, 101 await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
103 job 102 return generateHlsPlaylistResolution({
103 video,
104 videoInputPath,
105 inputFileMutexReleaser,
106 resolution: payload.resolution,
107 copyCodecs: payload.copyCodecs,
108 job
109 })
104 }) 110 })
105 }) 111 } finally {
112 inputFileMutexReleaser()
113 }
106 114
107 logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) 115 logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
108 116
@@ -177,38 +185,44 @@ async function onVideoFirstWebTorrentTranscoding (
177 transcodeType: TranscodeVODOptionsType, 185 transcodeType: TranscodeVODOptionsType,
178 user: MUserId 186 user: MUserId
179) { 187) {
180 const { resolution, audioStream } = await videoArg.probeMaxQualityFile() 188 const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
181 189
182 // Maybe the video changed in database, refresh it 190 try {
183 const videoDatabase = await VideoModel.loadFull(videoArg.uuid) 191 // Maybe the video changed in database, refresh it
184 // Video does not exist anymore 192 const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
185 if (!videoDatabase) return undefined 193 // Video does not exist anymore
186 194 if (!videoDatabase) return undefined
187 // Generate HLS version of the original file 195
188 const originalFileHLSPayload = { 196 const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
189 ...payload, 197
190 198 // Generate HLS version of the original file
191 hasAudio: !!audioStream, 199 const originalFileHLSPayload = {
192 resolution: videoDatabase.getMaxQualityFile().resolution, 200 ...payload,
193 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues 201
194 copyCodecs: transcodeType !== 'quick-transcode', 202 hasAudio: !!audioStream,
195 isMaxQuality: true 203 resolution: videoDatabase.getMaxQualityFile().resolution,
196 } 204 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
197 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) 205 copyCodecs: transcodeType !== 'quick-transcode',
198 const hasNewResolutions = await createLowerResolutionsJobs({ 206 isMaxQuality: true
199 video: videoDatabase, 207 }
200 user, 208 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
201 videoFileResolution: resolution, 209 const hasNewResolutions = await createLowerResolutionsJobs({
202 hasAudio: !!audioStream, 210 video: videoDatabase,
203 type: 'webtorrent', 211 user,
204 isNewVideo: payload.isNewVideo ?? true 212 videoFileResolution: resolution,
205 }) 213 hasAudio: !!audioStream,
206 214 type: 'webtorrent',
207 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') 215 isNewVideo: payload.isNewVideo ?? true
208 216 })
209 // Move to next state if there are no other resolutions to generate 217
210 if (!hasHls && !hasNewResolutions) { 218 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
211 await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) 219
220 // Move to next state if there are no other resolutions to generate
221 if (!hasHls && !hasNewResolutions) {
222 await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
223 }
224 } finally {
225 mutexReleaser()
212 } 226 }
213} 227}
214 228
@@ -266,7 +280,7 @@ async function createLowerResolutionsJobs (options: {
266 280
267 // Create transcoding jobs if there are enabled resolutions 281 // Create transcoding jobs if there are enabled resolutions
268 const resolutionsEnabled = await Hooks.wrapObject( 282 const resolutionsEnabled = await Hooks.wrapObject(
269 computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true }), 283 computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }),
270 'filter:transcoding.auto.resolutions-to-transcode.result', 284 'filter:transcoding.auto.resolutions-to-transcode.result',
271 options 285 options
272 ) 286 )
@@ -274,8 +288,6 @@ async function createLowerResolutionsJobs (options: {
274 const resolutionCreated: string[] = [] 288 const resolutionCreated: string[] = []
275 289
276 for (const resolution of resolutionsEnabled) { 290 for (const resolution of resolutionsEnabled) {
277 if (resolution === VideoResolution.H_NOVIDEO && hasAudio === false) continue
278
279 let dataInput: VideoTranscodingPayload 291 let dataInput: VideoTranscodingPayload
280 292
281 if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') { 293 if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') {
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 655be6568..6bc59732f 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -63,6 +63,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending'
63import { processVideoStudioEdition } from './handlers/video-studio-edition' 63import { processVideoStudioEdition } from './handlers/video-studio-edition'
64import { processVideoTranscoding } from './handlers/video-transcoding' 64import { processVideoTranscoding } from './handlers/video-transcoding'
65import { processVideosViewsStats } from './handlers/video-views-stats' 65import { processVideosViewsStats } from './handlers/video-views-stats'
66import { Redis } from '../redis'
66 67
67export type CreateJobArgument = 68export type CreateJobArgument =
68 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 69 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -183,7 +184,7 @@ class JobQueue {
183 } 184 }
184 185
185 this.flowProducer = new FlowProducer({ 186 this.flowProducer = new FlowProducer({
186 connection: this.getRedisConnection(), 187 connection: Redis.getRedisClientOptions('FlowProducer'),
187 prefix: this.jobRedisPrefix 188 prefix: this.jobRedisPrefix
188 }) 189 })
189 this.flowProducer.on('error', err => { logger.error('Error in flow producer', { err }) }) 190 this.flowProducer.on('error', err => { logger.error('Error in flow producer', { err }) })
@@ -196,7 +197,7 @@ class JobQueue {
196 autorun: false, 197 autorun: false,
197 concurrency: this.getJobConcurrency(handlerName), 198 concurrency: this.getJobConcurrency(handlerName),
198 prefix: this.jobRedisPrefix, 199 prefix: this.jobRedisPrefix,
199 connection: this.getRedisConnection() 200 connection: Redis.getRedisClientOptions('Worker')
200 } 201 }
201 202
202 const handler = function (job: Job) { 203 const handler = function (job: Job) {
@@ -236,7 +237,7 @@ class JobQueue {
236 237
237 private buildQueue (handlerName: JobType) { 238 private buildQueue (handlerName: JobType) {
238 const queueOptions: QueueOptions = { 239 const queueOptions: QueueOptions = {
239 connection: this.getRedisConnection(), 240 connection: Redis.getRedisClientOptions('Queue'),
240 prefix: this.jobRedisPrefix 241 prefix: this.jobRedisPrefix
241 } 242 }
242 243
@@ -249,7 +250,7 @@ class JobQueue {
249 private buildQueueScheduler (handlerName: JobType) { 250 private buildQueueScheduler (handlerName: JobType) {
250 const queueSchedulerOptions: QueueSchedulerOptions = { 251 const queueSchedulerOptions: QueueSchedulerOptions = {
251 autorun: false, 252 autorun: false,
252 connection: this.getRedisConnection(), 253 connection: Redis.getRedisClientOptions('QueueScheduler'),
253 prefix: this.jobRedisPrefix, 254 prefix: this.jobRedisPrefix,
254 maxStalledCount: 10 255 maxStalledCount: 10
255 } 256 }
@@ -263,7 +264,7 @@ class JobQueue {
263 private buildQueueEvent (handlerName: JobType) { 264 private buildQueueEvent (handlerName: JobType) {
264 const queueEventsOptions: QueueEventsOptions = { 265 const queueEventsOptions: QueueEventsOptions = {
265 autorun: false, 266 autorun: false,
266 connection: this.getRedisConnection(), 267 connection: Redis.getRedisClientOptions('QueueEvent'),
267 prefix: this.jobRedisPrefix 268 prefix: this.jobRedisPrefix
268 } 269 }
269 270
@@ -273,16 +274,6 @@ class JobQueue {
273 this.queueEvents[handlerName] = queueEvents 274 this.queueEvents[handlerName] = queueEvents
274 } 275 }
275 276
276 private getRedisConnection () {
277 return {
278 password: CONFIG.REDIS.AUTH,
279 db: CONFIG.REDIS.DB,
280 host: CONFIG.REDIS.HOSTNAME,
281 port: CONFIG.REDIS.PORT,
282 path: CONFIG.REDIS.SOCKET
283 }
284 }
285
286 // --------------------------------------------------------------------------- 277 // ---------------------------------------------------------------------------
287 278
288 async terminate () { 279 async terminate () {
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 16715862b..5e459f3c3 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -21,14 +21,14 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
21import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 21import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
22import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' 22import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models'
23import { pick, wait } from '@shared/core-utils' 23import { pick, wait } from '@shared/core-utils'
24import { LiveVideoError, VideoState, VideoStreamingPlaylistType } from '@shared/models' 24import { LiveVideoError, VideoState, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
25import { federateVideoIfNeeded } from '../activitypub/videos' 25import { federateVideoIfNeeded } from '../activitypub/videos'
26import { JobQueue } from '../job-queue' 26import { JobQueue } from '../job-queue'
27import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' 27import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths'
28import { PeerTubeSocket } from '../peertube-socket' 28import { PeerTubeSocket } from '../peertube-socket'
29import { Hooks } from '../plugins/hooks' 29import { Hooks } from '../plugins/hooks'
30import { LiveQuotaStore } from './live-quota-store' 30import { LiveQuotaStore } from './live-quota-store'
31import { cleanupPermanentLive } from './live-utils' 31import { cleanupAndDestroyPermanentLive } from './live-utils'
32import { MuxingSession } from './shared' 32import { MuxingSession } from './shared'
33 33
34const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') 34const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
@@ -224,7 +224,7 @@ class LiveManager {
224 if (oldStreamingPlaylist) { 224 if (oldStreamingPlaylist) {
225 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) 225 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid)
226 226
227 await cleanupPermanentLive(video, oldStreamingPlaylist) 227 await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
228 } 228 }
229 229
230 this.videoSessions.set(video.id, sessionId) 230 this.videoSessions.set(video.id, sessionId)
@@ -245,7 +245,7 @@ class LiveManager {
245 ) 245 )
246 246
247 const allResolutions = await Hooks.wrapObject( 247 const allResolutions = await Hooks.wrapObject(
248 this.buildAllResolutionsToTranscode(resolution), 248 this.buildAllResolutionsToTranscode(resolution, hasAudio),
249 'filter:transcoding.auto.resolutions-to-transcode.result', 249 'filter:transcoding.auto.resolutions-to-transcode.result',
250 { video } 250 { video }
251 ) 251 )
@@ -301,7 +301,7 @@ class LiveManager {
301 ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) 301 ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ])
302 }) 302 })
303 303
304 muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) 304 muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
305 305
306 muxingSession.on('bad-socket-health', ({ videoId }) => { 306 muxingSession.on('bad-socket-health', ({ videoId }) => {
307 logger.error( 307 logger.error(
@@ -460,11 +460,11 @@ class LiveManager {
460 return join(directory, files.sort().reverse()[0]) 460 return join(directory, files.sort().reverse()[0])
461 } 461 }
462 462
463 private buildAllResolutionsToTranscode (originResolution: number) { 463 private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) {
464 const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 464 const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
465 465
466 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED 466 const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
467 ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false }) 467 ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio })
468 : [] 468 : []
469 469
470 if (resolutionsEnabled.length === 0) { 470 if (resolutionsEnabled.length === 0) {
@@ -485,6 +485,10 @@ class LiveManager {
485 485
486 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) 486 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
487 487
488 playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED
489 ? VideoStorage.OBJECT_STORAGE
490 : VideoStorage.FILE_SYSTEM
491
488 return playlist.save() 492 return playlist.save()
489 } 493 }
490 494
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
index 4af6f3ebf..4d03754a9 100644
--- a/server/lib/live/live-segment-sha-store.ts
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -1,61 +1,79 @@
1import { writeJson } from 'fs-extra'
1import { basename } from 'path' 2import { basename } from 'path'
3import { mapToJSON } from '@server/helpers/core-utils'
2import { logger, loggerTagsFactory } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { MStreamingPlaylistVideo } from '@server/types/models'
3import { buildSha256Segment } from '../hls' 6import { buildSha256Segment } from '../hls'
7import { storeHLSFileFromPath } from '../object-storage'
8import PQueue from 'p-queue'
4 9
5const lTags = loggerTagsFactory('live') 10const lTags = loggerTagsFactory('live')
6 11
7class LiveSegmentShaStore { 12class LiveSegmentShaStore {
8 13
9 private static instance: LiveSegmentShaStore 14 private readonly segmentsSha256 = new Map<string, string>()
10 15
11 private readonly segmentsSha256 = new Map<string, Map<string, string>>() 16 private readonly videoUUID: string
12 17 private readonly sha256Path: string
13 private constructor () { 18 private readonly streamingPlaylist: MStreamingPlaylistVideo
14 } 19 private readonly sendToObjectStorage: boolean
15 20 private readonly writeQueue = new PQueue({ concurrency: 1 })
16 getSegmentsSha256 (videoUUID: string) { 21
17 return this.segmentsSha256.get(videoUUID) 22 constructor (options: {
23 videoUUID: string
24 sha256Path: string
25 streamingPlaylist: MStreamingPlaylistVideo
26 sendToObjectStorage: boolean
27 }) {
28 this.videoUUID = options.videoUUID
29 this.sha256Path = options.sha256Path
30 this.streamingPlaylist = options.streamingPlaylist
31 this.sendToObjectStorage = options.sendToObjectStorage
18 } 32 }
19 33
20 async addSegmentSha (videoUUID: string, segmentPath: string) { 34 async addSegmentSha (segmentPath: string) {
21 const segmentName = basename(segmentPath) 35 logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID))
22 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
23 36
24 const shaResult = await buildSha256Segment(segmentPath) 37 const shaResult = await buildSha256Segment(segmentPath)
25 38
26 if (!this.segmentsSha256.has(videoUUID)) { 39 const segmentName = basename(segmentPath)
27 this.segmentsSha256.set(videoUUID, new Map()) 40 this.segmentsSha256.set(segmentName, shaResult)
28 }
29 41
30 const filesMap = this.segmentsSha256.get(videoUUID) 42 try {
31 filesMap.set(segmentName, shaResult) 43 await this.writeToDisk()
44 } catch (err) {
45 logger.error('Cannot write sha segments to disk.', { err })
46 }
32 } 47 }
33 48
34 removeSegmentSha (videoUUID: string, segmentPath: string) { 49 async removeSegmentSha (segmentPath: string) {
35 const segmentName = basename(segmentPath) 50 const segmentName = basename(segmentPath)
36 51
37 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) 52 logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID))
38 53
39 const filesMap = this.segmentsSha256.get(videoUUID) 54 if (!this.segmentsSha256.has(segmentName)) {
40 if (!filesMap) { 55 logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID))
41 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
42 return 56 return
43 } 57 }
44 58
45 if (!filesMap.has(segmentName)) { 59 this.segmentsSha256.delete(segmentName)
46 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
47 return
48 }
49 60
50 filesMap.delete(segmentName) 61 await this.writeToDisk()
51 } 62 }
52 63
53 cleanupShaSegments (videoUUID: string) { 64 private writeToDisk () {
54 this.segmentsSha256.delete(videoUUID) 65 return this.writeQueue.add(async () => {
55 } 66 await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256))
67
68 if (this.sendToObjectStorage) {
69 const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path)
56 70
57 static get Instance () { 71 if (this.streamingPlaylist.segmentsSha256Url !== url) {
58 return this.instance || (this.instance = new this()) 72 this.streamingPlaylist.segmentsSha256Url = url
73 await this.streamingPlaylist.save()
74 }
75 }
76 })
59 } 77 }
60} 78}
61 79
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index bba876642..c0dec9829 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,9 +1,10 @@
1import { pathExists, readdir, remove } from 'fs-extra' 1import { pathExists, readdir, remove } from 'fs-extra'
2import { basename, join } from 'path' 2import { basename, join } from 'path'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { MStreamingPlaylist, MVideo } from '@server/types/models' 4import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
5import { VideoStorage } from '@shared/models'
6import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage'
5import { getLiveDirectory } from '../paths' 7import { getLiveDirectory } from '../paths'
6import { LiveSegmentShaStore } from './live-segment-sha-store'
7 8
8function buildConcatenatedName (segmentOrPlaylistPath: string) { 9function buildConcatenatedName (segmentOrPlaylistPath: string) {
9 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) 10 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
@@ -11,8 +12,8 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
11 return 'concat-' + num[1] + '.ts' 12 return 'concat-' + num[1] + '.ts'
12} 13}
13 14
14async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 15async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
15 await cleanupTMPLiveFiles(video) 16 await cleanupTMPLiveFiles(video, streamingPlaylist)
16 17
17 await streamingPlaylist.destroy() 18 await streamingPlaylist.destroy()
18} 19}
@@ -20,32 +21,51 @@ async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamin
20async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 21async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
21 const hlsDirectory = getLiveDirectory(video) 22 const hlsDirectory = getLiveDirectory(video)
22 23
24 // We uploaded files to object storage too, remove them
25 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
26 await removeHLSObjectStorage(streamingPlaylist.withVideo(video))
27 }
28
23 await remove(hlsDirectory) 29 await remove(hlsDirectory)
24 30
25 await streamingPlaylist.destroy() 31 await streamingPlaylist.destroy()
32}
26 33
27 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) 34async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
35 await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video))
36
37 await cleanupTMPLiveFilesFromFilesystem(video)
28} 38}
29 39
30async function cleanupTMPLiveFiles (video: MVideo) { 40export {
31 const hlsDirectory = getLiveDirectory(video) 41 cleanupAndDestroyPermanentLive,
42 cleanupUnsavedNormalLive,
43 cleanupTMPLiveFiles,
44 buildConcatenatedName
45}
46
47// ---------------------------------------------------------------------------
32 48
33 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) 49function isTMPLiveFile (name: string) {
50 return name.endsWith('.ts') ||
51 name.endsWith('.m3u8') ||
52 name.endsWith('.json') ||
53 name.endsWith('.mpd') ||
54 name.endsWith('.m4s') ||
55 name.endsWith('.tmp')
56}
57
58async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
59 const hlsDirectory = getLiveDirectory(video)
34 60
35 if (!await pathExists(hlsDirectory)) return 61 if (!await pathExists(hlsDirectory)) return
36 62
37 logger.info('Cleanup TMP live files of %s.', hlsDirectory) 63 logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory)
38 64
39 const files = await readdir(hlsDirectory) 65 const files = await readdir(hlsDirectory)
40 66
41 for (const filename of files) { 67 for (const filename of files) {
42 if ( 68 if (isTMPLiveFile(filename)) {
43 filename.endsWith('.ts') ||
44 filename.endsWith('.m3u8') ||
45 filename.endsWith('.mpd') ||
46 filename.endsWith('.m4s') ||
47 filename.endsWith('.tmp')
48 ) {
49 const p = join(hlsDirectory, filename) 69 const p = join(hlsDirectory, filename)
50 70
51 remove(p) 71 remove(p)
@@ -54,9 +74,16 @@ async function cleanupTMPLiveFiles (video: MVideo) {
54 } 74 }
55} 75}
56 76
57export { 77async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
58 cleanupPermanentLive, 78 if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
59 cleanupUnsavedNormalLive, 79
60 cleanupTMPLiveFiles, 80 logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid)
61 buildConcatenatedName 81
82 const keys = await listHLSFileKeysOf(streamingPlaylist)
83
84 for (const key of keys) {
85 if (isTMPLiveFile(key)) {
86 await removeHLSFileObjectStorageByFullKey(key)
87 }
88 }
62} 89}
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index 505717dce..6ec126955 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -3,14 +3,17 @@ import { mapSeries } from 'bluebird'
3import { FSWatcher, watch } from 'chokidar' 3import { FSWatcher, watch } from 'chokidar'
4import { FfmpegCommand } from 'fluent-ffmpeg' 4import { FfmpegCommand } from 'fluent-ffmpeg'
5import { appendFile, ensureDir, readFile, stat } from 'fs-extra' 5import { appendFile, ensureDir, readFile, stat } from 'fs-extra'
6import PQueue from 'p-queue'
6import { basename, join } from 'path' 7import { basename, join } from 'path'
7import { EventEmitter } from 'stream' 8import { EventEmitter } from 'stream'
8import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg' 9import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg'
9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' 10import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
10import { CONFIG } from '@server/initializers/config' 11import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 12import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
13import { removeHLSFileObjectStorageByPath, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage'
12import { VideoFileModel } from '@server/models/video/video-file' 14import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 15import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
16import { VideoStorage } from '@shared/models'
14import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' 17import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths'
15import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' 18import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles'
16import { isAbleToUploadVideo } from '../../user' 19import { isAbleToUploadVideo } from '../../user'
@@ -19,9 +22,8 @@ import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils' 22import { buildConcatenatedName } from '../live-utils'
20 23
21import memoizee = require('memoizee') 24import memoizee = require('memoizee')
22
23interface MuxingSessionEvents { 25interface MuxingSessionEvents {
24 'master-playlist-created': (options: { videoId: number }) => void 26 'live-ready': (options: { videoId: number }) => void
25 27
26 'bad-socket-health': (options: { videoId: number }) => void 28 'bad-socket-health': (options: { videoId: number }) => void
27 'duration-exceeded': (options: { videoId: number }) => void 29 'duration-exceeded': (options: { videoId: number }) => void
@@ -68,12 +70,18 @@ class MuxingSession extends EventEmitter {
68 private readonly outDirectory: string 70 private readonly outDirectory: string
69 private readonly replayDirectory: string 71 private readonly replayDirectory: string
70 72
73 private readonly liveSegmentShaStore: LiveSegmentShaStore
74
71 private readonly lTags: LoggerTagsFn 75 private readonly lTags: LoggerTagsFn
72 76
73 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} 77 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
74 78
75 private tsWatcher: FSWatcher 79 private tsWatcher: FSWatcher
76 private masterWatcher: FSWatcher 80 private masterWatcher: FSWatcher
81 private m3u8Watcher: FSWatcher
82
83 private masterPlaylistCreated = false
84 private liveReady = false
77 85
78 private aborted = false 86 private aborted = false
79 87
@@ -123,6 +131,13 @@ class MuxingSession extends EventEmitter {
123 this.outDirectory = getLiveDirectory(this.videoLive.Video) 131 this.outDirectory = getLiveDirectory(this.videoLive.Video)
124 this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) 132 this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString())
125 133
134 this.liveSegmentShaStore = new LiveSegmentShaStore({
135 videoUUID: this.videoLive.Video.uuid,
136 sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename),
137 streamingPlaylist: this.streamingPlaylist,
138 sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED
139 })
140
126 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) 141 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID)
127 } 142 }
128 143
@@ -159,8 +174,9 @@ class MuxingSession extends EventEmitter {
159 174
160 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) 175 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags())
161 176
162 this.watchTSFiles()
163 this.watchMasterFile() 177 this.watchMasterFile()
178 this.watchTSFiles()
179 this.watchM3U8File()
164 180
165 let ffmpegShellCommand: string 181 let ffmpegShellCommand: string
166 this.ffmpegCommand.on('start', cmdline => { 182 this.ffmpegCommand.on('start', cmdline => {
@@ -219,7 +235,7 @@ class MuxingSession extends EventEmitter {
219 setTimeout(() => { 235 setTimeout(() => {
220 // Wait latest segments generation, and close watchers 236 // Wait latest segments generation, and close watchers
221 237
222 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) 238 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
223 .then(() => { 239 .then(() => {
224 // Process remaining segments hash 240 // Process remaining segments hash
225 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { 241 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
@@ -240,14 +256,48 @@ class MuxingSession extends EventEmitter {
240 private watchMasterFile () { 256 private watchMasterFile () {
241 this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) 257 this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
242 258
243 this.masterWatcher.on('add', () => { 259 this.masterWatcher.on('add', async () => {
244 this.emit('master-playlist-created', { videoId: this.videoId }) 260 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
261 try {
262 const url = await storeHLSFileFromFilename(this.streamingPlaylist, this.streamingPlaylist.playlistFilename)
263
264 this.streamingPlaylist.playlistUrl = url
265 await this.streamingPlaylist.save()
266 } catch (err) {
267 logger.error('Cannot upload live master file to object storage.', { err, ...this.lTags() })
268 }
269 }
270
271 this.masterPlaylistCreated = true
245 272
246 this.masterWatcher.close() 273 this.masterWatcher.close()
247 .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) 274 .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
248 }) 275 })
249 } 276 }
250 277
278 private watchM3U8File () {
279 this.m3u8Watcher = watch(this.outDirectory + '/*.m3u8')
280
281 const sendQueues = new Map<string, PQueue>()
282
283 const onChangeOrAdd = async (m3u8Path: string) => {
284 if (this.streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
285
286 try {
287 if (!sendQueues.has(m3u8Path)) {
288 sendQueues.set(m3u8Path, new PQueue({ concurrency: 1 }))
289 }
290
291 const queue = sendQueues.get(m3u8Path)
292 await queue.add(() => storeHLSFileFromPath(this.streamingPlaylist, m3u8Path))
293 } catch (err) {
294 logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() })
295 }
296 }
297
298 this.m3u8Watcher.on('change', onChangeOrAdd)
299 }
300
251 private watchTSFiles () { 301 private watchTSFiles () {
252 const startStreamDateTime = new Date().getTime() 302 const startStreamDateTime = new Date().getTime()
253 303
@@ -282,7 +332,21 @@ class MuxingSession extends EventEmitter {
282 } 332 }
283 } 333 }
284 334
285 const deleteHandler = (segmentPath: string) => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) 335 const deleteHandler = async (segmentPath: string) => {
336 try {
337 await this.liveSegmentShaStore.removeSegmentSha(segmentPath)
338 } catch (err) {
339 logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() })
340 }
341
342 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
343 try {
344 await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath)
345 } catch (err) {
346 logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() })
347 }
348 }
349 }
286 350
287 this.tsWatcher.on('add', p => addHandler(p)) 351 this.tsWatcher.on('add', p => addHandler(p))
288 this.tsWatcher.on('unlink', p => deleteHandler(p)) 352 this.tsWatcher.on('unlink', p => deleteHandler(p))
@@ -315,6 +379,7 @@ class MuxingSession extends EventEmitter {
315 extname: '.ts', 379 extname: '.ts',
316 infoHash: null, 380 infoHash: null,
317 fps: this.fps, 381 fps: this.fps,
382 storage: this.streamingPlaylist.storage,
318 videoStreamingPlaylistId: this.streamingPlaylist.id 383 videoStreamingPlaylistId: this.streamingPlaylist.id
319 }) 384 })
320 385
@@ -343,18 +408,36 @@ class MuxingSession extends EventEmitter {
343 } 408 }
344 409
345 private processSegments (segmentPaths: string[]) { 410 private processSegments (segmentPaths: string[]) {
346 mapSeries(segmentPaths, async previousSegment => { 411 mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment))
347 // Add sha hash of previous segments, because ffmpeg should have finished generating them 412 .catch(err => {
348 await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) 413 if (this.aborted) return
414
415 logger.error('Cannot process segments', { err, ...this.lTags() })
416 })
417 }
418
419 private async processSegment (segmentPath: string) {
420 // Add sha hash of previous segments, because ffmpeg should have finished generating them
421 await this.liveSegmentShaStore.addSegmentSha(segmentPath)
422
423 if (this.saveReplay) {
424 await this.addSegmentToReplay(segmentPath)
425 }
349 426
350 if (this.saveReplay) { 427 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
351 await this.addSegmentToReplay(previousSegment) 428 try {
429 await storeHLSFileFromPath(this.streamingPlaylist, segmentPath)
430 } catch (err) {
431 logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })
352 } 432 }
353 }).catch(err => { 433 }
354 if (this.aborted) return
355 434
356 logger.error('Cannot process segments', { err, ...this.lTags() }) 435 // Master playlist and segment JSON file are created, live is ready
357 }) 436 if (this.masterPlaylistCreated && !this.liveReady) {
437 this.liveReady = true
438
439 this.emit('live-ready', { videoId: this.videoId })
440 }
358 } 441 }
359 442
360 private hasClientSocketInBadHealth (sessionId: string) { 443 private hasClientSocketInBadHealth (sessionId: string) {
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index c23f5b6a6..dc5d8c83c 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -1,4 +1,4 @@
1import { VideoUploadFile } from 'express' 1import express, { VideoUploadFile } from 'express'
2import { PathLike } from 'fs-extra' 2import { PathLike } from 'fs-extra'
3import { Transaction } from 'sequelize/types' 3import { Transaction } from 'sequelize/types'
4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' 4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
@@ -13,18 +13,15 @@ import {
13 MAbuseFull, 13 MAbuseFull,
14 MAccountDefault, 14 MAccountDefault,
15 MAccountLight, 15 MAccountLight,
16 MComment,
16 MCommentAbuseAccountVideo, 17 MCommentAbuseAccountVideo,
17 MCommentOwnerVideo, 18 MCommentOwnerVideo,
18 MUser, 19 MUser,
19 MVideoAbuseVideoFull, 20 MVideoAbuseVideoFull,
20 MVideoAccountLightBlacklistAllFiles 21 MVideoAccountLightBlacklistAllFiles
21} from '@server/types/models' 22} from '@server/types/models'
22import { ActivityCreate } from '../../shared/models/activitypub'
23import { VideoObject } from '../../shared/models/activitypub/objects'
24import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
25import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' 23import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos'
26import { VideoCommentCreate } from '../../shared/models/videos/comment' 24import { VideoCommentCreate } from '../../shared/models/videos/comment'
27import { ActorModel } from '../models/actor/actor'
28import { UserModel } from '../models/user/user' 25import { UserModel } from '../models/user/user'
29import { VideoModel } from '../models/video/video' 26import { VideoModel } from '../models/video/video'
30import { VideoCommentModel } from '../models/video/video-comment' 27import { VideoCommentModel } from '../models/video/video-comment'
@@ -36,7 +33,9 @@ export type AcceptResult = {
36 errorMessage?: string 33 errorMessage?: string
37} 34}
38 35
39// Can be filtered by plugins 36// ---------------------------------------------------------------------------
37
38// Stub function that can be filtered by plugins
40function isLocalVideoAccepted (object: { 39function isLocalVideoAccepted (object: {
41 videoBody: VideoCreate 40 videoBody: VideoCreate
42 videoFile: VideoUploadFile 41 videoFile: VideoUploadFile
@@ -45,6 +44,9 @@ function isLocalVideoAccepted (object: {
45 return { accepted: true } 44 return { accepted: true }
46} 45}
47 46
47// ---------------------------------------------------------------------------
48
49// Stub function that can be filtered by plugins
48function isLocalLiveVideoAccepted (object: { 50function isLocalLiveVideoAccepted (object: {
49 liveVideoBody: LiveVideoCreate 51 liveVideoBody: LiveVideoCreate
50 user: UserModel 52 user: UserModel
@@ -52,7 +54,11 @@ function isLocalLiveVideoAccepted (object: {
52 return { accepted: true } 54 return { accepted: true }
53} 55}
54 56
57// ---------------------------------------------------------------------------
58
59// Stub function that can be filtered by plugins
55function isLocalVideoThreadAccepted (_object: { 60function isLocalVideoThreadAccepted (_object: {
61 req: express.Request
56 commentBody: VideoCommentCreate 62 commentBody: VideoCommentCreate
57 video: VideoModel 63 video: VideoModel
58 user: UserModel 64 user: UserModel
@@ -60,7 +66,9 @@ function isLocalVideoThreadAccepted (_object: {
60 return { accepted: true } 66 return { accepted: true }
61} 67}
62 68
69// Stub function that can be filtered by plugins
63function isLocalVideoCommentReplyAccepted (_object: { 70function isLocalVideoCommentReplyAccepted (_object: {
71 req: express.Request
64 commentBody: VideoCommentCreate 72 commentBody: VideoCommentCreate
65 parentComment: VideoCommentModel 73 parentComment: VideoCommentModel
66 video: VideoModel 74 video: VideoModel
@@ -69,22 +77,18 @@ function isLocalVideoCommentReplyAccepted (_object: {
69 return { accepted: true } 77 return { accepted: true }
70} 78}
71 79
72function isRemoteVideoAccepted (_object: { 80// ---------------------------------------------------------------------------
73 activity: ActivityCreate
74 videoAP: VideoObject
75 byActor: ActorModel
76}): AcceptResult {
77 return { accepted: true }
78}
79 81
82// Stub function that can be filtered by plugins
80function isRemoteVideoCommentAccepted (_object: { 83function isRemoteVideoCommentAccepted (_object: {
81 activity: ActivityCreate 84 comment: MComment
82 commentAP: VideoCommentObject
83 byActor: ActorModel
84}): AcceptResult { 85}): AcceptResult {
85 return { accepted: true } 86 return { accepted: true }
86} 87}
87 88
89// ---------------------------------------------------------------------------
90
91// Stub function that can be filtered by plugins
88function isPreImportVideoAccepted (object: { 92function isPreImportVideoAccepted (object: {
89 videoImportBody: VideoImportCreate 93 videoImportBody: VideoImportCreate
90 user: MUser 94 user: MUser
@@ -92,6 +96,7 @@ function isPreImportVideoAccepted (object: {
92 return { accepted: true } 96 return { accepted: true }
93} 97}
94 98
99// Stub function that can be filtered by plugins
95function isPostImportVideoAccepted (object: { 100function isPostImportVideoAccepted (object: {
96 videoFilePath: PathLike 101 videoFilePath: PathLike
97 videoFile: VideoFileModel 102 videoFile: VideoFileModel
@@ -100,6 +105,8 @@ function isPostImportVideoAccepted (object: {
100 return { accepted: true } 105 return { accepted: true }
101} 106}
102 107
108// ---------------------------------------------------------------------------
109
103async function createVideoAbuse (options: { 110async function createVideoAbuse (options: {
104 baseAbuse: FilteredModelAttributes<AbuseModel> 111 baseAbuse: FilteredModelAttributes<AbuseModel>
105 videoInstance: MVideoAccountLightBlacklistAllFiles 112 videoInstance: MVideoAccountLightBlacklistAllFiles
@@ -189,12 +196,13 @@ function createAccountAbuse (options: {
189 }) 196 })
190} 197}
191 198
199// ---------------------------------------------------------------------------
200
192export { 201export {
193 isLocalLiveVideoAccepted, 202 isLocalLiveVideoAccepted,
194 203
195 isLocalVideoAccepted, 204 isLocalVideoAccepted,
196 isLocalVideoThreadAccepted, 205 isLocalVideoThreadAccepted,
197 isRemoteVideoAccepted,
198 isRemoteVideoCommentAccepted, 206 isRemoteVideoCommentAccepted,
199 isLocalVideoCommentReplyAccepted, 207 isLocalVideoCommentReplyAccepted,
200 isPreImportVideoAccepted, 208 isPreImportVideoAccepted,
@@ -212,7 +220,7 @@ async function createAbuse (options: {
212 base: FilteredModelAttributes<AbuseModel> 220 base: FilteredModelAttributes<AbuseModel>
213 reporterAccount: MAccountDefault 221 reporterAccount: MAccountDefault
214 flaggedAccount: MAccountLight 222 flaggedAccount: MAccountLight
215 associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean} > 223 associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean }>
216 skipNotification: boolean 224 skipNotification: boolean
217 transaction: Transaction 225 transaction: Transaction
218}) { 226}) {
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index 16161362c..3046d76bc 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -1,19 +1,23 @@
1import { map } from 'bluebird'
1import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' 2import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra'
2import { dirname } from 'path' 3import { dirname } from 'path'
3import { Readable } from 'stream' 4import { Readable } from 'stream'
4import { 5import {
6 _Object,
5 CompleteMultipartUploadCommandOutput, 7 CompleteMultipartUploadCommandOutput,
6 DeleteObjectCommand, 8 DeleteObjectCommand,
7 GetObjectCommand, 9 GetObjectCommand,
8 ListObjectsV2Command, 10 ListObjectsV2Command,
9 PutObjectCommandInput 11 PutObjectAclCommand,
12 PutObjectCommandInput,
13 S3Client
10} from '@aws-sdk/client-s3' 14} from '@aws-sdk/client-s3'
11import { Upload } from '@aws-sdk/lib-storage' 15import { Upload } from '@aws-sdk/lib-storage'
12import { pipelinePromise } from '@server/helpers/core-utils' 16import { pipelinePromise } from '@server/helpers/core-utils'
13import { isArray } from '@server/helpers/custom-validators/misc' 17import { isArray } from '@server/helpers/custom-validators/misc'
14import { logger } from '@server/helpers/logger' 18import { logger } from '@server/helpers/logger'
15import { CONFIG } from '@server/initializers/config' 19import { CONFIG } from '@server/initializers/config'
16import { getPrivateUrl } from '../urls' 20import { getInternalUrl } from '../urls'
17import { getClient } from './client' 21import { getClient } from './client'
18import { lTags } from './logger' 22import { lTags } from './logger'
19 23
@@ -22,73 +26,125 @@ type BucketInfo = {
22 PREFIX?: string 26 PREFIX?: string
23} 27}
24 28
29async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
30 const s3Client = getClient()
31
32 const commandPrefix = bucketInfo.PREFIX + prefix
33 const listCommand = new ListObjectsV2Command({
34 Bucket: bucketInfo.BUCKET_NAME,
35 Prefix: commandPrefix
36 })
37
38 const listedObjects = await s3Client.send(listCommand)
39
40 if (isArray(listedObjects.Contents) !== true) return []
41
42 return listedObjects.Contents.map(c => c.Key)
43}
44
45// ---------------------------------------------------------------------------
46
25async function storeObject (options: { 47async function storeObject (options: {
26 inputPath: string 48 inputPath: string
27 objectStorageKey: string 49 objectStorageKey: string
28 bucketInfo: BucketInfo 50 bucketInfo: BucketInfo
51 isPrivate: boolean
29}): Promise<string> { 52}): Promise<string> {
30 const { inputPath, objectStorageKey, bucketInfo } = options 53 const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
31 54
32 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) 55 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
33 56
34 const fileStream = createReadStream(inputPath) 57 const fileStream = createReadStream(inputPath)
35 58
36 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) 59 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
37} 60}
38 61
39async function removeObject (filename: string, bucketInfo: BucketInfo) { 62// ---------------------------------------------------------------------------
40 const command = new DeleteObjectCommand({ 63
64function updateObjectACL (options: {
65 objectStorageKey: string
66 bucketInfo: BucketInfo
67 isPrivate: boolean
68}) {
69 const { objectStorageKey, bucketInfo, isPrivate } = options
70
71 const key = buildKey(objectStorageKey, bucketInfo)
72
73 logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
74
75 const command = new PutObjectAclCommand({
41 Bucket: bucketInfo.BUCKET_NAME, 76 Bucket: bucketInfo.BUCKET_NAME,
42 Key: buildKey(filename, bucketInfo) 77 Key: key,
78 ACL: getACL(isPrivate)
43 }) 79 })
44 80
45 return getClient().send(command) 81 return getClient().send(command)
46} 82}
47 83
48async function removePrefix (prefix: string, bucketInfo: BucketInfo) { 84function updatePrefixACL (options: {
49 const s3Client = getClient() 85 prefix: string
50 86 bucketInfo: BucketInfo
51 const commandPrefix = bucketInfo.PREFIX + prefix 87 isPrivate: boolean
52 const listCommand = new ListObjectsV2Command({ 88}) {
53 Bucket: bucketInfo.BUCKET_NAME, 89 const { prefix, bucketInfo, isPrivate } = options
54 Prefix: commandPrefix 90
91 logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
92
93 return applyOnPrefix({
94 prefix,
95 bucketInfo,
96 commandBuilder: obj => {
97 logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
98
99 return new PutObjectAclCommand({
100 Bucket: bucketInfo.BUCKET_NAME,
101 Key: obj.Key,
102 ACL: getACL(isPrivate)
103 })
104 }
55 }) 105 })
106}
56 107
57 const listedObjects = await s3Client.send(listCommand) 108// ---------------------------------------------------------------------------
58
59 // FIXME: use bulk delete when s3ninja will support this operation
60 // const deleteParams = {
61 // Bucket: bucketInfo.BUCKET_NAME,
62 // Delete: { Objects: [] }
63 // }
64 109
65 if (isArray(listedObjects.Contents) !== true) { 110function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
66 const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` 111 const key = buildKey(objectStorageKey, bucketInfo)
67 112
68 logger.error(message, { response: listedObjects, ...lTags() }) 113 return removeObjectByFullKey(key, bucketInfo)
69 throw new Error(message) 114}
70 }
71 115
72 for (const object of listedObjects.Contents) { 116function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
73 const command = new DeleteObjectCommand({ 117 logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
74 Bucket: bucketInfo.BUCKET_NAME,
75 Key: object.Key
76 })
77 118
78 await s3Client.send(command) 119 const command = new DeleteObjectCommand({
120 Bucket: bucketInfo.BUCKET_NAME,
121 Key: fullKey
122 })
79 123
80 // FIXME: use bulk delete when s3ninja will support this operation 124 return getClient().send(command)
81 // deleteParams.Delete.Objects.push({ Key: object.Key }) 125}
82 }
83 126
127async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
84 // FIXME: use bulk delete when s3ninja will support this operation 128 // FIXME: use bulk delete when s3ninja will support this operation
85 // const deleteCommand = new DeleteObjectsCommand(deleteParams)
86 // await s3Client.send(deleteCommand)
87 129
88 // Repeat if not all objects could be listed at once (limit of 1000?) 130 logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
89 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) 131
132 return applyOnPrefix({
133 prefix,
134 bucketInfo,
135 commandBuilder: obj => {
136 logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
137
138 return new DeleteObjectCommand({
139 Bucket: bucketInfo.BUCKET_NAME,
140 Key: obj.Key
141 })
142 }
143 })
90} 144}
91 145
146// ---------------------------------------------------------------------------
147
92async function makeAvailable (options: { 148async function makeAvailable (options: {
93 key: string 149 key: string
94 destination: string 150 destination: string
@@ -116,13 +172,43 @@ function buildKey (key: string, bucketInfo: BucketInfo) {
116 172
117// --------------------------------------------------------------------------- 173// ---------------------------------------------------------------------------
118 174
175async function createObjectReadStream (options: {
176 key: string
177 bucketInfo: BucketInfo
178 rangeHeader: string
179}) {
180 const { key, bucketInfo, rangeHeader } = options
181
182 const command = new GetObjectCommand({
183 Bucket: bucketInfo.BUCKET_NAME,
184 Key: buildKey(key, bucketInfo),
185 Range: rangeHeader
186 })
187
188 const response = await getClient().send(command)
189
190 return response.Body as Readable
191}
192
193// ---------------------------------------------------------------------------
194
119export { 195export {
120 BucketInfo, 196 BucketInfo,
121 buildKey, 197 buildKey,
198
122 storeObject, 199 storeObject,
200
123 removeObject, 201 removeObject,
202 removeObjectByFullKey,
124 removePrefix, 203 removePrefix,
125 makeAvailable 204
205 makeAvailable,
206
207 updateObjectACL,
208 updatePrefixACL,
209
210 listKeysOfPrefix,
211 createObjectReadStream
126} 212}
127 213
128// --------------------------------------------------------------------------- 214// ---------------------------------------------------------------------------
@@ -131,17 +217,15 @@ async function uploadToStorage (options: {
131 content: ReadStream 217 content: ReadStream
132 objectStorageKey: string 218 objectStorageKey: string
133 bucketInfo: BucketInfo 219 bucketInfo: BucketInfo
220 isPrivate: boolean
134}) { 221}) {
135 const { content, objectStorageKey, bucketInfo } = options 222 const { content, objectStorageKey, bucketInfo, isPrivate } = options
136 223
137 const input: PutObjectCommandInput = { 224 const input: PutObjectCommandInput = {
138 Body: content, 225 Body: content,
139 Bucket: bucketInfo.BUCKET_NAME, 226 Bucket: bucketInfo.BUCKET_NAME,
140 Key: buildKey(objectStorageKey, bucketInfo) 227 Key: buildKey(objectStorageKey, bucketInfo),
141 } 228 ACL: getACL(isPrivate)
142
143 if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) {
144 input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL
145 } 229 }
146 230
147 const parallelUploads3 = new Upload({ 231 const parallelUploads3 = new Upload({
@@ -171,5 +255,50 @@ async function uploadToStorage (options: {
171 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() 255 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
172 ) 256 )
173 257
174 return getPrivateUrl(bucketInfo, objectStorageKey) 258 return getInternalUrl(bucketInfo, objectStorageKey)
259}
260
261async function applyOnPrefix (options: {
262 prefix: string
263 bucketInfo: BucketInfo
264 commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
265
266 continuationToken?: string
267}) {
268 const { prefix, bucketInfo, commandBuilder, continuationToken } = options
269
270 const s3Client = getClient()
271
272 const commandPrefix = buildKey(prefix, bucketInfo)
273 const listCommand = new ListObjectsV2Command({
274 Bucket: bucketInfo.BUCKET_NAME,
275 Prefix: commandPrefix,
276 ContinuationToken: continuationToken
277 })
278
279 const listedObjects = await s3Client.send(listCommand)
280
281 if (isArray(listedObjects.Contents) !== true) {
282 const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
283
284 logger.error(message, { response: listedObjects, ...lTags() })
285 throw new Error(message)
286 }
287
288 await map(listedObjects.Contents, object => {
289 const command = commandBuilder(object)
290
291 return s3Client.send(command)
292 }, { concurrency: 10 })
293
294 // Repeat if not all objects could be listed at once (limit of 1000?)
295 if (listedObjects.IsTruncated) {
296 await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
297 }
298}
299
300function getACL (isPrivate: boolean) {
301 return isPrivate
302 ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE
303 : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC
175} 304}
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts
index 2a889190b..a47a98b98 100644
--- a/server/lib/object-storage/urls.ts
+++ b/server/lib/object-storage/urls.ts
@@ -1,10 +1,14 @@
1import { CONFIG } from '@server/initializers/config' 1import { CONFIG } from '@server/initializers/config'
2import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants'
3import { MVideoUUID } from '@server/types/models'
2import { BucketInfo, buildKey, getEndpointParsed } from './shared' 4import { BucketInfo, buildKey, getEndpointParsed } from './shared'
3 5
4function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { 6function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
5 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) 7 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
6} 8}
7 9
10// ---------------------------------------------------------------------------
11
8function getWebTorrentPublicFileUrl (fileUrl: string) { 12function getWebTorrentPublicFileUrl (fileUrl: string) {
9 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL 13 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
10 if (!baseUrl) return fileUrl 14 if (!baseUrl) return fileUrl
@@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) {
19 return replaceByBaseUrl(fileUrl, baseUrl) 23 return replaceByBaseUrl(fileUrl, baseUrl)
20} 24}
21 25
26// ---------------------------------------------------------------------------
27
28function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
29 return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
30}
31
32function getWebTorrentPrivateFileUrl (filename: string) {
33 return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename
34}
35
36// ---------------------------------------------------------------------------
37
22export { 38export {
23 getPrivateUrl, 39 getInternalUrl,
40
24 getWebTorrentPublicFileUrl, 41 getWebTorrentPublicFileUrl,
25 replaceByBaseUrl, 42 getHLSPublicFileUrl,
26 getHLSPublicFileUrl 43
44 getHLSPrivateFileUrl,
45 getWebTorrentPrivateFileUrl,
46
47 replaceByBaseUrl
27} 48}
28 49
29// --------------------------------------------------------------------------- 50// ---------------------------------------------------------------------------
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 66e738200..b764e4b22 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,39 +1,102 @@
1import { join } from 'path' 1import { basename, join } from 'path'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' 4import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models'
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { VideoPathManager } from '../video-path-manager'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 7import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 8import {
9 createObjectReadStream,
10 listKeysOfPrefix,
11 lTags,
12 makeAvailable,
13 removeObject,
14 removeObjectByFullKey,
15 removePrefix,
16 storeObject,
17 updateObjectACL,
18 updatePrefixACL
19} from './shared'
20
21function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
22 return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
23}
24
25// ---------------------------------------------------------------------------
8 26
9function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) { 27function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
10 return storeObject({ 28 return storeObject({
11 inputPath: path ?? join(getHLSDirectory(playlist.Video), filename), 29 inputPath: join(getHLSDirectory(playlist.Video), filename),
12 objectStorageKey: generateHLSObjectStorageKey(playlist, filename), 30 objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
13 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 31 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
32 isPrivate: playlist.Video.hasPrivateStaticPath()
14 }) 33 })
15} 34}
16 35
17function storeWebTorrentFile (filename: string) { 36function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
18 return storeObject({ 37 return storeObject({
19 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), 38 inputPath: path,
20 objectStorageKey: generateWebTorrentObjectStorageKey(filename), 39 objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
21 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS 40 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
41 isPrivate: playlist.Video.hasPrivateStaticPath()
42 })
43}
44
45// ---------------------------------------------------------------------------
46
47function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
48 return storeObject({
49 inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
50 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
51 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
52 isPrivate: video.hasPrivateStaticPath()
53 })
54}
55
56// ---------------------------------------------------------------------------
57
58function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) {
59 return updateObjectACL({
60 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
61 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
62 isPrivate: video.hasPrivateStaticPath()
63 })
64}
65
66function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
67 return updatePrefixACL({
68 prefix: generateHLSObjectBaseStorageKey(playlist),
69 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
70 isPrivate: playlist.Video.hasPrivateStaticPath()
22 }) 71 })
23} 72}
24 73
74// ---------------------------------------------------------------------------
75
25function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { 76function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
26 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 77 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
27} 78}
28 79
29function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) { 80function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
30 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 81 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
31} 82}
32 83
84function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
85 return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
86}
87
88function removeHLSFileObjectStorageByFullKey (key: string) {
89 return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
90}
91
92// ---------------------------------------------------------------------------
93
33function removeWebTorrentObjectStorage (videoFile: MVideoFile) { 94function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
34 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) 95 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
35} 96}
36 97
98// ---------------------------------------------------------------------------
99
37async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { 100async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
38 const key = generateHLSObjectStorageKey(playlist, filename) 101 const key = generateHLSObjectStorageKey(playlist, filename)
39 102
@@ -62,14 +125,61 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
62 return destination 125 return destination
63} 126}
64 127
128// ---------------------------------------------------------------------------
129
130function getWebTorrentFileReadStream (options: {
131 filename: string
132 rangeHeader: string
133}) {
134 const { filename, rangeHeader } = options
135
136 const key = generateWebTorrentObjectStorageKey(filename)
137
138 return createObjectReadStream({
139 key,
140 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
141 rangeHeader
142 })
143}
144
145function getHLSFileReadStream (options: {
146 playlist: MStreamingPlaylistVideo
147 filename: string
148 rangeHeader: string
149}) {
150 const { playlist, filename, rangeHeader } = options
151
152 const key = generateHLSObjectStorageKey(playlist, filename)
153
154 return createObjectReadStream({
155 key,
156 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
157 rangeHeader
158 })
159}
160
161// ---------------------------------------------------------------------------
162
65export { 163export {
164 listHLSFileKeysOf,
165
66 storeWebTorrentFile, 166 storeWebTorrentFile,
67 storeHLSFile, 167 storeHLSFileFromFilename,
168 storeHLSFileFromPath,
169
170 updateWebTorrentFileACL,
171 updateHLSFilesACL,
68 172
69 removeHLSObjectStorage, 173 removeHLSObjectStorage,
70 removeHLSFileObjectStorage, 174 removeHLSFileObjectStorageByFilename,
175 removeHLSFileObjectStorageByPath,
176 removeHLSFileObjectStorageByFullKey,
177
71 removeWebTorrentObjectStorage, 178 removeWebTorrentObjectStorage,
72 179
73 makeWebTorrentFileAvailable, 180 makeWebTorrentFileAvailable,
74 makeHLSFileAvailable 181 makeHLSFileAvailable,
182
183 getWebTorrentFileReadStream,
184 getHLSFileReadStream
75} 185}
diff --git a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts
index c81959616..56713ede8 100644
--- a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts
+++ b/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts
@@ -1,4 +1,4 @@
1import { Meter } from '@opentelemetry/api-metrics' 1import { Meter } from '@opentelemetry/api'
2import { JobQueue } from '@server/lib/job-queue' 2import { JobQueue } from '@server/lib/job-queue'
3 3
4export class JobQueueObserversBuilder { 4export class JobQueueObserversBuilder {
diff --git a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts
index e27bd8548..5effc18e1 100644
--- a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts
+++ b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts
@@ -1,4 +1,4 @@
1import { Meter } from '@opentelemetry/api-metrics' 1import { Meter } from '@opentelemetry/api'
2import { VideoModel } from '@server/models/video/video' 2import { VideoModel } from '@server/models/video/video'
3 3
4export class LivesObserversBuilder { 4export class LivesObserversBuilder {
diff --git a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts
index 781722108..b66fa474c 100644
--- a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts
+++ b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts
@@ -1,7 +1,7 @@
1import { readdir } from 'fs-extra' 1import { readdir } from 'fs-extra'
2import { constants, PerformanceObserver } from 'perf_hooks' 2import { constants, PerformanceObserver } from 'perf_hooks'
3import * as process from 'process' 3import * as process from 'process'
4import { Meter, ObservableResult } from '@opentelemetry/api-metrics' 4import { Meter, ObservableResult } from '@opentelemetry/api'
5import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics' 5import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics'
6import { View } from '@opentelemetry/sdk-metrics/build/src/view/View' 6import { View } from '@opentelemetry/sdk-metrics/build/src/view/View'
7import { logger } from '@server/helpers/logger' 7import { logger } from '@server/helpers/logger'
diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts
index 9dd481838..406789618 100644
--- a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts
+++ b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts
@@ -1,4 +1,4 @@
1import { Counter, Meter } from '@opentelemetry/api-metrics' 1import { Counter, Meter } from '@opentelemetry/api'
2import { MVideoImmutable } from '@server/types/models' 2import { MVideoImmutable } from '@server/types/models'
3import { PlaybackMetricCreate } from '@shared/models' 3import { PlaybackMetricCreate } from '@shared/models'
4 4
diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
index 3d28ffdd8..9f5f22e1b 100644
--- a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
+++ b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts
@@ -1,5 +1,5 @@
1import memoizee from 'memoizee' 1import memoizee from 'memoizee'
2import { Meter } from '@opentelemetry/api-metrics' 2import { Meter } from '@opentelemetry/api'
3import { MEMOIZE_TTL } from '@server/initializers/constants' 3import { MEMOIZE_TTL } from '@server/initializers/constants'
4import { buildAvailableActivities } from '@server/lib/activitypub/activity' 4import { buildAvailableActivities } from '@server/lib/activitypub/activity'
5import { StatsManager } from '@server/lib/stat-manager' 5import { StatsManager } from '@server/lib/stat-manager'
diff --git a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts
index 634e5bbc9..c65f8ddae 100644
--- a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts
+++ b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts
@@ -1,4 +1,4 @@
1import { Meter } from '@opentelemetry/api-metrics' 1import { Meter } from '@opentelemetry/api'
2import { VideoScope, ViewerScope } from '@server/lib/views/shared' 2import { VideoScope, ViewerScope } from '@server/lib/views/shared'
3import { VideoViewsManager } from '@server/lib/views/video-views-manager' 3import { VideoViewsManager } from '@server/lib/views/video-views-manager'
4 4
diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts
index fc1666604..226d514c0 100644
--- a/server/lib/opentelemetry/metrics.ts
+++ b/server/lib/opentelemetry/metrics.ts
@@ -1,5 +1,5 @@
1import { Application, Request, Response } from 'express' 1import { Application, Request, Response } from 'express'
2import { Meter, metrics } from '@opentelemetry/api-metrics' 2import { Meter, metrics } from '@opentelemetry/api'
3import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' 3import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
4import { MeterProvider } from '@opentelemetry/sdk-metrics' 4import { MeterProvider } from '@opentelemetry/sdk-metrics'
5import { logger } from '@server/helpers/logger' 5import { logger } from '@server/helpers/logger'
@@ -52,7 +52,10 @@ class OpenTelemetryMetrics {
52 ] 52 ]
53 }) 53 })
54 54
55 provider.addMetricReader(new PrometheusExporter({ port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT })) 55 provider.addMetricReader(new PrometheusExporter({
56 host: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.HOSTNAME,
57 port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT
58 }))
56 59
57 metrics.setGlobalMeterProvider(provider) 60 metrics.setGlobalMeterProvider(provider)
58 61
diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts
index b1c3bd173..9a81680b2 100644
--- a/server/lib/opentelemetry/tracing.ts
+++ b/server/lib/opentelemetry/tracing.ts
@@ -6,8 +6,8 @@ import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns'
6import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express' 6import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'
7import FsInstrumentation from '@opentelemetry/instrumentation-fs' 7import FsInstrumentation from '@opentelemetry/instrumentation-fs'
8import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' 8import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
9import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'
9import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' 10import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'
10import { RedisInstrumentation } from '@opentelemetry/instrumentation-redis-4'
11import { Resource } from '@opentelemetry/resources' 11import { Resource } from '@opentelemetry/resources'
12import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' 12import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
13import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' 13import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
@@ -58,7 +58,7 @@ function registerOpentelemetryTracing () {
58 new DnsInstrumentation(), 58 new DnsInstrumentation(),
59 new HttpInstrumentation(), 59 new HttpInstrumentation(),
60 new ExpressInstrumentation(), 60 new ExpressInstrumentation(),
61 new RedisInstrumentation({ 61 new IORedisInstrumentation({
62 dbStatementSerializer: function (cmdName, cmdArgs) { 62 dbStatementSerializer: function (cmdName, cmdArgs) {
63 return [ cmdName, ...cmdArgs ].join(' ') 63 return [ cmdName, ...cmdArgs ].join(' ')
64 } 64 }
diff --git a/server/lib/paths.ts b/server/lib/paths.ts
index b29854700..470970f55 100644
--- a/server/lib/paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,10 @@
1import { join } from 'path' 1import { join } from 'path'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, VIDEO_LIVE } from '@server/initializers/constants' 3import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants'
4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' 4import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
5import { removeFragmentedMP4Ext } from '@shared/core-utils' 5import { removeFragmentedMP4Ext } from '@shared/core-utils'
6import { buildUUID } from '@shared/extra-utils' 6import { buildUUID } from '@shared/extra-utils'
7import { isVideoInPrivateDirectory } from './video-privacy'
7 8
8// ################## Video file name ################## 9// ################## Video file name ##################
9 10
@@ -17,20 +18,24 @@ function generateHLSVideoFilename (resolution: number) {
17 18
18// ################## Streaming playlist ################## 19// ################## Streaming playlist ##################
19 20
20function getLiveDirectory (video: MVideoUUID) { 21function getLiveDirectory (video: MVideo) {
21 return getHLSDirectory(video) 22 return getHLSDirectory(video)
22} 23}
23 24
24function getLiveReplayBaseDirectory (video: MVideoUUID) { 25function getLiveReplayBaseDirectory (video: MVideo) {
25 return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) 26 return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY)
26} 27}
27 28
28function getHLSDirectory (video: MVideoUUID) { 29function getHLSDirectory (video: MVideo) {
29 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 30 if (isVideoInPrivateDirectory(video.privacy)) {
31 return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid)
32 }
33
34 return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid)
30} 35}
31 36
32function getHLSRedundancyDirectory (video: MVideoUUID) { 37function getHLSRedundancyDirectory (video: MVideoUUID) {
33 return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 38 return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
34} 39}
35 40
36function getHlsResolutionPlaylistFilename (videoFilename: string) { 41function getHlsResolutionPlaylistFilename (videoFilename: string) {
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts
index 4e799b3d4..7b1def6e3 100644
--- a/server/lib/plugins/plugin-helpers-builder.ts
+++ b/server/lib/plugins/plugin-helpers-builder.ts
@@ -1,4 +1,5 @@
1import express from 'express' 1import express from 'express'
2import { Server } from 'http'
2import { join } from 'path' 3import { join } from 'path'
3import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' 4import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils'
4import { buildLogger } from '@server/helpers/logger' 5import { buildLogger } from '@server/helpers/logger'
@@ -13,15 +14,16 @@ import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
13import { UserModel } from '@server/models/user/user' 14import { UserModel } from '@server/models/user/user'
14import { VideoModel } from '@server/models/video/video' 15import { VideoModel } from '@server/models/video/video'
15import { VideoBlacklistModel } from '@server/models/video/video-blacklist' 16import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
16import { MPlugin } from '@server/types/models' 17import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models'
17import { PeerTubeHelpers } from '@server/types/plugins' 18import { PeerTubeHelpers } from '@server/types/plugins'
18import { VideoBlacklistCreate, VideoStorage } from '@shared/models' 19import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
19import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' 20import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
21import { PeerTubeSocket } from '../peertube-socket'
20import { ServerConfigManager } from '../server-config-manager' 22import { ServerConfigManager } from '../server-config-manager'
21import { blacklistVideo, unblacklistVideo } from '../video-blacklist' 23import { blacklistVideo, unblacklistVideo } from '../video-blacklist'
22import { VideoPathManager } from '../video-path-manager' 24import { VideoPathManager } from '../video-path-manager'
23 25
24function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { 26function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers {
25 const logger = buildPluginLogger(npmName) 27 const logger = buildPluginLogger(npmName)
26 28
27 const database = buildDatabaseHelpers() 29 const database = buildDatabaseHelpers()
@@ -29,12 +31,14 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel
29 31
30 const config = buildConfigHelpers() 32 const config = buildConfigHelpers()
31 33
32 const server = buildServerHelpers() 34 const server = buildServerHelpers(httpServer)
33 35
34 const moderation = buildModerationHelpers() 36 const moderation = buildModerationHelpers()
35 37
36 const plugin = buildPluginRelatedHelpers(pluginModel, npmName) 38 const plugin = buildPluginRelatedHelpers(pluginModel, npmName)
37 39
40 const socket = buildSocketHelpers()
41
38 const user = buildUserHelpers() 42 const user = buildUserHelpers()
39 43
40 return { 44 return {
@@ -45,6 +49,7 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel
45 moderation, 49 moderation,
46 plugin, 50 plugin,
47 server, 51 server,
52 socket,
48 user 53 user
49 } 54 }
50} 55}
@@ -65,8 +70,10 @@ function buildDatabaseHelpers () {
65 } 70 }
66} 71}
67 72
68function buildServerHelpers () { 73function buildServerHelpers (httpServer: Server) {
69 return { 74 return {
75 getHTTPServer: () => httpServer,
76
70 getServerActor: () => getServerActor() 77 getServerActor: () => getServerActor()
71 } 78 }
72} 79}
@@ -214,10 +221,23 @@ function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) {
214 221
215 getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, 222 getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`,
216 223
224 getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`,
225
217 getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) 226 getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName)
218 } 227 }
219} 228}
220 229
230function buildSocketHelpers () {
231 return {
232 sendNotification: (userId: number, notification: UserNotificationModelForApi) => {
233 PeerTubeSocket.Instance.sendNotification(userId, notification)
234 },
235 sendVideoLiveNewState: (video: MVideo) => {
236 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
237 }
238 }
239}
240
221function buildUserHelpers () { 241function buildUserHelpers () {
222 return { 242 return {
223 loadById: (id: number) => { 243 loadById: (id: number) => {
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index a46b97fa4..c4d9b6574 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -1,6 +1,7 @@
1import express from 'express' 1import express from 'express'
2import { createReadStream, createWriteStream } from 'fs' 2import { createReadStream, createWriteStream } from 'fs'
3import { ensureDir, outputFile, readJSON } from 'fs-extra' 3import { ensureDir, outputFile, readJSON } from 'fs-extra'
4import { Server } from 'http'
4import { basename, join } from 'path' 5import { basename, join } from 'path'
5import { decachePlugin } from '@server/helpers/decache' 6import { decachePlugin } from '@server/helpers/decache'
6import { ApplicationModel } from '@server/models/application/application' 7import { ApplicationModel } from '@server/models/application/application'
@@ -67,9 +68,37 @@ export class PluginManager implements ServerHook {
67 private hooks: { [name: string]: HookInformationValue[] } = {} 68 private hooks: { [name: string]: HookInformationValue[] } = {}
68 private translations: PluginLocalesTranslations = {} 69 private translations: PluginLocalesTranslations = {}
69 70
71 private server: Server
72
70 private constructor () { 73 private constructor () {
71 } 74 }
72 75
76 init (server: Server) {
77 this.server = server
78 }
79
80 registerWebSocketRouter () {
81 this.server.on('upgrade', (request, socket, head) => {
82 const url = request.url
83
84 const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`)
85 if (!matched) return
86
87 const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN)
88 const subRoute = matched[3]
89
90 const result = this.getRegisteredPluginOrTheme(npmName)
91 if (!result) return
92
93 const routes = result.registerHelpers.getWebSocketRoutes()
94
95 const wss = routes.find(r => r.route.startsWith(subRoute))
96 if (!wss) return
97
98 wss.handler(request, socket, head)
99 })
100 }
101
73 // ###################### Getters ###################### 102 // ###################### Getters ######################
74 103
75 isRegistered (npmName: string) { 104 isRegistered (npmName: string) {
@@ -581,7 +610,7 @@ export class PluginManager implements ServerHook {
581 }) 610 })
582 } 611 }
583 612
584 const registerHelpers = new RegisterHelpers(npmName, plugin, onHookAdded.bind(this)) 613 const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this))
585 614
586 return { 615 return {
587 registerStore: registerHelpers, 616 registerStore: registerHelpers,
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts
index f4d405676..1aaef3606 100644
--- a/server/lib/plugins/register-helpers.ts
+++ b/server/lib/plugins/register-helpers.ts
@@ -1,4 +1,5 @@
1import express from 'express' 1import express from 'express'
2import { Server } from 'http'
2import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
3import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' 4import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth'
4import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' 5import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory'
@@ -8,7 +9,8 @@ import {
8 RegisterServerAuthExternalResult, 9 RegisterServerAuthExternalResult,
9 RegisterServerAuthPassOptions, 10 RegisterServerAuthPassOptions,
10 RegisterServerExternalAuthenticatedResult, 11 RegisterServerExternalAuthenticatedResult,
11 RegisterServerOptions 12 RegisterServerOptions,
13 RegisterServerWebSocketRouteOptions
12} from '@server/types/plugins' 14} from '@server/types/plugins'
13import { 15import {
14 EncoderOptionsBuilder, 16 EncoderOptionsBuilder,
@@ -49,12 +51,15 @@ export class RegisterHelpers {
49 51
50 private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] 52 private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = []
51 53
54 private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = []
55
52 private readonly router: express.Router 56 private readonly router: express.Router
53 private readonly videoConstantManagerFactory: VideoConstantManagerFactory 57 private readonly videoConstantManagerFactory: VideoConstantManagerFactory
54 58
55 constructor ( 59 constructor (
56 private readonly npmName: string, 60 private readonly npmName: string,
57 private readonly plugin: PluginModel, 61 private readonly plugin: PluginModel,
62 private readonly server: Server,
58 private readonly onHookAdded: (options: RegisterServerHookOptions) => void 63 private readonly onHookAdded: (options: RegisterServerHookOptions) => void
59 ) { 64 ) {
60 this.router = express.Router() 65 this.router = express.Router()
@@ -66,6 +71,7 @@ export class RegisterHelpers {
66 const registerSetting = this.buildRegisterSetting() 71 const registerSetting = this.buildRegisterSetting()
67 72
68 const getRouter = this.buildGetRouter() 73 const getRouter = this.buildGetRouter()
74 const registerWebSocketRoute = this.buildRegisterWebSocketRoute()
69 75
70 const settingsManager = this.buildSettingsManager() 76 const settingsManager = this.buildSettingsManager()
71 const storageManager = this.buildStorageManager() 77 const storageManager = this.buildStorageManager()
@@ -85,13 +91,14 @@ export class RegisterHelpers {
85 const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() 91 const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth()
86 const unregisterExternalAuth = this.buildUnregisterExternalAuth() 92 const unregisterExternalAuth = this.buildUnregisterExternalAuth()
87 93
88 const peertubeHelpers = buildPluginHelpers(this.plugin, this.npmName) 94 const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName)
89 95
90 return { 96 return {
91 registerHook, 97 registerHook,
92 registerSetting, 98 registerSetting,
93 99
94 getRouter, 100 getRouter,
101 registerWebSocketRoute,
95 102
96 settingsManager, 103 settingsManager,
97 storageManager, 104 storageManager,
@@ -180,10 +187,20 @@ export class RegisterHelpers {
180 return this.onSettingsChangeCallbacks 187 return this.onSettingsChangeCallbacks
181 } 188 }
182 189
190 getWebSocketRoutes () {
191 return this.webSocketRoutes
192 }
193
183 private buildGetRouter () { 194 private buildGetRouter () {
184 return () => this.router 195 return () => this.router
185 } 196 }
186 197
198 private buildRegisterWebSocketRoute () {
199 return (options: RegisterServerWebSocketRouteOptions) => {
200 this.webSocketRoutes.push(options)
201 }
202 }
203
187 private buildRegisterSetting () { 204 private buildRegisterSetting () {
188 return (options: RegisterServerSettingOptions) => { 205 return (options: RegisterServerSettingOptions) => {
189 this.settings.push(options) 206 this.settings.push(options)
diff --git a/server/lib/plugins/yarn.ts b/server/lib/plugins/yarn.ts
index d105b95e0..9cf6ec9e9 100644
--- a/server/lib/plugins/yarn.ts
+++ b/server/lib/plugins/yarn.ts
@@ -1,7 +1,7 @@
1import { outputJSON, pathExists } from 'fs-extra' 1import { outputJSON, pathExists } from 'fs-extra'
2import { join } from 'path' 2import { join } from 'path'
3import { execShell } from '../../helpers/core-utils' 3import { execShell } from '../../helpers/core-utils'
4import { isNpmPluginNameValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins' 4import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins'
5import { logger } from '../../helpers/logger' 5import { logger } from '../../helpers/logger'
6import { CONFIG } from '../../initializers/config' 6import { CONFIG } from '../../initializers/config'
7import { getLatestPluginVersion } from './plugin-index' 7import { getLatestPluginVersion } from './plugin-index'
@@ -69,5 +69,5 @@ function checkNpmPluginNameOrThrow (name: string) {
69} 69}
70 70
71function checkPluginVersionOrThrow (name: string) { 71function checkPluginVersionOrThrow (name: string) {
72 if (!isPluginVersionValid(name)) throw new Error('Invalid NPM plugin version to install') 72 if (!isPluginStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install')
73} 73}
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 9b3c72300..c0e9aece7 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -1,4 +1,4 @@
1import { createClient, RedisClientOptions, RedisModules } from 'redis' 1import IoRedis, { RedisOptions } from 'ioredis'
2import { exists } from '@server/helpers/custom-validators/misc' 2import { exists } from '@server/helpers/custom-validators/misc'
3import { sha256 } from '@shared/extra-utils' 3import { sha256 } from '@shared/extra-utils'
4import { logger } from '../helpers/logger' 4import { logger } from '../helpers/logger'
@@ -9,6 +9,7 @@ import {
9 CONTACT_FORM_LIFETIME, 9 CONTACT_FORM_LIFETIME,
10 RESUMABLE_UPLOAD_SESSION_LIFETIME, 10 RESUMABLE_UPLOAD_SESSION_LIFETIME,
11 TRACKER_RATE_LIMITS, 11 TRACKER_RATE_LIMITS,
12 TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
12 USER_EMAIL_VERIFY_LIFETIME, 13 USER_EMAIL_VERIFY_LIFETIME,
13 USER_PASSWORD_CREATE_LIFETIME, 14 USER_PASSWORD_CREATE_LIFETIME,
14 USER_PASSWORD_RESET_LIFETIME, 15 USER_PASSWORD_RESET_LIFETIME,
@@ -21,7 +22,7 @@ class Redis {
21 private static instance: Redis 22 private static instance: Redis
22 private initialized = false 23 private initialized = false
23 private connected = false 24 private connected = false
24 private client: ReturnType<typeof createClient> 25 private client: IoRedis
25 private prefix: string 26 private prefix: string
26 27
27 private constructor () { 28 private constructor () {
@@ -32,46 +33,42 @@ class Redis {
32 if (this.initialized === true) return 33 if (this.initialized === true) return
33 this.initialized = true 34 this.initialized = true
34 35
35 this.client = createClient(Redis.getRedisClientOptions())
36 this.client.on('error', err => logger.error('Redis Client Error', { err }))
37
38 logger.info('Connecting to redis...') 36 logger.info('Connecting to redis...')
39 37
40 this.client.connect() 38 this.client = new IoRedis(Redis.getRedisClientOptions('', { enableAutoPipelining: true }))
41 .then(() => { 39 this.client.on('error', err => logger.error('Redis failed to connect', { err }))
42 logger.info('Connected to redis.') 40 this.client.on('connect', () => {
43 41 logger.info('Connected to redis.')
44 this.connected = true 42
45 }).catch(err => { 43 this.connected = true
46 logger.error('Cannot connect to redis', { err }) 44 })
47 process.exit(-1) 45 this.client.on('reconnecting', (ms) => {
48 }) 46 logger.error(`Reconnecting to redis in ${ms}.`)
47 })
48 this.client.on('close', () => {
49 logger.error('Connection to redis has closed.')
50 this.connected = false
51 })
52
53 this.client.on('end', () => {
54 logger.error('Connection to redis has closed and no more reconnects will be done.')
55 })
49 56
50 this.prefix = 'redis-' + WEBSERVER.HOST + '-' 57 this.prefix = 'redis-' + WEBSERVER.HOST + '-'
51 } 58 }
52 59
53 static getRedisClientOptions () { 60 static getRedisClientOptions (connectionName?: string, options: RedisOptions = {}): RedisOptions {
54 let config: RedisClientOptions<RedisModules, {}> = { 61 return {
55 socket: { 62 connectionName: [ 'PeerTube', connectionName ].join(''),
56 connectTimeout: 20000 // Could be slow since node use sync call to compile PeerTube 63 connectTimeout: 20000, // Could be slow since node use sync call to compile PeerTube
57 } 64 password: CONFIG.REDIS.AUTH,
58 } 65 db: CONFIG.REDIS.DB,
59 66 host: CONFIG.REDIS.HOSTNAME,
60 if (CONFIG.REDIS.AUTH) { 67 port: CONFIG.REDIS.PORT,
61 config = { ...config, password: CONFIG.REDIS.AUTH } 68 path: CONFIG.REDIS.SOCKET,
62 } 69 showFriendlyErrorStack: true,
63 70 ...options
64 if (CONFIG.REDIS.DB) {
65 config = { ...config, database: CONFIG.REDIS.DB }
66 }
67
68 if (CONFIG.REDIS.HOSTNAME && CONFIG.REDIS.PORT) {
69 config.socket = { ...config.socket, host: CONFIG.REDIS.HOSTNAME, port: CONFIG.REDIS.PORT }
70 } else {
71 config.socket = { ...config.socket, path: CONFIG.REDIS.SOCKET }
72 } 71 }
73
74 return config
75 } 72 }
76 73
77 getClient () { 74 getClient () {
@@ -108,10 +105,24 @@ class Redis {
108 return this.removeValue(this.generateResetPasswordKey(userId)) 105 return this.removeValue(this.generateResetPasswordKey(userId))
109 } 106 }
110 107
111 async getResetPasswordLink (userId: number) { 108 async getResetPasswordVerificationString (userId: number) {
112 return this.getValue(this.generateResetPasswordKey(userId)) 109 return this.getValue(this.generateResetPasswordKey(userId))
113 } 110 }
114 111
112 /* ************ Two factor auth request ************ */
113
114 async setTwoFactorRequest (userId: number, otpSecret: string) {
115 const requestToken = await generateRandomString(32)
116
117 await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME)
118
119 return requestToken
120 }
121
122 async getTwoFactorRequestToken (userId: number, requestToken: string) {
123 return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken))
124 }
125
115 /* ************ Email verification ************ */ 126 /* ************ Email verification ************ */
116 127
117 async setVerifyEmailVerificationString (userId: number) { 128 async setVerifyEmailVerificationString (userId: number) {
@@ -342,6 +353,10 @@ class Redis {
342 return 'reset-password-' + userId 353 return 'reset-password-' + userId
343 } 354 }
344 355
356 private generateTwoFactorRequestKey (userId: number, token: string) {
357 return 'two-factor-request-' + userId + '-' + token
358 }
359
345 private generateVerifyEmailKey (userId: number) { 360 private generateVerifyEmailKey (userId: number) {
346 return 'verify-email-' + userId 361 return 'verify-email-' + userId
347 } 362 }
@@ -369,15 +384,15 @@ class Redis {
369 } 384 }
370 385
371 private getSet (key: string) { 386 private getSet (key: string) {
372 return this.client.sMembers(this.prefix + key) 387 return this.client.smembers(this.prefix + key)
373 } 388 }
374 389
375 private addToSet (key: string, value: string) { 390 private addToSet (key: string, value: string) {
376 return this.client.sAdd(this.prefix + key, value) 391 return this.client.sadd(this.prefix + key, value)
377 } 392 }
378 393
379 private deleteFromSet (key: string, value: string) { 394 private deleteFromSet (key: string, value: string) {
380 return this.client.sRem(this.prefix + key, value) 395 return this.client.srem(this.prefix + key, value)
381 } 396 }
382 397
383 private deleteKey (key: string) { 398 private deleteKey (key: string) {
@@ -391,16 +406,14 @@ class Redis {
391 return JSON.parse(value) 406 return JSON.parse(value)
392 } 407 }
393 408
394 private setObject (key: string, value: { [ id: string ]: number | string }) { 409 private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) {
395 return this.setValue(key, JSON.stringify(value)) 410 return this.setValue(key, JSON.stringify(value), expirationMilliseconds)
396 } 411 }
397 412
398 private async setValue (key: string, value: string, expirationMilliseconds?: number) { 413 private async setValue (key: string, value: string, expirationMilliseconds?: number) {
399 const options = expirationMilliseconds 414 const result = expirationMilliseconds !== undefined
400 ? { PX: expirationMilliseconds } 415 ? await this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds)
401 : {} 416 : await this.client.set(this.prefix + key, value)
402
403 const result = await this.client.set(this.prefix + key, value, options)
404 417
405 if (result !== 'OK') throw new Error('Redis set result is not OK.') 418 if (result !== 'OK') throw new Error('Redis set result is not OK.')
406 } 419 }
diff --git a/server/lib/schedulers/plugins-check-scheduler.ts b/server/lib/schedulers/plugins-check-scheduler.ts
index 06450fa01..820c01693 100644
--- a/server/lib/schedulers/plugins-check-scheduler.ts
+++ b/server/lib/schedulers/plugins-check-scheduler.ts
@@ -33,7 +33,7 @@ export class PluginsCheckScheduler extends AbstractScheduler {
33 const chunks = chunk(plugins, 10) 33 const chunks = chunk(plugins, 10)
34 for (const chunk of chunks) { 34 for (const chunk of chunks) {
35 // Find plugins according to their npm name 35 // Find plugins according to their npm name
36 const pluginIndex: { [npmName: string]: PluginModel} = {} 36 const pluginIndex: { [npmName: string]: PluginModel } = {}
37 for (const plugin of chunk) { 37 for (const plugin of chunk) {
38 pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin 38 pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin
39 } 39 }
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 5bfbc3cd2..e38685c04 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -1,11 +1,14 @@
1import { VideoModel } from '@server/models/video/video' 1import { VideoModel } from '@server/models/video/video'
2import { MVideoFullLight } from '@server/types/models' 2import { MScheduleVideoUpdate } from '@server/types/models'
3import { VideoPrivacy, VideoState } from '@shared/models'
3import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
4import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
5import { sequelizeTypescript } from '../../initializers/database' 6import { sequelizeTypescript } from '../../initializers/database'
6import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' 7import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
7import { federateVideoIfNeeded } from '../activitypub/videos'
8import { Notifier } from '../notifier' 8import { Notifier } from '../notifier'
9import { addVideoJobsAfterUpdate } from '../video'
10import { VideoPathManager } from '../video-path-manager'
11import { setVideoPrivacy } from '../video-privacy'
9import { AbstractScheduler } from './abstract-scheduler' 12import { AbstractScheduler } from './abstract-scheduler'
10 13
11export class UpdateVideosScheduler extends AbstractScheduler { 14export class UpdateVideosScheduler extends AbstractScheduler {
@@ -26,35 +29,58 @@ export class UpdateVideosScheduler extends AbstractScheduler {
26 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined 29 if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
27 30
28 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() 31 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
29 const publishedVideos: MVideoFullLight[] = []
30 32
31 for (const schedule of schedules) { 33 for (const schedule of schedules) {
32 await sequelizeTypescript.transaction(async t => { 34 const videoOnly = await VideoModel.load(schedule.videoId)
33 const video = await VideoModel.loadFull(schedule.videoId, t) 35 const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
34 36
35 logger.info('Executing scheduled video update on %s.', video.uuid) 37 try {
38 const { video, published } = await this.updateAVideo(schedule)
36 39
37 if (schedule.privacy) { 40 if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
38 const wasConfidentialVideo = video.isConfidential() 41 } catch (err) {
39 const isNewVideo = video.isNewVideo(schedule.privacy) 42 logger.error('Cannot update video', { err })
43 }
40 44
41 video.setPrivacy(schedule.privacy) 45 mutexReleaser()
42 await video.save({ transaction: t }) 46 }
43 await federateVideoIfNeeded(video, isNewVideo, t) 47 }
48
49 private async updateAVideo (schedule: MScheduleVideoUpdate) {
50 let oldPrivacy: VideoPrivacy
51 let isNewVideo: boolean
52 let published = false
53
54 const video = await sequelizeTypescript.transaction(async t => {
55 const video = await VideoModel.loadFull(schedule.videoId, t)
56 if (video.state === VideoState.TO_TRANSCODE) return null
57
58 logger.info('Executing scheduled video update on %s.', video.uuid)
59
60 if (schedule.privacy) {
61 isNewVideo = video.isNewVideo(schedule.privacy)
62 oldPrivacy = video.privacy
44 63
45 if (wasConfidentialVideo) { 64 setVideoPrivacy(video, schedule.privacy)
46 publishedVideos.push(video) 65 await video.save({ transaction: t })
47 } 66
67 if (oldPrivacy === VideoPrivacy.PRIVATE) {
68 published = true
48 } 69 }
70 }
49 71
50 await schedule.destroy({ transaction: t }) 72 await schedule.destroy({ transaction: t })
51 }) 73
52 } 74 return video
75 })
53 76
54 for (const v of publishedVideos) { 77 if (!video) {
55 Notifier.Instance.notifyOnNewVideoIfNeeded(v) 78 return { video, published: false }
56 Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v)
57 } 79 }
80
81 await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false })
82
83 return { video, published }
58 } 84 }
59 85
60 static get Instance () { 86 static get Instance () {
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
index a527f68b5..efb957fac 100644
--- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { VideoChannelModel } from '@server/models/video/video-channel' 3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
5import { VideoChannelSyncState } from '@shared/models'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { synchronizeChannel } from '../sync-channel' 6import { synchronizeChannel } from '../sync-channel'
8import { AbstractScheduler } from './abstract-scheduler' 7import { AbstractScheduler } from './abstract-scheduler'
@@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
28 for (const sync of channelSyncs) { 27 for (const sync of channelSyncs) {
29 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) 28 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
30 29
31 try { 30 logger.info(
32 logger.info( 31 'Creating video import jobs for "%s" sync with external channel "%s"',
33 'Creating video import jobs for "%s" sync with external channel "%s"', 32 channel.Actor.preferredUsername, sync.externalChannelUrl
34 channel.Actor.preferredUsername, sync.externalChannelUrl 33 )
35 ) 34
36 35 const onlyAfter = sync.lastSyncAt || sync.createdAt
37 const onlyAfter = sync.lastSyncAt || sync.createdAt 36
38 37 await synchronizeChannel({
39 await synchronizeChannel({ 38 channel,
40 channel, 39 externalChannelUrl: sync.externalChannelUrl,
41 externalChannelUrl: sync.externalChannelUrl, 40 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
42 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, 41 channelSync: sync,
43 channelSync: sync, 42 onlyAfter
44 onlyAfter 43 })
45 })
46 } catch (err) {
47 logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
48 sync.state = VideoChannelSyncState.FAILED
49 await sync.save()
50 }
51 } 44 }
52 } 45 }
53 46
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 91c217615..dc450c338 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -16,7 +16,7 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
16import { logger, loggerTagsFactory } from '../../helpers/logger' 16import { logger, loggerTagsFactory } from '../../helpers/logger'
17import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 17import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
18import { CONFIG } from '../../initializers/config' 18import { CONFIG } from '../../initializers/config'
19import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' 19import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 20import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
21import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 21import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
22import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' 22import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
@@ -115,16 +115,29 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
115 for (const redundancyModel of expired) { 115 for (const redundancyModel of expired) {
116 try { 116 try {
117 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 117 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
118
119 // If the admin disabled the redundancy, remove this redundancy instead of extending it
120 if (!redundancyConfig) {
121 logger.info(
122 'Destroying redundancy %s because the redundancy %s does not exist anymore.',
123 redundancyModel.url, redundancyModel.strategy
124 )
125
126 await removeVideoRedundancy(redundancyModel)
127 continue
128 }
129
118 const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) 130 const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy)
119 131
120 // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it 132 // If the admin decreased the cache size, remove this redundancy instead of extending it
121 if (!redundancyConfig || totalUsed > redundancyConfig.size) { 133 if (totalUsed > redundancyConfig.size) {
122 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) 134 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
123 135
124 await removeVideoRedundancy(redundancyModel) 136 await removeVideoRedundancy(redundancyModel)
125 } else { 137 continue
126 await this.extendsRedundancy(redundancyModel)
127 } 138 }
139
140 await this.extendsRedundancy(redundancyModel)
128 } catch (err) { 141 } catch (err) {
129 logger.error( 142 logger.error(
130 'Cannot extend or remove expiration of %s video from our redundancy system.', 143 'Cannot extend or remove expiration of %s video from our redundancy system.',
@@ -262,7 +275,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
262 275
263 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) 276 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
264 277
265 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 278 const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
266 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) 279 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
267 280
268 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 281 const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index f91599c14..10167ee38 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -1,7 +1,7 @@
1import { logger } from '@server/helpers/logger' 1import { logger } from '@server/helpers/logger'
2import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' 2import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { buildYoutubeDLImport } from '@server/lib/video-import' 4import { buildYoutubeDLImport } from '@server/lib/video-pre-import'
5import { UserModel } from '@server/models/user/user' 5import { UserModel } from '@server/models/user/user'
6import { VideoImportModel } from '@server/models/video/video-import' 6import { VideoImportModel } from '@server/models/video/video-import'
7import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' 7import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
@@ -12,8 +12,8 @@ import { ServerConfigManager } from './server-config-manager'
12export async function synchronizeChannel (options: { 12export async function synchronizeChannel (options: {
13 channel: MChannelAccountDefault 13 channel: MChannelAccountDefault
14 externalChannelUrl: string 14 externalChannelUrl: string
15 videosCountLimit: number
15 channelSync?: MChannelSync 16 channelSync?: MChannelSync
16 videosCountLimit?: number
17 onlyAfter?: Date 17 onlyAfter?: Date
18}) { 18}) {
19 const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options 19 const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
@@ -24,56 +24,62 @@ export async function synchronizeChannel (options: {
24 await channelSync.save() 24 await channelSync.save()
25 } 25 }
26 26
27 const user = await UserModel.loadByChannelActorId(channel.actorId) 27 try {
28 const youtubeDL = new YoutubeDLWrapper( 28 const user = await UserModel.loadByChannelActorId(channel.actorId)
29 externalChannelUrl, 29 const youtubeDL = new YoutubeDLWrapper(
30 ServerConfigManager.Instance.getEnabledResolutions('vod'), 30 externalChannelUrl,
31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 31 ServerConfigManager.Instance.getEnabledResolutions('vod'),
32 ) 32 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
33 )
33 34
34 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) 35 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
35 36
36 logger.info( 37 logger.info(
37 'Fetched %d candidate URLs for sync channel %s.', 38 'Fetched %d candidate URLs for sync channel %s.',
38 targetUrls.length, channel.Actor.preferredUsername, { targetUrls } 39 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
39 ) 40 )
40 41
41 if (targetUrls.length === 0) { 42 if (targetUrls.length === 0) {
42 if (channelSync) { 43 if (channelSync) {
43 channelSync.state = VideoChannelSyncState.SYNCED 44 channelSync.state = VideoChannelSyncState.SYNCED
44 await channelSync.save() 45 await channelSync.save()
46 }
47
48 return
45 } 49 }
46 50
47 return 51 const children: CreateJobArgument[] = []
48 }
49 52
50 const children: CreateJobArgument[] = [] 53 for (const targetUrl of targetUrls) {
54 if (await skipImport(channel, targetUrl, onlyAfter)) continue
51 55
52 for (const targetUrl of targetUrls) { 56 const { job } = await buildYoutubeDLImport({
53 if (await skipImport(channel, targetUrl, onlyAfter)) continue 57 user,
58 channel,
59 targetUrl,
60 channelSync,
61 importDataOverride: {
62 privacy: VideoPrivacy.PUBLIC
63 }
64 })
54 65
55 const { job } = await buildYoutubeDLImport({ 66 children.push(job)
56 user, 67 }
57 channel,
58 targetUrl,
59 channelSync,
60 importDataOverride: {
61 privacy: VideoPrivacy.PUBLIC
62 }
63 })
64
65 children.push(job)
66 }
67 68
68 // Will update the channel sync status 69 // Will update the channel sync status
69 const parent: CreateJobArgument = { 70 const parent: CreateJobArgument = {
70 type: 'after-video-channel-import', 71 type: 'after-video-channel-import',
71 payload: { 72 payload: {
72 channelSyncId: channelSync?.id 73 channelSyncId: channelSync?.id
74 }
73 } 75 }
74 }
75 76
76 await JobQueue.Instance.createJobWithChildren(parent, children) 77 await JobQueue.Instance.createJobWithChildren(parent, children)
78 } catch (err) {
79 logger.error(`Failed to import channel ${channel.name}`, { err })
80 channelSync.state = VideoChannelSyncState.FAILED
81 await channelSync.save()
82 }
77} 83}
78 84
79// --------------------------------------------------------------------------- 85// ---------------------------------------------------------------------------
diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts
index 44e26754d..c7b61e9ba 100644
--- a/server/lib/transcoding/transcoding.ts
+++ b/server/lib/transcoding/transcoding.ts
@@ -1,3 +1,4 @@
1import { MutexInterface } from 'async-mutex'
1import { Job } from 'bullmq' 2import { Job } from 'bullmq'
2import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' 3import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
3import { basename, extname as extnameUtil, join } from 'path' 4import { basename, extname as extnameUtil, join } from 'path'
@@ -6,11 +7,13 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils'
6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 7import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
7import { sequelizeTypescript } from '@server/initializers/database' 8import { sequelizeTypescript } from '@server/initializers/database'
8import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 9import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
10import { pick } from '@shared/core-utils'
9import { VideoResolution, VideoStorage } from '../../../shared/models/videos' 11import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
10import { 12import {
11 buildFileMetadata, 13 buildFileMetadata,
12 canDoQuickTranscode, 14 canDoQuickTranscode,
13 computeResolutionsToTranscode, 15 computeResolutionsToTranscode,
16 ffprobePromise,
14 getVideoStreamDuration, 17 getVideoStreamDuration,
15 getVideoStreamFPS, 18 getVideoStreamFPS,
16 transcodeVOD, 19 transcodeVOD,
@@ -33,7 +36,7 @@ import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
33 */ 36 */
34 37
35// Optimize the original video file and replace it. The resolution is not changed. 38// Optimize the original video file and replace it. The resolution is not changed.
36function optimizeOriginalVideofile (options: { 39async function optimizeOriginalVideofile (options: {
37 video: MVideoFullLight 40 video: MVideoFullLight
38 inputVideoFile: MVideoFile 41 inputVideoFile: MVideoFile
39 job: Job 42 job: Job
@@ -43,49 +46,62 @@ function optimizeOriginalVideofile (options: {
43 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 46 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
44 const newExtname = '.mp4' 47 const newExtname = '.mp4'
45 48
46 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => { 49 // Will be released by our transcodeVOD function once ffmpeg is ran
47 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 50 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
48 51
49 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) 52 try {
50 ? 'quick-transcode' 53 await video.reload()
51 : 'video'
52 54
53 const resolution = buildOriginalFileResolution(inputVideoFile.resolution) 55 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
54 56
55 const transcodeOptions: TranscodeVODOptions = { 57 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
56 type: transcodeType, 58 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
57 59
58 inputPath: videoInputPath, 60 const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
59 outputPath: videoTranscodedPath, 61 ? 'quick-transcode'
62 : 'video'
60 63
61 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 64 const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
62 profile: CONFIG.TRANSCODING.PROFILE,
63 65
64 resolution, 66 const transcodeOptions: TranscodeVODOptions = {
67 type: transcodeType,
65 68
66 job 69 inputPath: videoInputPath,
67 } 70 outputPath: videoTranscodedPath,
68 71
69 // Could be very long! 72 inputFileMutexReleaser,
70 await transcodeVOD(transcodeOptions)
71 73
72 // Important to do this before getVideoFilename() to take in account the new filename 74 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
73 inputVideoFile.resolution = resolution 75 profile: CONFIG.TRANSCODING.PROFILE,
74 inputVideoFile.extname = newExtname
75 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
76 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
77 76
78 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 77 resolution,
79 78
80 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 79 job
81 await remove(videoInputPath) 80 }
82 81
83 return { transcodeType, videoFile } 82 // Could be very long!
84 }) 83 await transcodeVOD(transcodeOptions)
84
85 // Important to do this before getVideoFilename() to take in account the new filename
86 inputVideoFile.resolution = resolution
87 inputVideoFile.extname = newExtname
88 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
89 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
90
91 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
92 await remove(videoInputPath)
93
94 return { transcodeType, videoFile }
95 })
96
97 return result
98 } finally {
99 inputFileMutexReleaser()
100 }
85} 101}
86 102
87// Transcode the original video file to a lower resolution compatible with WebTorrent 103// Transcode the original video file to a lower resolution compatible with WebTorrent
88function transcodeNewWebTorrentResolution (options: { 104async function transcodeNewWebTorrentResolution (options: {
89 video: MVideoFullLight 105 video: MVideoFullLight
90 resolution: VideoResolution 106 resolution: VideoResolution
91 job: Job 107 job: Job
@@ -95,53 +111,68 @@ function transcodeNewWebTorrentResolution (options: {
95 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 111 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
96 const newExtname = '.mp4' 112 const newExtname = '.mp4'
97 113
98 return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => { 114 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
99 const newVideoFile = new VideoFileModel({
100 resolution,
101 extname: newExtname,
102 filename: generateWebTorrentVideoFilename(resolution, newExtname),
103 size: 0,
104 videoId: video.id
105 })
106 115
107 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) 116 try {
108 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 117 await video.reload()
109 118
110 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 119 const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
111 ? {
112 type: 'only-audio' as 'only-audio',
113 120
114 inputPath: videoInputPath, 121 const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
115 outputPath: videoTranscodedPath, 122 const newVideoFile = new VideoFileModel({
123 resolution,
124 extname: newExtname,
125 filename: generateWebTorrentVideoFilename(resolution, newExtname),
126 size: 0,
127 videoId: video.id
128 })
116 129
117 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 130 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
118 profile: CONFIG.TRANSCODING.PROFILE,
119 131
120 resolution, 132 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
133 ? {
134 type: 'only-audio' as 'only-audio',
121 135
122 job 136 inputPath: videoInputPath,
123 } 137 outputPath: videoTranscodedPath,
124 : {
125 type: 'video' as 'video',
126 inputPath: videoInputPath,
127 outputPath: videoTranscodedPath,
128 138
129 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 139 inputFileMutexReleaser,
130 profile: CONFIG.TRANSCODING.PROFILE,
131 140
132 resolution, 141 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
142 profile: CONFIG.TRANSCODING.PROFILE,
133 143
134 job 144 resolution,
135 }
136 145
137 await transcodeVOD(transcodeOptions) 146 job
147 }
148 : {
149 type: 'video' as 'video',
150 inputPath: videoInputPath,
151 outputPath: videoTranscodedPath,
138 152
139 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 153 inputFileMutexReleaser,
140 }) 154
155 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
156 profile: CONFIG.TRANSCODING.PROFILE,
157
158 resolution,
159
160 job
161 }
162
163 await transcodeVOD(transcodeOptions)
164
165 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
166 })
167
168 return result
169 } finally {
170 inputFileMutexReleaser()
171 }
141} 172}
142 173
143// Merge an image with an audio file to create a video 174// Merge an image with an audio file to create a video
144function mergeAudioVideofile (options: { 175async function mergeAudioVideofile (options: {
145 video: MVideoFullLight 176 video: MVideoFullLight
146 resolution: VideoResolution 177 resolution: VideoResolution
147 job: Job 178 job: Job
@@ -151,54 +182,67 @@ function mergeAudioVideofile (options: {
151 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 182 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
152 const newExtname = '.mp4' 183 const newExtname = '.mp4'
153 184
154 const inputVideoFile = video.getMinQualityFile() 185 const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
155 186
156 return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => { 187 try {
157 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 188 await video.reload()
158 189
159 // If the user updates the video preview during transcoding 190 const inputVideoFile = video.getMinQualityFile()
160 const previewPath = video.getPreview().getPath()
161 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
162 await copyFile(previewPath, tmpPreviewPath)
163 191
164 const transcodeOptions = { 192 const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
165 type: 'merge-audio' as 'merge-audio',
166 193
167 inputPath: tmpPreviewPath, 194 const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
168 outputPath: videoTranscodedPath, 195 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
169 196
170 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 197 // If the user updates the video preview during transcoding
171 profile: CONFIG.TRANSCODING.PROFILE, 198 const previewPath = video.getPreview().getPath()
199 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
200 await copyFile(previewPath, tmpPreviewPath)
172 201
173 audioPath: audioInputPath, 202 const transcodeOptions = {
174 resolution, 203 type: 'merge-audio' as 'merge-audio',
175 204
176 job 205 inputPath: tmpPreviewPath,
177 } 206 outputPath: videoTranscodedPath,
178 207
179 try { 208 inputFileMutexReleaser,
180 await transcodeVOD(transcodeOptions)
181 209
182 await remove(audioInputPath) 210 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
183 await remove(tmpPreviewPath) 211 profile: CONFIG.TRANSCODING.PROFILE,
184 } catch (err) {
185 await remove(tmpPreviewPath)
186 throw err
187 }
188 212
189 // Important to do this before getVideoFilename() to take in account the new file extension 213 audioPath: audioInputPath,
190 inputVideoFile.extname = newExtname 214 resolution,
191 inputVideoFile.resolution = resolution
192 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
193 215
194 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) 216 job
195 // ffmpeg generated a new video file, so update the video duration 217 }
196 // See https://trac.ffmpeg.org/ticket/5456
197 video.duration = await getVideoStreamDuration(videoTranscodedPath)
198 await video.save()
199 218
200 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 219 try {
201 }) 220 await transcodeVOD(transcodeOptions)
221
222 await remove(audioInputPath)
223 await remove(tmpPreviewPath)
224 } catch (err) {
225 await remove(tmpPreviewPath)
226 throw err
227 }
228
229 // Important to do this before getVideoFilename() to take in account the new file extension
230 inputVideoFile.extname = newExtname
231 inputVideoFile.resolution = resolution
232 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
233
234 // ffmpeg generated a new video file, so update the video duration
235 // See https://trac.ffmpeg.org/ticket/5456
236 video.duration = await getVideoStreamDuration(videoTranscodedPath)
237 await video.save()
238
239 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
240 })
241
242 return result
243 } finally {
244 inputFileMutexReleaser()
245 }
202} 246}
203 247
204// Concat TS segments from a live video to a fragmented mp4 HLS playlist 248// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -207,13 +251,13 @@ async function generateHlsPlaylistResolutionFromTS (options: {
207 concatenatedTsFilePath: string 251 concatenatedTsFilePath: string
208 resolution: VideoResolution 252 resolution: VideoResolution
209 isAAC: boolean 253 isAAC: boolean
254 inputFileMutexReleaser: MutexInterface.Releaser
210}) { 255}) {
211 return generateHlsPlaylistCommon({ 256 return generateHlsPlaylistCommon({
212 video: options.video,
213 resolution: options.resolution,
214 inputPath: options.concatenatedTsFilePath,
215 type: 'hls-from-ts' as 'hls-from-ts', 257 type: 'hls-from-ts' as 'hls-from-ts',
216 isAAC: options.isAAC 258 inputPath: options.concatenatedTsFilePath,
259
260 ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
217 }) 261 })
218} 262}
219 263
@@ -223,15 +267,14 @@ function generateHlsPlaylistResolution (options: {
223 videoInputPath: string 267 videoInputPath: string
224 resolution: VideoResolution 268 resolution: VideoResolution
225 copyCodecs: boolean 269 copyCodecs: boolean
270 inputFileMutexReleaser: MutexInterface.Releaser
226 job?: Job 271 job?: Job
227}) { 272}) {
228 return generateHlsPlaylistCommon({ 273 return generateHlsPlaylistCommon({
229 video: options.video,
230 resolution: options.resolution,
231 copyCodecs: options.copyCodecs,
232 inputPath: options.videoInputPath,
233 type: 'hls' as 'hls', 274 type: 'hls' as 'hls',
234 job: options.job 275 inputPath: options.videoInputPath,
276
277 ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
235 }) 278 })
236} 279}
237 280
@@ -251,27 +294,39 @@ async function onWebTorrentVideoFileTranscoding (
251 video: MVideoFullLight, 294 video: MVideoFullLight,
252 videoFile: MVideoFile, 295 videoFile: MVideoFile,
253 transcodingPath: string, 296 transcodingPath: string,
254 outputPath: string 297 newVideoFile: MVideoFile
255) { 298) {
256 const stats = await stat(transcodingPath) 299 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
257 const fps = await getVideoStreamFPS(transcodingPath)
258 const metadata = await buildFileMetadata(transcodingPath)
259 300
260 await move(transcodingPath, outputPath, { overwrite: true }) 301 try {
302 await video.reload()
261 303
262 videoFile.size = stats.size 304 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
263 videoFile.fps = fps
264 videoFile.metadata = metadata
265 305
266 await createTorrentAndSetInfoHash(video, videoFile) 306 const stats = await stat(transcodingPath)
267 307
268 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) 308 const probe = await ffprobePromise(transcodingPath)
269 if (oldFile) await video.removeWebTorrentFile(oldFile) 309 const fps = await getVideoStreamFPS(transcodingPath, probe)
310 const metadata = await buildFileMetadata(transcodingPath, probe)
270 311
271 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 312 await move(transcodingPath, outputPath, { overwrite: true })
272 video.VideoFiles = await video.$get('VideoFiles')
273 313
274 return { video, videoFile } 314 videoFile.size = stats.size
315 videoFile.fps = fps
316 videoFile.metadata = metadata
317
318 await createTorrentAndSetInfoHash(video, videoFile)
319
320 const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
321 if (oldFile) await video.removeWebTorrentFile(oldFile)
322
323 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
324 video.VideoFiles = await video.$get('VideoFiles')
325
326 return { video, videoFile }
327 } finally {
328 mutexReleaser()
329 }
275} 330}
276 331
277async function generateHlsPlaylistCommon (options: { 332async function generateHlsPlaylistCommon (options: {
@@ -279,12 +334,15 @@ async function generateHlsPlaylistCommon (options: {
279 video: MVideo 334 video: MVideo
280 inputPath: string 335 inputPath: string
281 resolution: VideoResolution 336 resolution: VideoResolution
337
338 inputFileMutexReleaser: MutexInterface.Releaser
339
282 copyCodecs?: boolean 340 copyCodecs?: boolean
283 isAAC?: boolean 341 isAAC?: boolean
284 342
285 job?: Job 343 job?: Job
286}) { 344}) {
287 const { type, video, inputPath, resolution, copyCodecs, isAAC, job } = options 345 const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
288 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 346 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
289 347
290 const videoTranscodedBasePath = join(transcodeDirectory, type) 348 const videoTranscodedBasePath = join(transcodeDirectory, type)
@@ -308,6 +366,8 @@ async function generateHlsPlaylistCommon (options: {
308 366
309 isAAC, 367 isAAC,
310 368
369 inputFileMutexReleaser,
370
311 hlsPlaylist: { 371 hlsPlaylist: {
312 videoFilename 372 videoFilename
313 }, 373 },
@@ -333,47 +393,73 @@ async function generateHlsPlaylistCommon (options: {
333 videoStreamingPlaylistId: playlist.id 393 videoStreamingPlaylistId: playlist.id
334 }) 394 })
335 395
336 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) 396 const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
337 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
338 397
339 // Move playlist file 398 try {
340 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) 399 // VOD transcoding is a long task, refresh video attributes
341 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 400 await video.reload()
342 // Move video file
343 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
344 401
345 // Update video duration if it was not set (in case of a live for example) 402 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
346 if (!video.duration) { 403 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
347 video.duration = await getVideoStreamDuration(videoFilePath)
348 await video.save()
349 }
350 404
351 const stats = await stat(videoFilePath) 405 // Move playlist file
406 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
407 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
408 // Move video file
409 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
352 410
353 newVideoFile.size = stats.size 411 // Update video duration if it was not set (in case of a live for example)
354 newVideoFile.fps = await getVideoStreamFPS(videoFilePath) 412 if (!video.duration) {
355 newVideoFile.metadata = await buildFileMetadata(videoFilePath) 413 video.duration = await getVideoStreamDuration(videoFilePath)
414 await video.save()
415 }
356 416
357 await createTorrentAndSetInfoHash(playlist, newVideoFile) 417 const stats = await stat(videoFilePath)
358 418
359 const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) 419 newVideoFile.size = stats.size
360 if (oldFile) { 420 newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
361 await video.removeStreamingPlaylistVideoFile(playlist, oldFile) 421 newVideoFile.metadata = await buildFileMetadata(videoFilePath)
362 await oldFile.destroy()
363 }
364 422
365 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 423 await createTorrentAndSetInfoHash(playlist, newVideoFile)
366 424
367 await updatePlaylistAfterFileChange(video, playlist) 425 const oldFile = await VideoFileModel.loadHLSFile({
426 playlistId: playlist.id,
427 fps: newVideoFile.fps,
428 resolution: newVideoFile.resolution
429 })
430
431 if (oldFile) {
432 await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
433 await oldFile.destroy()
434 }
435
436 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
437
438 await updatePlaylistAfterFileChange(video, playlist)
368 439
369 return { resolutionPlaylistPath, videoFile: savedVideoFile } 440 return { resolutionPlaylistPath, videoFile: savedVideoFile }
441 } finally {
442 mutexReleaser()
443 }
370} 444}
371 445
372function buildOriginalFileResolution (inputResolution: number) { 446function buildOriginalFileResolution (inputResolution: number) {
373 if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution) 447 if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) {
448 return toEven(inputResolution)
449 }
374 450
375 const resolutions = computeResolutionsToTranscode({ input: inputResolution, type: 'vod', includeInput: false, strictLower: false }) 451 const resolutions = computeResolutionsToTranscode({
376 if (resolutions.length === 0) return toEven(inputResolution) 452 input: inputResolution,
453 type: 'vod',
454 includeInput: false,
455 strictLower: false,
456 // We don't really care about the audio resolution in this context
457 hasAudio: true
458 })
459
460 if (resolutions.length === 0) {
461 return toEven(inputResolution)
462 }
377 463
378 return Math.max(...resolutions) 464 return Math.max(...resolutions)
379} 465}
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts
index 9484eff75..58040cb6d 100644
--- a/server/lib/uploadx.ts
+++ b/server/lib/uploadx.ts
@@ -1,6 +1,10 @@
1import express from 'express' 1import express from 'express'
2import { buildLogger } from '@server/helpers/logger'
2import { getResumableUploadPath } from '@server/helpers/upload' 3import { getResumableUploadPath } from '@server/helpers/upload'
3import { Uploadx } from '@uploadx/core' 4import { CONFIG } from '@server/initializers/config'
5import { LogLevel, Uploadx } from '@uploadx/core'
6
7const logger = buildLogger('uploadx')
4 8
5const uploadx = new Uploadx({ 9const uploadx = new Uploadx({
6 directory: getResumableUploadPath(), 10 directory: getResumableUploadPath(),
@@ -10,6 +14,14 @@ const uploadx = new Uploadx({
10 // Could be big with thumbnails/previews 14 // Could be big with thumbnails/previews
11 maxMetadataSize: '10MB', 15 maxMetadataSize: '10MB',
12 16
17 logger: {
18 logLevel: CONFIG.LOG.LEVEL as LogLevel,
19 debug: logger.debug.bind(logger),
20 info: logger.info.bind(logger),
21 warn: logger.warn.bind(logger),
22 error: logger.error.bind(logger)
23 },
24
13 userIdentifier: (_, res: express.Response) => { 25 userIdentifier: (_, res: express.Response) => {
14 if (!res.locals.oauth) return undefined 26 if (!res.locals.oauth) return undefined
15 27
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
index c3f55fd95..9953cae5d 100644
--- a/server/lib/video-path-manager.ts
+++ b/server/lib/video-path-manager.ts
@@ -1,29 +1,31 @@
1import { Mutex } from 'async-mutex'
1import { remove } from 'fs-extra' 2import { remove } from 'fs-extra'
2import { extname, join } from 'path' 3import { extname, join } from 'path'
4import { logger, loggerTagsFactory } from '@server/helpers/logger'
3import { extractVideo } from '@server/helpers/video' 5import { extractVideo } from '@server/helpers/video'
4import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
5import { 7import { DIRECTORIES } from '@server/initializers/constants'
6 MStreamingPlaylistVideo, 8import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models'
7 MVideo,
8 MVideoFile,
9 MVideoFileStreamingPlaylistVideo,
10 MVideoFileVideo,
11 MVideoUUID
12} from '@server/types/models'
13import { buildUUID } from '@shared/extra-utils' 9import { buildUUID } from '@shared/extra-utils'
14import { VideoStorage } from '@shared/models' 10import { VideoStorage } from '@shared/models'
15import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' 11import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
16import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' 12import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
13import { isVideoInPrivateDirectory } from './video-privacy'
17 14
18type MakeAvailableCB <T> = (path: string) => Promise<T> | T 15type MakeAvailableCB <T> = (path: string) => Promise<T> | T
19 16
17const lTags = loggerTagsFactory('video-path-manager')
18
20class VideoPathManager { 19class VideoPathManager {
21 20
22 private static instance: VideoPathManager 21 private static instance: VideoPathManager
23 22
23 // Key is a video UUID
24 private readonly videoFileMutexStore = new Map<string, Mutex>()
25
24 private constructor () {} 26 private constructor () {}
25 27
26 getFSHLSOutputPath (video: MVideoUUID, filename?: string) { 28 getFSHLSOutputPath (video: MVideo, filename?: string) {
27 const base = getHLSDirectory(video) 29 const base = getHLSDirectory(video)
28 if (!filename) return base 30 if (!filename) return base
29 31
@@ -41,13 +43,17 @@ class VideoPathManager {
41 } 43 }
42 44
43 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { 45 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
44 if (videoFile.isHLS()) { 46 const video = extractVideo(videoOrPlaylist)
45 const video = extractVideo(videoOrPlaylist)
46 47
48 if (videoFile.isHLS()) {
47 return join(getHLSDirectory(video), videoFile.filename) 49 return join(getHLSDirectory(video), videoFile.filename)
48 } 50 }
49 51
50 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) 52 if (isVideoInPrivateDirectory(video.privacy)) {
53 return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename)
54 }
55
56 return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename)
51 } 57 }
52 58
53 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { 59 async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
@@ -113,6 +119,27 @@ class VideoPathManager {
113 ) 119 )
114 } 120 }
115 121
122 async lockFiles (videoUUID: string) {
123 if (!this.videoFileMutexStore.has(videoUUID)) {
124 this.videoFileMutexStore.set(videoUUID, new Mutex())
125 }
126
127 const mutex = this.videoFileMutexStore.get(videoUUID)
128 const releaser = await mutex.acquire()
129
130 logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID))
131
132 return releaser
133 }
134
135 unlockFiles (videoUUID: string) {
136 const mutex = this.videoFileMutexStore.get(videoUUID)
137
138 mutex.release()
139
140 logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID))
141 }
142
116 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { 143 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
117 let result: T 144 let result: T
118 145
diff --git a/server/lib/video-import.ts b/server/lib/video-pre-import.ts
index 796079875..796079875 100644
--- a/server/lib/video-import.ts
+++ b/server/lib/video-pre-import.ts
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts
new file mode 100644
index 000000000..41f9d62b3
--- /dev/null
+++ b/server/lib/video-privacy.ts
@@ -0,0 +1,127 @@
1import { move } from 'fs-extra'
2import { join } from 'path'
3import { logger } from '@server/helpers/logger'
4import { DIRECTORIES } from '@server/initializers/constants'
5import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
6import { VideoPrivacy, VideoStorage } from '@shared/models'
7import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage'
8
9function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
10 if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
11 video.publishedAt = new Date()
12 }
13
14 video.privacy = newPrivacy
15}
16
17function isVideoInPrivateDirectory (privacy: VideoPrivacy) {
18 return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL
19}
20
21function isVideoInPublicDirectory (privacy: VideoPrivacy) {
22 return !isVideoInPrivateDirectory(privacy)
23}
24
25async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) {
26 // Now public, previously private
27 if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) {
28 await moveFiles({ type: 'private-to-public', video })
29
30 return true
31 }
32
33 // Now private, previously public
34 if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) {
35 await moveFiles({ type: 'public-to-private', video })
36
37 return true
38 }
39
40 return false
41}
42
43export {
44 setVideoPrivacy,
45
46 isVideoInPrivateDirectory,
47 isVideoInPublicDirectory,
48
49 moveFilesIfPrivacyChanged
50}
51
52// ---------------------------------------------------------------------------
53
54type MoveType = 'private-to-public' | 'public-to-private'
55
56async function moveFiles (options: {
57 type: MoveType
58 video: MVideoFullLight
59}) {
60 const { type, video } = options
61
62 for (const file of video.VideoFiles) {
63 if (file.storage === VideoStorage.FILE_SYSTEM) {
64 await moveWebTorrentFileOnFS(type, video, file)
65 } else {
66 await updateWebTorrentFileACL(video, file)
67 }
68 }
69
70 const hls = video.getHLSPlaylist()
71
72 if (hls) {
73 if (hls.storage === VideoStorage.FILE_SYSTEM) {
74 await moveHLSFilesOnFS(type, video)
75 } else {
76 await updateHLSFilesACL(hls)
77 }
78 }
79}
80
81async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) {
82 const directories = getWebTorrentDirectories(type)
83
84 const source = join(directories.old, file.filename)
85 const destination = join(directories.new, file.filename)
86
87 try {
88 logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
89
90 await move(source, destination)
91 } catch (err) {
92 logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
93 }
94}
95
96function getWebTorrentDirectories (moveType: MoveType) {
97 if (moveType === 'private-to-public') {
98 return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }
99 }
100
101 return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }
102}
103
104// ---------------------------------------------------------------------------
105
106async function moveHLSFilesOnFS (type: MoveType, video: MVideo) {
107 const directories = getHLSDirectories(type)
108
109 const source = join(directories.old, video.uuid)
110 const destination = join(directories.new, video.uuid)
111
112 try {
113 logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
114
115 await move(source, destination)
116 } catch (err) {
117 logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
118 }
119}
120
121function getHLSDirectories (moveType: MoveType) {
122 if (moveType === 'private-to-public') {
123 return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
124 }
125
126 return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
127}
diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts
new file mode 100644
index 000000000..c43085d16
--- /dev/null
+++ b/server/lib/video-tokens-manager.ts
@@ -0,0 +1,49 @@
1import LRUCache from 'lru-cache'
2import { LRU_CACHE } from '@server/initializers/constants'
3import { buildUUID } from '@shared/extra-utils'
4
5// ---------------------------------------------------------------------------
6// Create temporary tokens that can be used as URL query parameters to access video static files
7// ---------------------------------------------------------------------------
8
9class VideoTokensManager {
10
11 private static instance: VideoTokensManager
12
13 private readonly lruCache = new LRUCache<string, string>({
14 max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE,
15 ttl: LRU_CACHE.VIDEO_TOKENS.TTL
16 })
17
18 private constructor () {}
19
20 create (videoUUID: string) {
21 const token = buildUUID()
22
23 const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL)
24
25 this.lruCache.set(token, videoUUID)
26
27 return { token, expires }
28 }
29
30 hasToken (options: {
31 token: string
32 videoUUID: string
33 }) {
34 const value = this.lruCache.get(options.token)
35 if (!value) return false
36
37 return value === options.videoUUID
38 }
39
40 static get Instance () {
41 return this.instance || (this.instance = new this())
42 }
43}
44
45// ---------------------------------------------------------------------------
46
47export {
48 VideoTokensManager
49}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 6c4f3ce7b..aacc41a7a 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -7,10 +7,11 @@ import { TagModel } from '@server/models/video/tag'
7import { VideoModel } from '@server/models/video/video' 7import { VideoModel } from '@server/models/video/video'
8import { VideoJobInfoModel } from '@server/models/video/video-job-info' 8import { VideoJobInfoModel } from '@server/models/video/video-job-info'
9import { FilteredModelAttributes } from '@server/types' 9import { FilteredModelAttributes } from '@server/types'
10import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 10import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
11import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' 11import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
12import { CreateJobOptions } from './job-queue/job-queue' 12import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 13import { updateVideoMiniatureFromExisting } from './thumbnail'
14import { moveFilesIfPrivacyChanged } from './video-privacy'
14 15
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 16function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
16 return { 17 return {
@@ -177,6 +178,59 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
177 178
178// --------------------------------------------------------------------------- 179// ---------------------------------------------------------------------------
179 180
181async function addVideoJobsAfterUpdate (options: {
182 video: MVideoFullLight
183 isNewVideo: boolean
184
185 nameChanged: boolean
186 oldPrivacy: VideoPrivacy
187}) {
188 const { video, nameChanged, oldPrivacy, isNewVideo } = options
189 const jobs: CreateJobArgument[] = []
190
191 const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
192
193 if (!video.isLive && (nameChanged || filePathChanged)) {
194 for (const file of (video.VideoFiles || [])) {
195 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
196
197 jobs.push({ type: 'manage-video-torrent', payload })
198 }
199
200 const hls = video.getHLSPlaylist()
201
202 for (const file of (hls?.VideoFiles || [])) {
203 const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
204
205 jobs.push({ type: 'manage-video-torrent', payload })
206 }
207 }
208
209 jobs.push({
210 type: 'federate-video',
211 payload: {
212 videoUUID: video.uuid,
213 isNewVideo
214 }
215 })
216
217 const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy)
218
219 if (wasConfidentialVideo) {
220 jobs.push({
221 type: 'notify',
222 payload: {
223 action: 'new-video',
224 videoUUID: video.uuid
225 }
226 })
227 }
228
229 return JobQueue.Instance.createSequentialJobFlow(...jobs)
230}
231
232// ---------------------------------------------------------------------------
233
180export { 234export {
181 buildLocalVideoFromReq, 235 buildLocalVideoFromReq,
182 buildVideoThumbnailsFromReq, 236 buildVideoThumbnailsFromReq,
@@ -185,5 +239,6 @@ export {
185 buildTranscodingJob, 239 buildTranscodingJob,
186 buildMoveToObjectStorageJob, 240 buildMoveToObjectStorageJob,
187 getTranscodingJobPriority, 241 getTranscodingJobPriority,
242 addVideoJobsAfterUpdate,
188 getCachedVideoDuration 243 getCachedVideoDuration
189} 244}
diff --git a/server/middlewares/activitypub.ts b/server/middlewares/activitypub.ts
index 0064a4760..261b9f690 100644
--- a/server/middlewares/activitypub.ts
+++ b/server/middlewares/activitypub.ts
@@ -125,7 +125,7 @@ async function checkJsonLDSignature (req: Request, res: Response) {
125 return wrapWithSpanAndContext('peertube.activitypub.JSONLDSignature', async () => { 125 return wrapWithSpanAndContext('peertube.activitypub.JSONLDSignature', async () => {
126 const signatureObject: ActivityPubSignature = req.body.signature 126 const signatureObject: ActivityPubSignature = req.body.signature
127 127
128 if (!signatureObject || !signatureObject.creator) { 128 if (!signatureObject?.creator) {
129 res.fail({ 129 res.fail({
130 status: HttpStatusCode.FORBIDDEN_403, 130 status: HttpStatusCode.FORBIDDEN_403,
131 message: 'Object and creator signature do not match' 131 message: 'Object and creator signature do not match'
diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts
index 904d47efd..e6025c8ce 100644
--- a/server/middlewares/auth.ts
+++ b/server/middlewares/auth.ts
@@ -5,8 +5,8 @@ import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { handleOAuthAuthenticate } from '../lib/auth/oauth' 6import { handleOAuthAuthenticate } from '../lib/auth/oauth'
7 7
8function authenticate (req: express.Request, res: express.Response, next: express.NextFunction, authenticateInQuery = false) { 8function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) {
9 handleOAuthAuthenticate(req, res, authenticateInQuery) 9 handleOAuthAuthenticate(req, res)
10 .then((token: any) => { 10 .then((token: any) => {
11 res.locals.oauth = { token } 11 res.locals.oauth = { token }
12 res.locals.authenticated = true 12 res.locals.authenticated = true
@@ -47,7 +47,7 @@ function authenticateSocket (socket: Socket, next: (err?: any) => void) {
47 .catch(err => logger.error('Cannot get access token.', { err })) 47 .catch(err => logger.error('Cannot get access token.', { err }))
48} 48}
49 49
50function authenticatePromise (req: express.Request, res: express.Response, authenticateInQuery = false) { 50function authenticatePromise (req: express.Request, res: express.Response) {
51 return new Promise<void>(resolve => { 51 return new Promise<void>(resolve => {
52 // Already authenticated? (or tried to) 52 // Already authenticated? (or tried to)
53 if (res.locals.oauth?.token.User) return resolve() 53 if (res.locals.oauth?.token.User) return resolve()
@@ -59,7 +59,7 @@ function authenticatePromise (req: express.Request, res: express.Response, authe
59 }) 59 })
60 } 60 }
61 61
62 authenticate(req, res, () => resolve(), authenticateInQuery) 62 authenticate(req, res, () => resolve())
63 }) 63 })
64} 64}
65 65
diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts
index abc919339..9e15bf2d6 100644
--- a/server/middlewares/cache/shared/api-cache.ts
+++ b/server/middlewares/cache/shared/api-cache.ts
@@ -49,7 +49,7 @@ export class ApiCache {
49 if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) 49 if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration)
50 50
51 try { 51 try {
52 const obj = await redis.hGetAll(key) 52 const obj = await redis.hgetall(key)
53 if (obj?.response) { 53 if (obj?.response) {
54 return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) 54 return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration)
55 } 55 }
@@ -100,8 +100,8 @@ export class ApiCache {
100 100
101 if (Redis.Instance.isConnected()) { 101 if (Redis.Instance.isConnected()) {
102 await Promise.all([ 102 await Promise.all([
103 redis.hSet(key, 'response', JSON.stringify(value)), 103 redis.hset(key, 'response', JSON.stringify(value)),
104 redis.hSet(key, 'duration', duration + ''), 104 redis.hset(key, 'duration', duration + ''),
105 redis.expire(key, duration / 1000) 105 redis.expire(key, duration / 1000)
106 ]) 106 ])
107 } 107 }
diff --git a/server/middlewares/pagination.ts b/server/middlewares/pagination.ts
index 9812af9e4..17e43f743 100644
--- a/server/middlewares/pagination.ts
+++ b/server/middlewares/pagination.ts
@@ -1,12 +1,13 @@
1import express from 'express' 1import express from 'express'
2import { forceNumber } from '@shared/core-utils'
2import { PAGINATION } from '../initializers/constants' 3import { PAGINATION } from '../initializers/constants'
3 4
4function setDefaultPagination (req: express.Request, res: express.Response, next: express.NextFunction) { 5function setDefaultPagination (req: express.Request, res: express.Response, next: express.NextFunction) {
5 if (!req.query.start) req.query.start = 0 6 if (!req.query.start) req.query.start = 0
6 else req.query.start = parseInt(req.query.start, 10) 7 else req.query.start = forceNumber(req.query.start)
7 8
8 if (!req.query.count) req.query.count = PAGINATION.GLOBAL.COUNT.DEFAULT 9 if (!req.query.count) req.query.count = PAGINATION.GLOBAL.COUNT.DEFAULT
9 else req.query.count = parseInt(req.query.count, 10) 10 else req.query.count = forceNumber(req.query.count)
10 11
11 return next() 12 return next()
12} 13}
diff --git a/server/middlewares/validators/abuse.ts b/server/middlewares/validators/abuse.ts
index 9b94008ce..70bae1775 100644
--- a/server/middlewares/validators/abuse.ts
+++ b/server/middlewares/validators/abuse.ts
@@ -18,6 +18,7 @@ import { AbuseMessageModel } from '@server/models/abuse/abuse-message'
18import { AbuseCreate, UserRight } from '@shared/models' 18import { AbuseCreate, UserRight } from '@shared/models'
19import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' 19import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
20import { areValidationErrors, doesAbuseExist, doesAccountIdExist, doesCommentIdExist, doesVideoExist } from './shared' 20import { areValidationErrors, doesAbuseExist, doesAccountIdExist, doesCommentIdExist, doesVideoExist } from './shared'
21import { forceNumber } from '@shared/core-utils'
21 22
22const abuseReportValidator = [ 23const abuseReportValidator = [
23 body('account.id') 24 body('account.id')
@@ -216,7 +217,7 @@ const deleteAbuseMessageValidator = [
216 const user = res.locals.oauth.token.user 217 const user = res.locals.oauth.token.user
217 const abuse = res.locals.abuse 218 const abuse = res.locals.abuse
218 219
219 const messageId = parseInt(req.params.messageId + '', 10) 220 const messageId = forceNumber(req.params.messageId)
220 const abuseMessage = await AbuseMessageModel.loadByIdAndAbuseId(messageId, abuse.id) 221 const abuseMessage = await AbuseMessageModel.loadByIdAndAbuseId(messageId, abuse.id)
221 222
222 if (!abuseMessage) { 223 if (!abuseMessage) {
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index ffadb3b49..9bc8887ff 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -1,7 +1,6 @@
1export * from './activitypub'
2export * from './videos'
3export * from './abuse' 1export * from './abuse'
4export * from './account' 2export * from './account'
3export * from './activitypub'
5export * from './actor-image' 4export * from './actor-image'
6export * from './blocklist' 5export * from './blocklist'
7export * from './bulk' 6export * from './bulk'
@@ -10,8 +9,9 @@ export * from './express'
10export * from './feeds' 9export * from './feeds'
11export * from './follows' 10export * from './follows'
12export * from './jobs' 11export * from './jobs'
13export * from './metrics'
14export * from './logs' 12export * from './logs'
13export * from './metrics'
14export * from './object-storage-proxy'
15export * from './oembed' 15export * from './oembed'
16export * from './pagination' 16export * from './pagination'
17export * from './plugins' 17export * from './plugins'
@@ -19,9 +19,11 @@ export * from './redundancy'
19export * from './search' 19export * from './search'
20export * from './server' 20export * from './server'
21export * from './sort' 21export * from './sort'
22export * from './static'
22export * from './themes' 23export * from './themes'
23export * from './user-history' 24export * from './user-history'
24export * from './user-notifications' 25export * from './user-notifications'
25export * from './user-subscriptions' 26export * from './user-subscriptions'
26export * from './users' 27export * from './users'
28export * from './videos'
27export * from './webfinger' 29export * from './webfinger'
diff --git a/server/middlewares/validators/object-storage-proxy.ts b/server/middlewares/validators/object-storage-proxy.ts
new file mode 100644
index 000000000..bbd77f262
--- /dev/null
+++ b/server/middlewares/validators/object-storage-proxy.ts
@@ -0,0 +1,20 @@
1import express from 'express'
2import { CONFIG } from '@server/initializers/config'
3import { HttpStatusCode } from '@shared/models'
4
5const ensurePrivateObjectStorageProxyIsEnabled = [
6 (req: express.Request, res: express.Response, next: express.NextFunction) => {
7 if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) {
8 return res.fail({
9 message: 'Private object storage proxy is not enabled',
10 status: HttpStatusCode.BAD_REQUEST_400
11 })
12 }
13
14 return next()
15 }
16]
17
18export {
19 ensurePrivateObjectStorageProxyIsEnabled
20}
diff --git a/server/middlewares/validators/plugins.ts b/server/middlewares/validators/plugins.ts
index 78c030333..64bef2648 100644
--- a/server/middlewares/validators/plugins.ts
+++ b/server/middlewares/validators/plugins.ts
@@ -4,7 +4,12 @@ import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
4import { PluginType } from '../../../shared/models/plugins/plugin.type' 4import { PluginType } from '../../../shared/models/plugins/plugin.type'
5import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/server/api/install-plugin.model' 5import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/server/api/install-plugin.model'
6import { exists, isBooleanValid, isSafePath, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' 6import { exists, isBooleanValid, isSafePath, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
7import { isNpmPluginNameValid, isPluginNameValid, isPluginTypeValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins' 7import {
8 isNpmPluginNameValid,
9 isPluginNameValid,
10 isPluginStableOrUnstableVersionValid,
11 isPluginTypeValid
12} from '../../helpers/custom-validators/plugins'
8import { CONFIG } from '../../initializers/config' 13import { CONFIG } from '../../initializers/config'
9import { PluginManager } from '../../lib/plugins/plugin-manager' 14import { PluginManager } from '../../lib/plugins/plugin-manager'
10import { PluginModel } from '../../models/server/plugin' 15import { PluginModel } from '../../models/server/plugin'
@@ -19,7 +24,7 @@ const getPluginValidator = (pluginType: PluginType, withVersion = true) => {
19 if (withVersion) { 24 if (withVersion) {
20 validators.push( 25 validators.push(
21 param('pluginVersion') 26 param('pluginVersion')
22 .custom(isPluginVersionValid) 27 .custom(isPluginStableOrUnstableVersionValid)
23 ) 28 )
24 } 29 }
25 30
@@ -113,7 +118,7 @@ const installOrUpdatePluginValidator = [
113 .custom(isNpmPluginNameValid), 118 .custom(isNpmPluginNameValid),
114 body('pluginVersion') 119 body('pluginVersion')
115 .optional() 120 .optional()
116 .custom(isPluginVersionValid), 121 .custom(isPluginStableOrUnstableVersionValid),
117 body('path') 122 body('path')
118 .optional() 123 .optional()
119 .custom(isSafePath), 124 .custom(isSafePath),
@@ -185,7 +190,7 @@ const listAvailablePluginsValidator = [
185 .custom(isPluginTypeValid), 190 .custom(isPluginTypeValid),
186 query('currentPeerTubeEngine') 191 query('currentPeerTubeEngine')
187 .optional() 192 .optional()
188 .custom(isPluginVersionValid), 193 .custom(isPluginStableOrUnstableVersionValid),
189 194
190 (req: express.Request, res: express.Response, next: express.NextFunction) => { 195 (req: express.Request, res: express.Response, next: express.NextFunction) => {
191 if (areValidationErrors(req, res)) return 196 if (areValidationErrors(req, res)) return
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
index 79460f63c..c80f9b728 100644
--- a/server/middlewares/validators/redundancy.ts
+++ b/server/middlewares/validators/redundancy.ts
@@ -1,6 +1,7 @@
1import express from 'express' 1import express from 'express'
2import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies' 3import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies'
4import { forceNumber } from '@shared/core-utils'
4import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' 5import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
5import { 6import {
6 exists, 7 exists,
@@ -171,7 +172,7 @@ const removeVideoRedundancyValidator = [
171 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 172 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
172 if (areValidationErrors(req, res)) return 173 if (areValidationErrors(req, res)) return
173 174
174 const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10)) 175 const redundancy = await VideoRedundancyModel.loadByIdWithVideo(forceNumber(req.params.redundancyId))
175 if (!redundancy) { 176 if (!redundancy) {
176 return res.fail({ 177 return res.fail({
177 status: HttpStatusCode.NOT_FOUND_404, 178 status: HttpStatusCode.NOT_FOUND_404,
diff --git a/server/middlewares/validators/shared/abuses.ts b/server/middlewares/validators/shared/abuses.ts
index 2b8d86ba5..2c988f9ec 100644
--- a/server/middlewares/validators/shared/abuses.ts
+++ b/server/middlewares/validators/shared/abuses.ts
@@ -1,9 +1,10 @@
1import { Response } from 'express' 1import { Response } from 'express'
2import { AbuseModel } from '@server/models/abuse/abuse' 2import { AbuseModel } from '@server/models/abuse/abuse'
3import { HttpStatusCode } from '@shared/models' 3import { HttpStatusCode } from '@shared/models'
4import { forceNumber } from '@shared/core-utils'
4 5
5async function doesAbuseExist (abuseId: number | string, res: Response) { 6async function doesAbuseExist (abuseId: number | string, res: Response) {
6 const abuse = await AbuseModel.loadByIdWithReporter(parseInt(abuseId + '', 10)) 7 const abuse = await AbuseModel.loadByIdWithReporter(forceNumber(abuseId))
7 8
8 if (!abuse) { 9 if (!abuse) {
9 res.fail({ 10 res.fail({
diff --git a/server/middlewares/validators/shared/accounts.ts b/server/middlewares/validators/shared/accounts.ts
index fe4f83aa0..72b0e235e 100644
--- a/server/middlewares/validators/shared/accounts.ts
+++ b/server/middlewares/validators/shared/accounts.ts
@@ -2,10 +2,11 @@ import { Response } from 'express'
2import { AccountModel } from '@server/models/account/account' 2import { AccountModel } from '@server/models/account/account'
3import { UserModel } from '@server/models/user/user' 3import { UserModel } from '@server/models/user/user'
4import { MAccountDefault } from '@server/types/models' 4import { MAccountDefault } from '@server/types/models'
5import { forceNumber } from '@shared/core-utils'
5import { HttpStatusCode } from '@shared/models' 6import { HttpStatusCode } from '@shared/models'
6 7
7function doesAccountIdExist (id: number | string, res: Response, sendNotFound = true) { 8function doesAccountIdExist (id: number | string, res: Response, sendNotFound = true) {
8 const promise = AccountModel.load(parseInt(id + '', 10)) 9 const promise = AccountModel.load(forceNumber(id))
9 10
10 return doesAccountExist(promise, res, sendNotFound) 11 return doesAccountExist(promise, res, sendNotFound)
11} 12}
@@ -40,7 +41,7 @@ async function doesAccountExist (p: Promise<MAccountDefault>, res: Response, sen
40} 41}
41 42
42async function doesUserFeedTokenCorrespond (id: number, token: string, res: Response) { 43async function doesUserFeedTokenCorrespond (id: number, token: string, res: Response) {
43 const user = await UserModel.loadByIdWithChannels(parseInt(id + '', 10)) 44 const user = await UserModel.loadByIdWithChannels(forceNumber(id))
44 45
45 if (token !== user.feedToken) { 46 if (token !== user.feedToken) {
46 res.fail({ 47 res.fail({
diff --git a/server/middlewares/validators/shared/index.ts b/server/middlewares/validators/shared/index.ts
index bbd03b248..de98cd442 100644
--- a/server/middlewares/validators/shared/index.ts
+++ b/server/middlewares/validators/shared/index.ts
@@ -1,5 +1,6 @@
1export * from './abuses' 1export * from './abuses'
2export * from './accounts' 2export * from './accounts'
3export * from './users'
3export * from './utils' 4export * from './utils'
4export * from './video-blacklists' 5export * from './video-blacklists'
5export * from './video-captions' 6export * from './video-captions'
diff --git a/server/middlewares/validators/shared/users.ts b/server/middlewares/validators/shared/users.ts
new file mode 100644
index 000000000..b8f1436d3
--- /dev/null
+++ b/server/middlewares/validators/shared/users.ts
@@ -0,0 +1,63 @@
1import express from 'express'
2import { ActorModel } from '@server/models/actor/actor'
3import { UserModel } from '@server/models/user/user'
4import { MUserDefault } from '@server/types/models'
5import { forceNumber } from '@shared/core-utils'
6import { HttpStatusCode } from '@shared/models'
7
8function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
9 const id = forceNumber(idArg)
10 return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res)
11}
12
13function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {
14 return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse)
15}
16
17async function checkUserNameOrEmailDoesNotAlreadyExist (username: string, email: string, res: express.Response) {
18 const user = await UserModel.loadByUsernameOrEmail(username, email)
19
20 if (user) {
21 res.fail({
22 status: HttpStatusCode.CONFLICT_409,
23 message: 'User with this username or email already exists.'
24 })
25 return false
26 }
27
28 const actor = await ActorModel.loadLocalByName(username)
29 if (actor) {
30 res.fail({
31 status: HttpStatusCode.CONFLICT_409,
32 message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.'
33 })
34 return false
35 }
36
37 return true
38}
39
40async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) {
41 const user = await finder()
42
43 if (!user) {
44 if (abortResponse === true) {
45 res.fail({
46 status: HttpStatusCode.NOT_FOUND_404,
47 message: 'User not found'
48 })
49 }
50
51 return false
52 }
53
54 res.locals.user = user
55 return true
56}
57
58export {
59 checkUserIdExist,
60 checkUserEmailExist,
61 checkUserNameOrEmailDoesNotAlreadyExist,
62 checkUserExist
63}
diff --git a/server/middlewares/validators/shared/video-comments.ts b/server/middlewares/validators/shared/video-comments.ts
index 8d1a16294..0961b3ec9 100644
--- a/server/middlewares/validators/shared/video-comments.ts
+++ b/server/middlewares/validators/shared/video-comments.ts
@@ -1,10 +1,11 @@
1import express from 'express' 1import express from 'express'
2import { VideoCommentModel } from '@server/models/video/video-comment' 2import { VideoCommentModel } from '@server/models/video/video-comment'
3import { MVideoId } from '@server/types/models' 3import { MVideoId } from '@server/types/models'
4import { forceNumber } from '@shared/core-utils'
4import { HttpStatusCode, ServerErrorCode } from '@shared/models' 5import { HttpStatusCode, ServerErrorCode } from '@shared/models'
5 6
6async function doesVideoCommentThreadExist (idArg: number | string, video: MVideoId, res: express.Response) { 7async function doesVideoCommentThreadExist (idArg: number | string, video: MVideoId, res: express.Response) {
7 const id = parseInt(idArg + '', 10) 8 const id = forceNumber(idArg)
8 const videoComment = await VideoCommentModel.loadById(id) 9 const videoComment = await VideoCommentModel.loadById(id)
9 10
10 if (!videoComment) { 11 if (!videoComment) {
@@ -33,7 +34,7 @@ async function doesVideoCommentThreadExist (idArg: number | string, video: MVide
33} 34}
34 35
35async function doesVideoCommentExist (idArg: number | string, video: MVideoId, res: express.Response) { 36async function doesVideoCommentExist (idArg: number | string, video: MVideoId, res: express.Response) {
36 const id = parseInt(idArg + '', 10) 37 const id = forceNumber(idArg)
37 const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id) 38 const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id)
38 39
39 if (!videoComment) { 40 if (!videoComment) {
@@ -57,7 +58,7 @@ async function doesVideoCommentExist (idArg: number | string, video: MVideoId, r
57} 58}
58 59
59async function doesCommentIdExist (idArg: number | string, res: express.Response) { 60async function doesCommentIdExist (idArg: number | string, res: express.Response) {
60 const id = parseInt(idArg + '', 10) 61 const id = forceNumber(idArg)
61 const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id) 62 const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id)
62 63
63 if (!videoComment) { 64 if (!videoComment) {
diff --git a/server/middlewares/validators/shared/video-ownerships.ts b/server/middlewares/validators/shared/video-ownerships.ts
index 680613cda..33ac9c8b6 100644
--- a/server/middlewares/validators/shared/video-ownerships.ts
+++ b/server/middlewares/validators/shared/video-ownerships.ts
@@ -1,9 +1,10 @@
1import express from 'express' 1import express from 'express'
2import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership' 2import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership'
3import { forceNumber } from '@shared/core-utils'
3import { HttpStatusCode } from '@shared/models' 4import { HttpStatusCode } from '@shared/models'
4 5
5async function doesChangeVideoOwnershipExist (idArg: number | string, res: express.Response) { 6async function doesChangeVideoOwnershipExist (idArg: number | string, res: express.Response) {
6 const id = parseInt(idArg + '', 10) 7 const id = forceNumber(idArg)
7 const videoChangeOwnership = await VideoChangeOwnershipModel.load(id) 8 const videoChangeOwnership = await VideoChangeOwnershipModel.load(id)
8 9
9 if (!videoChangeOwnership) { 10 if (!videoChangeOwnership) {
diff --git a/server/middlewares/validators/shared/videos.ts b/server/middlewares/validators/shared/videos.ts
index e3a98c58f..ebbfc0a0a 100644
--- a/server/middlewares/validators/shared/videos.ts
+++ b/server/middlewares/validators/shared/videos.ts
@@ -1,7 +1,7 @@
1import { Request, Response } from 'express' 1import { Request, Response } from 'express'
2import { isUUIDValid } from '@server/helpers/custom-validators/misc'
3import { loadVideo, VideoLoadType } from '@server/lib/model-loaders' 2import { loadVideo, VideoLoadType } from '@server/lib/model-loaders'
4import { isAbleToUploadVideo } from '@server/lib/user' 3import { isAbleToUploadVideo } from '@server/lib/user'
4import { VideoTokensManager } from '@server/lib/video-tokens-manager'
5import { authenticatePromise } from '@server/middlewares/auth' 5import { authenticatePromise } from '@server/middlewares/auth'
6import { VideoModel } from '@server/models/video/video' 6import { VideoModel } from '@server/models/video/video'
7import { VideoChannelModel } from '@server/models/video/video-channel' 7import { VideoChannelModel } from '@server/models/video/video-channel'
@@ -108,26 +108,21 @@ async function checkCanSeeVideo (options: {
108 res: Response 108 res: Response
109 paramId: string 109 paramId: string
110 video: MVideo 110 video: MVideo
111 authenticateInQuery?: boolean // default false
112}) { 111}) {
113 const { req, res, video, paramId, authenticateInQuery = false } = options 112 const { req, res, video, paramId } = options
114 113
115 if (video.requiresAuth()) { 114 if (video.requiresAuth({ urlParamId: paramId, checkBlacklist: true })) {
116 return checkCanSeeAuthVideo(req, res, video, authenticateInQuery) 115 return checkCanSeeAuthVideo(req, res, video)
117 } 116 }
118 117
119 if (video.privacy === VideoPrivacy.UNLISTED) { 118 if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) {
120 if (isUUIDValid(paramId)) return true 119 return true
121
122 return checkCanSeeAuthVideo(req, res, video, authenticateInQuery)
123 } 120 }
124 121
125 if (video.privacy === VideoPrivacy.PUBLIC) return true 122 throw new Error('Unknown video privacy when checking video right ' + video.url)
126
127 throw new Error('Fatal error when checking video right ' + video.url)
128} 123}
129 124
130async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights, authenticateInQuery = false) { 125async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights) {
131 const fail = () => { 126 const fail = () => {
132 res.fail({ 127 res.fail({
133 status: HttpStatusCode.FORBIDDEN_403, 128 status: HttpStatusCode.FORBIDDEN_403,
@@ -137,7 +132,7 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
137 return false 132 return false
138 } 133 }
139 134
140 await authenticatePromise(req, res, authenticateInQuery) 135 await authenticatePromise(req, res)
141 136
142 const user = res.locals.oauth?.token.User 137 const user = res.locals.oauth?.token.User
143 if (!user) return fail() 138 if (!user) return fail()
@@ -173,6 +168,36 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
173 168
174// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
175 170
171async function checkCanAccessVideoStaticFiles (options: {
172 video: MVideo
173 req: Request
174 res: Response
175 paramId: string
176}) {
177 const { video, req, res } = options
178
179 if (res.locals.oauth?.token.User) {
180 return checkCanSeeVideo(options)
181 }
182
183 if (!video.hasPrivateStaticPath()) return true
184
185 const videoFileToken = req.query.videoFileToken
186 if (!videoFileToken) {
187 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
188 return false
189 }
190
191 if (VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) {
192 return true
193 }
194
195 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
196 return false
197}
198
199// ---------------------------------------------------------------------------
200
176function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) { 201function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) {
177 // Retrieve the user who did the request 202 // Retrieve the user who did the request
178 if (onlyOwned && video.isOwned() === false) { 203 if (onlyOwned && video.isOwned() === false) {
@@ -220,6 +245,7 @@ export {
220 doesVideoExist, 245 doesVideoExist,
221 doesVideoFileOfVideoExist, 246 doesVideoFileOfVideoExist,
222 247
248 checkCanAccessVideoStaticFiles,
223 checkUserCanManageVideo, 249 checkUserCanManageVideo,
224 checkCanSeeVideo, 250 checkCanSeeVideo,
225 checkUserQuota 251 checkUserQuota
diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts
new file mode 100644
index 000000000..13fde6dd1
--- /dev/null
+++ b/server/middlewares/validators/static.ts
@@ -0,0 +1,169 @@
1import express from 'express'
2import { query } from 'express-validator'
3import LRUCache from 'lru-cache'
4import { basename, dirname } from 'path'
5import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc'
6import { logger } from '@server/helpers/logger'
7import { LRU_CACHE } from '@server/initializers/constants'
8import { VideoModel } from '@server/models/video/video'
9import { VideoFileModel } from '@server/models/video/video-file'
10import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models'
11import { HttpStatusCode } from '@shared/models'
12import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared'
13
14type LRUValue = {
15 allowed: boolean
16 video?: MVideoThumbnail
17 file?: MVideoFile
18 playlist?: MStreamingPlaylist }
19
20const staticFileTokenBypass = new LRUCache<string, LRUValue>({
21 max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
22 ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
23})
24
25const ensureCanAccessVideoPrivateWebTorrentFiles = [
26 query('videoFileToken').optional().custom(exists),
27
28 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
29 if (areValidationErrors(req, res)) return
30
31 const token = extractTokenOrDie(req, res)
32 if (!token) return
33
34 const cacheKey = token + '-' + req.originalUrl
35
36 if (staticFileTokenBypass.has(cacheKey)) {
37 const { allowed, file, video } = staticFileTokenBypass.get(cacheKey)
38
39 if (allowed === true) {
40 res.locals.onlyVideo = video
41 res.locals.videoFile = file
42
43 return next()
44 }
45
46 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
47 }
48
49 const result = await isWebTorrentAllowed(req, res)
50
51 staticFileTokenBypass.set(cacheKey, result)
52
53 if (result.allowed !== true) return
54
55 res.locals.onlyVideo = result.video
56 res.locals.videoFile = result.file
57
58 return next()
59 }
60]
61
62const ensureCanAccessPrivateVideoHLSFiles = [
63 query('videoFileToken').optional().custom(exists),
64
65 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
66 if (areValidationErrors(req, res)) return
67
68 const videoUUID = basename(dirname(req.originalUrl))
69
70 if (!isUUIDValid(videoUUID)) {
71 logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl)
72
73 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
74 }
75
76 const token = extractTokenOrDie(req, res)
77 if (!token) return
78
79 const cacheKey = token + '-' + videoUUID
80
81 if (staticFileTokenBypass.has(cacheKey)) {
82 const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey)
83
84 if (allowed === true) {
85 res.locals.onlyVideo = video
86 res.locals.videoFile = file
87 res.locals.videoStreamingPlaylist = playlist
88
89 return next()
90 }
91
92 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
93 }
94
95 const result = await isHLSAllowed(req, res, videoUUID)
96
97 staticFileTokenBypass.set(cacheKey, result)
98
99 if (result.allowed !== true) return
100
101 res.locals.onlyVideo = result.video
102 res.locals.videoFile = result.file
103 res.locals.videoStreamingPlaylist = result.playlist
104
105 return next()
106 }
107]
108
109export {
110 ensureCanAccessVideoPrivateWebTorrentFiles,
111 ensureCanAccessPrivateVideoHLSFiles
112}
113
114// ---------------------------------------------------------------------------
115
116async function isWebTorrentAllowed (req: express.Request, res: express.Response) {
117 const filename = basename(req.path)
118
119 const file = await VideoFileModel.loadWithVideoByFilename(filename)
120 if (!file) {
121 logger.debug('Unknown static file %s to serve', req.originalUrl, { filename })
122
123 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
124 return { allowed: false }
125 }
126
127 const video = await VideoModel.load(file.getVideo().id)
128
129 return {
130 file,
131 video,
132 allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
133 }
134}
135
136async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) {
137 const filename = basename(req.path)
138
139 const video = await VideoModel.loadWithFiles(videoUUID)
140
141 if (!video) {
142 logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID })
143
144 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
145 return { allowed: false }
146 }
147
148 const file = await VideoFileModel.loadByFilename(filename)
149
150 return {
151 file,
152 video,
153 playlist: video.getHLSPlaylist(),
154 allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
155 }
156}
157
158function extractTokenOrDie (req: express.Request, res: express.Response) {
159 const token = res.locals.oauth?.token.accessToken || req.query.videoFileToken
160
161 if (!token) {
162 return res.fail({
163 message: 'Bearer token is missing in headers or video file token is missing in URL query parameters',
164 status: HttpStatusCode.FORBIDDEN_403
165 })
166 }
167
168 return token
169}
diff --git a/server/middlewares/validators/themes.ts b/server/middlewares/validators/themes.ts
index c130801a0..080b3e096 100644
--- a/server/middlewares/validators/themes.ts
+++ b/server/middlewares/validators/themes.ts
@@ -2,7 +2,7 @@ import express from 'express'
2import { param } from 'express-validator' 2import { param } from 'express-validator'
3import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' 3import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
4import { isSafePath } from '../../helpers/custom-validators/misc' 4import { isSafePath } from '../../helpers/custom-validators/misc'
5import { isPluginNameValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins' 5import { isPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins'
6import { PluginManager } from '../../lib/plugins/plugin-manager' 6import { PluginManager } from '../../lib/plugins/plugin-manager'
7import { areValidationErrors } from './shared' 7import { areValidationErrors } from './shared'
8 8
@@ -10,7 +10,7 @@ const serveThemeCSSValidator = [
10 param('themeName') 10 param('themeName')
11 .custom(isPluginNameValid), 11 .custom(isPluginNameValid),
12 param('themeVersion') 12 param('themeVersion')
13 .custom(isPluginVersionValid), 13 .custom(isPluginStableOrUnstableVersionValid),
14 param('staticEndpoint') 14 param('staticEndpoint')
15 .custom(isSafePath), 15 .custom(isSafePath),
16 16
diff --git a/server/middlewares/validators/two-factor.ts b/server/middlewares/validators/two-factor.ts
new file mode 100644
index 000000000..106b579b5
--- /dev/null
+++ b/server/middlewares/validators/two-factor.ts
@@ -0,0 +1,81 @@
1import express from 'express'
2import { body, param } from 'express-validator'
3import { HttpStatusCode, UserRight } from '@shared/models'
4import { exists, isIdValid } from '../../helpers/custom-validators/misc'
5import { areValidationErrors, checkUserIdExist } from './shared'
6
7const requestOrConfirmTwoFactorValidator = [
8 param('id').custom(isIdValid),
9
10 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
11 if (areValidationErrors(req, res)) return
12
13 if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return
14
15 if (res.locals.user.otpSecret) {
16 return res.fail({
17 status: HttpStatusCode.BAD_REQUEST_400,
18 message: `Two factor is already enabled.`
19 })
20 }
21
22 return next()
23 }
24]
25
26const confirmTwoFactorValidator = [
27 body('requestToken').custom(exists),
28 body('otpToken').custom(exists),
29
30 (req: express.Request, res: express.Response, next: express.NextFunction) => {
31 if (areValidationErrors(req, res)) return
32
33 return next()
34 }
35]
36
37const disableTwoFactorValidator = [
38 param('id').custom(isIdValid),
39
40 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
41 if (areValidationErrors(req, res)) return
42
43 if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return
44
45 if (!res.locals.user.otpSecret) {
46 return res.fail({
47 status: HttpStatusCode.BAD_REQUEST_400,
48 message: `Two factor is already disabled.`
49 })
50 }
51
52 return next()
53 }
54]
55
56// ---------------------------------------------------------------------------
57
58export {
59 requestOrConfirmTwoFactorValidator,
60 confirmTwoFactorValidator,
61 disableTwoFactorValidator
62}
63
64// ---------------------------------------------------------------------------
65
66async function checkCanEnableOrDisableTwoFactor (userId: number | string, res: express.Response) {
67 const authUser = res.locals.oauth.token.user
68
69 if (!await checkUserIdExist(userId, res)) return
70
71 if (res.locals.user.id !== authUser.id && authUser.hasRight(UserRight.MANAGE_USERS) !== true) {
72 res.fail({
73 status: HttpStatusCode.FORBIDDEN_403,
74 message: `User ${authUser.username} does not have right to change two factor setting of this user.`
75 })
76
77 return false
78 }
79
80 return true
81}
diff --git a/server/middlewares/validators/user-subscriptions.ts b/server/middlewares/validators/user-subscriptions.ts
index d01043c17..d8d3fc28b 100644
--- a/server/middlewares/validators/user-subscriptions.ts
+++ b/server/middlewares/validators/user-subscriptions.ts
@@ -1,6 +1,6 @@
1import { arrayify } from '@shared/core-utils'
2import express from 'express' 1import express from 'express'
3import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { arrayify } from '@shared/core-utils'
4import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' 4import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
5import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor' 5import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
6import { WEBSERVER } from '../../initializers/constants' 6import { WEBSERVER } from '../../initializers/constants'
@@ -60,7 +60,7 @@ const userSubscriptionGetValidator = [
60 state: 'accepted' 60 state: 'accepted'
61 }) 61 })
62 62
63 if (!subscription || !subscription.ActorFollowing.VideoChannel) { 63 if (!subscription?.ActorFollowing.VideoChannel) {
64 return res.fail({ 64 return res.fail({
65 status: HttpStatusCode.NOT_FOUND_404, 65 status: HttpStatusCode.NOT_FOUND_404,
66 message: `Subscription ${req.params.uri} not found.` 66 message: `Subscription ${req.params.uri} not found.`
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index 2de5265fb..50327b6ae 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -1,9 +1,9 @@
1import express from 'express' 1import express from 'express'
2import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { Hooks } from '@server/lib/plugins/hooks' 3import { Hooks } from '@server/lib/plugins/hooks'
4import { MUserDefault } from '@server/types/models' 4import { forceNumber } from '@shared/core-utils'
5import { HttpStatusCode, UserRegister, UserRight, UserRole } from '@shared/models' 5import { HttpStatusCode, UserRegister, UserRight, UserRole } from '@shared/models'
6import { isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' 6import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
7import { isThemeNameValid } from '../../helpers/custom-validators/plugins' 7import { isThemeNameValid } from '../../helpers/custom-validators/plugins'
8import { 8import {
9 isUserAdminFlagsValid, 9 isUserAdminFlagsValid,
@@ -30,8 +30,15 @@ import { isThemeRegistered } from '../../lib/plugins/theme-utils'
30import { Redis } from '../../lib/redis' 30import { Redis } from '../../lib/redis'
31import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../lib/signup' 31import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../lib/signup'
32import { ActorModel } from '../../models/actor/actor' 32import { ActorModel } from '../../models/actor/actor'
33import { UserModel } from '../../models/user/user' 33import {
34import { areValidationErrors, doesVideoChannelIdExist, doesVideoExist, isValidVideoIdParam } from './shared' 34 areValidationErrors,
35 checkUserEmailExist,
36 checkUserIdExist,
37 checkUserNameOrEmailDoesNotAlreadyExist,
38 doesVideoChannelIdExist,
39 doesVideoExist,
40 isValidVideoIdParam
41} from './shared'
35 42
36const usersListValidator = [ 43const usersListValidator = [
37 query('blocked') 44 query('blocked')
@@ -411,6 +418,13 @@ const usersAskResetPasswordValidator = [
411 return res.status(HttpStatusCode.NO_CONTENT_204).end() 418 return res.status(HttpStatusCode.NO_CONTENT_204).end()
412 } 419 }
413 420
421 if (res.locals.user.pluginAuth) {
422 return res.fail({
423 status: HttpStatusCode.CONFLICT_409,
424 message: 'Cannot recover password of a user that uses a plugin authentication.'
425 })
426 }
427
414 return next() 428 return next()
415 } 429 }
416] 430]
@@ -428,7 +442,7 @@ const usersResetPasswordValidator = [
428 if (!await checkUserIdExist(req.params.id, res)) return 442 if (!await checkUserIdExist(req.params.id, res)) return
429 443
430 const user = res.locals.user 444 const user = res.locals.user
431 const redisVerificationString = await Redis.Instance.getResetPasswordLink(user.id) 445 const redisVerificationString = await Redis.Instance.getResetPasswordVerificationString(user.id)
432 446
433 if (redisVerificationString !== req.body.verificationString) { 447 if (redisVerificationString !== req.body.verificationString) {
434 return res.fail({ 448 return res.fail({
@@ -454,6 +468,13 @@ const usersAskSendVerifyEmailValidator = [
454 return res.status(HttpStatusCode.NO_CONTENT_204).end() 468 return res.status(HttpStatusCode.NO_CONTENT_204).end()
455 } 469 }
456 470
471 if (res.locals.user.pluginAuth) {
472 return res.fail({
473 status: HttpStatusCode.CONFLICT_409,
474 message: 'Cannot ask verification email of a user that uses a plugin authentication.'
475 })
476 }
477
457 return next() 478 return next()
458 } 479 }
459] 480]
@@ -486,6 +507,41 @@ const usersVerifyEmailValidator = [
486 } 507 }
487] 508]
488 509
510const usersCheckCurrentPasswordFactory = (targetUserIdGetter: (req: express.Request) => number | string) => {
511 return [
512 body('currentPassword').optional().custom(exists),
513
514 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
515 if (areValidationErrors(req, res)) return
516
517 const user = res.locals.oauth.token.User
518 const isAdminOrModerator = user.role === UserRole.ADMINISTRATOR || user.role === UserRole.MODERATOR
519 const targetUserId = forceNumber(targetUserIdGetter(req))
520
521 // Admin/moderator action on another user, skip the password check
522 if (isAdminOrModerator && targetUserId !== user.id) {
523 return next()
524 }
525
526 if (!req.body.currentPassword) {
527 return res.fail({
528 status: HttpStatusCode.BAD_REQUEST_400,
529 message: 'currentPassword is missing'
530 })
531 }
532
533 if (await user.isPasswordMatch(req.body.currentPassword) !== true) {
534 return res.fail({
535 status: HttpStatusCode.FORBIDDEN_403,
536 message: 'currentPassword is invalid.'
537 })
538 }
539
540 return next()
541 }
542 ]
543}
544
489const userAutocompleteValidator = [ 545const userAutocompleteValidator = [
490 param('search') 546 param('search')
491 .isString() 547 .isString()
@@ -553,6 +609,7 @@ export {
553 usersUpdateValidator, 609 usersUpdateValidator,
554 usersUpdateMeValidator, 610 usersUpdateMeValidator,
555 usersVideoRatingValidator, 611 usersVideoRatingValidator,
612 usersCheckCurrentPasswordFactory,
556 ensureUserRegistrationAllowed, 613 ensureUserRegistrationAllowed,
557 ensureUserRegistrationAllowedForIP, 614 ensureUserRegistrationAllowedForIP,
558 usersGetValidator, 615 usersGetValidator,
@@ -566,55 +623,3 @@ export {
566 ensureCanModerateUser, 623 ensureCanModerateUser,
567 ensureCanManageChannelOrAccount 624 ensureCanManageChannelOrAccount
568} 625}
569
570// ---------------------------------------------------------------------------
571
572function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) {
573 const id = parseInt(idArg + '', 10)
574 return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res)
575}
576
577function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) {
578 return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse)
579}
580
581async function checkUserNameOrEmailDoesNotAlreadyExist (username: string, email: string, res: express.Response) {
582 const user = await UserModel.loadByUsernameOrEmail(username, email)
583
584 if (user) {
585 res.fail({
586 status: HttpStatusCode.CONFLICT_409,
587 message: 'User with this username or email already exists.'
588 })
589 return false
590 }
591
592 const actor = await ActorModel.loadLocalByName(username)
593 if (actor) {
594 res.fail({
595 status: HttpStatusCode.CONFLICT_409,
596 message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.'
597 })
598 return false
599 }
600
601 return true
602}
603
604async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) {
605 const user = await finder()
606
607 if (!user) {
608 if (abortResponse === true) {
609 res.fail({
610 status: HttpStatusCode.NOT_FOUND_404,
611 message: 'User not found'
612 })
613 }
614
615 return false
616 }
617
618 res.locals.user = user
619 return true
620}
diff --git a/server/middlewares/validators/videos/video-comments.ts b/server/middlewares/validators/videos/video-comments.ts
index 69062701b..133feb7bd 100644
--- a/server/middlewares/validators/videos/video-comments.ts
+++ b/server/middlewares/validators/videos/video-comments.ts
@@ -208,7 +208,8 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon
208 const acceptParameters = { 208 const acceptParameters = {
209 video, 209 video,
210 commentBody: req.body, 210 commentBody: req.body,
211 user: res.locals.oauth.token.User 211 user: res.locals.oauth.token.User,
212 req
212 } 213 }
213 214
214 let acceptedResult: AcceptResult 215 let acceptedResult: AcceptResult
@@ -234,7 +235,7 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon
234 235
235 res.fail({ 236 res.fail({
236 status: HttpStatusCode.FORBIDDEN_403, 237 status: HttpStatusCode.FORBIDDEN_403,
237 message: acceptedResult?.errorMessage || 'Refused local comment' 238 message: acceptedResult?.errorMessage || 'Comment has been rejected.'
238 }) 239 })
239 return false 240 return false
240 } 241 }
diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts
index f295b1885..72442aeb6 100644
--- a/server/middlewares/validators/videos/video-imports.ts
+++ b/server/middlewares/validators/videos/video-imports.ts
@@ -4,6 +4,7 @@ import { isResolvingToUnicastOnly } from '@server/helpers/dns'
4import { isPreImportVideoAccepted } from '@server/lib/moderation' 4import { isPreImportVideoAccepted } from '@server/lib/moderation'
5import { Hooks } from '@server/lib/plugins/hooks' 5import { Hooks } from '@server/lib/plugins/hooks'
6import { MUserAccountId, MVideoImport } from '@server/types/models' 6import { MUserAccountId, MVideoImport } from '@server/types/models'
7import { forceNumber } from '@shared/core-utils'
7import { HttpStatusCode, UserRight, VideoImportState } from '@shared/models' 8import { HttpStatusCode, UserRight, VideoImportState } from '@shared/models'
8import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model' 9import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model'
9import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc' 10import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc'
@@ -130,7 +131,7 @@ const videoImportCancelValidator = [
130 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 131 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
131 if (areValidationErrors(req, res)) return 132 if (areValidationErrors(req, res)) return
132 133
133 if (!await doesVideoImportExist(parseInt(req.params.id), res)) return 134 if (!await doesVideoImportExist(forceNumber(req.params.id), res)) return
134 if (!checkUserCanManageImport(res.locals.oauth.token.user, res.locals.videoImport, res)) return 135 if (!checkUserCanManageImport(res.locals.oauth.token.user, res.locals.videoImport, res)) return
135 136
136 if (res.locals.videoImport.state !== VideoImportState.PENDING) { 137 if (res.locals.videoImport.state !== VideoImportState.PENDING) {
diff --git a/server/middlewares/validators/videos/video-playlists.ts b/server/middlewares/validators/videos/video-playlists.ts
index 6d4b8a6f1..e4b7e5c56 100644
--- a/server/middlewares/validators/videos/video-playlists.ts
+++ b/server/middlewares/validators/videos/video-playlists.ts
@@ -2,6 +2,7 @@ import express from 'express'
2import { body, param, query, ValidationChain } from 'express-validator' 2import { body, param, query, ValidationChain } from 'express-validator'
3import { ExpressPromiseHandler } from '@server/types/express-handler' 3import { ExpressPromiseHandler } from '@server/types/express-handler'
4import { MUserAccountId } from '@server/types/models' 4import { MUserAccountId } from '@server/types/models'
5import { forceNumber } from '@shared/core-utils'
5import { 6import {
6 HttpStatusCode, 7 HttpStatusCode,
7 UserRight, 8 UserRight,
@@ -258,7 +259,7 @@ const videoPlaylistElementAPGetValidator = [
258 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 259 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
259 if (areValidationErrors(req, res)) return 260 if (areValidationErrors(req, res)) return
260 261
261 const playlistElementId = parseInt(req.params.playlistElementId + '', 10) 262 const playlistElementId = forceNumber(req.params.playlistElementId)
262 const playlistId = req.params.playlistId 263 const playlistId = req.params.playlistId
263 264
264 const videoPlaylistElement = await VideoPlaylistElementModel.loadByPlaylistAndElementIdForAP(playlistId, playlistElementId) 265 const videoPlaylistElement = await VideoPlaylistElementModel.loadByPlaylistAndElementIdForAP(playlistId, playlistElementId)
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index 7fd2b03d1..e29eb4a32 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -7,7 +7,7 @@ import { getServerActor } from '@server/models/application/application'
7import { ExpressPromiseHandler } from '@server/types/express-handler' 7import { ExpressPromiseHandler } from '@server/types/express-handler'
8import { MUserAccountId, MVideoFullLight } from '@server/types/models' 8import { MUserAccountId, MVideoFullLight } from '@server/types/models'
9import { arrayify, getAllPrivacies } from '@shared/core-utils' 9import { arrayify, getAllPrivacies } from '@shared/core-utils'
10import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude } from '@shared/models' 10import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
11import { 11import {
12 exists, 12 exists,
13 isBooleanValid, 13 isBooleanValid,
@@ -48,6 +48,7 @@ import { Hooks } from '../../../lib/plugins/hooks'
48import { VideoModel } from '../../../models/video/video' 48import { VideoModel } from '../../../models/video/video'
49import { 49import {
50 areValidationErrors, 50 areValidationErrors,
51 checkCanAccessVideoStaticFiles,
51 checkCanSeeVideo, 52 checkCanSeeVideo,
52 checkUserCanManageVideo, 53 checkUserCanManageVideo,
53 checkUserQuota, 54 checkUserQuota,
@@ -232,6 +233,11 @@ const videosUpdateValidator = getCommonVideoEditAttributes().concat([
232 if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) 233 if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
233 if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req) 234 if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req)
234 235
236 const video = getVideoWithAttributes(res)
237 if (req.body.privacy && video.isLive && video.state !== VideoState.WAITING_FOR_LIVE) {
238 return res.fail({ message: 'Cannot update privacy of a live that has already started' })
239 }
240
235 // Check if the user who did the request is able to update the video 241 // Check if the user who did the request is able to update the video
236 const user = res.locals.oauth.token.User 242 const user = res.locals.oauth.token.User
237 if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) 243 if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
@@ -271,10 +277,7 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R
271 }) 277 })
272} 278}
273 279
274const videosCustomGetValidator = ( 280const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => {
275 fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes',
276 authenticateInQuery = false
277) => {
278 return [ 281 return [
279 isValidVideoIdParam('id'), 282 isValidVideoIdParam('id'),
280 283
@@ -287,7 +290,7 @@ const videosCustomGetValidator = (
287 290
288 const video = getVideoWithAttributes(res) as MVideoFullLight 291 const video = getVideoWithAttributes(res) as MVideoFullLight
289 292
290 if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id, authenticateInQuery })) return 293 if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return
291 294
292 return next() 295 return next()
293 } 296 }
@@ -295,7 +298,6 @@ const videosCustomGetValidator = (
295} 298}
296 299
297const videosGetValidator = videosCustomGetValidator('all') 300const videosGetValidator = videosCustomGetValidator('all')
298const videosDownloadValidator = videosCustomGetValidator('all', true)
299 301
300const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ 302const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
301 isValidVideoIdParam('id'), 303 isValidVideoIdParam('id'),
@@ -311,6 +313,21 @@ const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
311 } 313 }
312]) 314])
313 315
316const videosDownloadValidator = [
317 isValidVideoIdParam('id'),
318
319 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
320 if (areValidationErrors(req, res)) return
321 if (!await doesVideoExist(req.params.id, res, 'all')) return
322
323 const video = getVideoWithAttributes(res)
324
325 if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return
326
327 return next()
328 }
329]
330
314const videosRemoveValidator = [ 331const videosRemoveValidator = [
315 isValidVideoIdParam('id'), 332 isValidVideoIdParam('id'),
316 333
@@ -372,7 +389,7 @@ function getCommonVideoEditAttributes () {
372 .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'), 389 .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'),
373 body('privacy') 390 body('privacy')
374 .optional() 391 .optional()
375 .customSanitizer(toValueOrNull) 392 .customSanitizer(toIntOrNull)
376 .custom(isVideoPrivacyValid), 393 .custom(isVideoPrivacyValid),
377 body('description') 394 body('description')
378 .optional() 395 .optional()
diff --git a/server/models/abuse/abuse-query-builder.ts b/server/models/abuse/abuse-query-builder.ts
index cfc924ba4..74f4542e5 100644
--- a/server/models/abuse/abuse-query-builder.ts
+++ b/server/models/abuse/abuse-query-builder.ts
@@ -1,5 +1,6 @@
1 1
2import { exists } from '@server/helpers/custom-validators/misc' 2import { exists } from '@server/helpers/custom-validators/misc'
3import { forceNumber } from '@shared/core-utils'
3import { AbuseFilter, AbuseState, AbuseVideoIs } from '@shared/models' 4import { AbuseFilter, AbuseState, AbuseVideoIs } from '@shared/models'
4import { buildBlockedAccountSQL, buildDirectionAndField } from '../utils' 5import { buildBlockedAccountSQL, buildDirectionAndField } from '../utils'
5 6
@@ -135,12 +136,12 @@ function buildAbuseListQuery (options: BuildAbusesQueryOptions, type: 'count' |
135 } 136 }
136 137
137 if (exists(options.count)) { 138 if (exists(options.count)) {
138 const count = parseInt(options.count + '', 10) 139 const count = forceNumber(options.count)
139 suffix += `LIMIT ${count} ` 140 suffix += `LIMIT ${count} `
140 } 141 }
141 142
142 if (exists(options.start)) { 143 if (exists(options.start)) {
143 const start = parseInt(options.start + '', 10) 144 const start = forceNumber(options.start)
144 suffix += `OFFSET ${start} ` 145 suffix += `OFFSET ${start} `
145 } 146 }
146 } 147 }
diff --git a/server/models/abuse/abuse.ts b/server/models/abuse/abuse.ts
index f85f48e86..4c6a96a86 100644
--- a/server/models/abuse/abuse.ts
+++ b/server/models/abuse/abuse.ts
@@ -436,7 +436,7 @@ export class AbuseModel extends Model<Partial<AttributesOnly<AbuseModel>>> {
436 436
437 buildBaseVideoCommentAbuse (this: MAbuseUserFormattable) { 437 buildBaseVideoCommentAbuse (this: MAbuseUserFormattable) {
438 // Associated video comment could have been destroyed if the video has been deleted 438 // Associated video comment could have been destroyed if the video has been deleted
439 if (!this.VideoCommentAbuse || !this.VideoCommentAbuse.VideoComment) return null 439 if (!this.VideoCommentAbuse?.VideoComment) return null
440 440
441 const entity = this.VideoCommentAbuse.VideoComment 441 const entity = this.VideoCommentAbuse.VideoComment
442 442
diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts
index 88db241dc..d7afa727d 100644
--- a/server/models/actor/actor.ts
+++ b/server/models/actor/actor.ts
@@ -18,7 +18,7 @@ import {
18import { activityPubContextify } from '@server/lib/activitypub/context' 18import { activityPubContextify } from '@server/lib/activitypub/context'
19import { getBiggestActorImage } from '@server/lib/actor-image' 19import { getBiggestActorImage } from '@server/lib/actor-image'
20import { ModelCache } from '@server/models/model-cache' 20import { ModelCache } from '@server/models/model-cache'
21import { getLowercaseExtension } from '@shared/core-utils' 21import { forceNumber, getLowercaseExtension } from '@shared/core-utils'
22import { ActivityIconObject, ActivityPubActorType, ActorImageType } from '@shared/models' 22import { ActivityIconObject, ActivityPubActorType, ActorImageType } from '@shared/models'
23import { AttributesOnly } from '@shared/typescript-utils' 23import { AttributesOnly } from '@shared/typescript-utils'
24import { 24import {
@@ -446,7 +446,7 @@ export class ActorModel extends Model<Partial<AttributesOnly<ActorModel>>> {
446 } 446 }
447 447
448 static rebuildFollowsCount (ofId: number, type: 'followers' | 'following', transaction?: Transaction) { 448 static rebuildFollowsCount (ofId: number, type: 'followers' | 'following', transaction?: Transaction) {
449 const sanitizedOfId = parseInt(ofId + '', 10) 449 const sanitizedOfId = forceNumber(ofId)
450 const where = { id: sanitizedOfId } 450 const where = { id: sanitizedOfId }
451 451
452 let columnToUpdate: string 452 let columnToUpdate: string
diff --git a/server/models/server/plugin.ts b/server/models/server/plugin.ts
index fa5b4cc4b..71c205ffa 100644
--- a/server/models/server/plugin.ts
+++ b/server/models/server/plugin.ts
@@ -7,8 +7,9 @@ import {
7 isPluginDescriptionValid, 7 isPluginDescriptionValid,
8 isPluginHomepage, 8 isPluginHomepage,
9 isPluginNameValid, 9 isPluginNameValid,
10 isPluginTypeValid, 10 isPluginStableOrUnstableVersionValid,
11 isPluginVersionValid 11 isPluginStableVersionValid,
12 isPluginTypeValid
12} from '../../helpers/custom-validators/plugins' 13} from '../../helpers/custom-validators/plugins'
13import { getSort, throwIfNotValid } from '../utils' 14import { getSort, throwIfNotValid } from '../utils'
14 15
@@ -40,12 +41,12 @@ export class PluginModel extends Model<Partial<AttributesOnly<PluginModel>>> {
40 type: number 41 type: number
41 42
42 @AllowNull(false) 43 @AllowNull(false)
43 @Is('PluginVersion', value => throwIfNotValid(value, isPluginVersionValid, 'version')) 44 @Is('PluginVersion', value => throwIfNotValid(value, isPluginStableOrUnstableVersionValid, 'version'))
44 @Column 45 @Column
45 version: string 46 version: string
46 47
47 @AllowNull(true) 48 @AllowNull(true)
48 @Is('PluginLatestVersion', value => throwIfNotValid(value, isPluginVersionValid, 'version')) 49 @Is('PluginLatestVersion', value => throwIfNotValid(value, isPluginStableVersionValid, 'version'))
49 @Column 50 @Column
50 latestVersion: string 51 latestVersion: string
51 52
@@ -121,7 +122,7 @@ export class PluginModel extends Model<Partial<AttributesOnly<PluginModel>>> {
121 122
122 return PluginModel.findOne(query) 123 return PluginModel.findOne(query)
123 .then(p => { 124 .then(p => {
124 if (!p || !p.settings || p.settings === undefined) { 125 if (!p?.settings || p.settings === undefined) {
125 const registered = registeredSettings.find(s => s.name === settingName) 126 const registered = registeredSettings.find(s => s.name === settingName)
126 if (!registered || registered.default === undefined) return undefined 127 if (!registered || registered.default === undefined) return undefined
127 128
@@ -151,7 +152,7 @@ export class PluginModel extends Model<Partial<AttributesOnly<PluginModel>>> {
151 const result: SettingEntries = {} 152 const result: SettingEntries = {}
152 153
153 for (const name of settingNames) { 154 for (const name of settingNames) {
154 if (!p || !p.settings || p.settings[name] === undefined) { 155 if (!p?.settings || p.settings[name] === undefined) {
155 const registered = registeredSettings.find(s => s.name === name) 156 const registered = registeredSettings.find(s => s.name === name)
156 157
157 if (registered?.default !== undefined) { 158 if (registered?.default !== undefined) {
diff --git a/server/models/user/user-notification.ts b/server/models/user/user-notification.ts
index 6209cb4bf..d37fa5dc7 100644
--- a/server/models/user/user-notification.ts
+++ b/server/models/user/user-notification.ts
@@ -2,6 +2,7 @@ import { ModelIndexesOptions, Op, WhereOptions } from 'sequelize'
2import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' 2import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
3import { getBiggestActorImage } from '@server/lib/actor-image' 3import { getBiggestActorImage } from '@server/lib/actor-image'
4import { UserNotificationIncludes, UserNotificationModelForApi } from '@server/types/models/user' 4import { UserNotificationIncludes, UserNotificationModelForApi } from '@server/types/models/user'
5import { forceNumber } from '@shared/core-utils'
5import { uuidToShort } from '@shared/extra-utils' 6import { uuidToShort } from '@shared/extra-utils'
6import { UserNotification, UserNotificationType } from '@shared/models' 7import { UserNotification, UserNotificationType } from '@shared/models'
7import { AttributesOnly } from '@shared/typescript-utils' 8import { AttributesOnly } from '@shared/typescript-utils'
@@ -284,7 +285,7 @@ export class UserNotificationModel extends Model<Partial<AttributesOnly<UserNoti
284 } 285 }
285 286
286 static removeNotificationsOf (options: { id: number, type: 'account' | 'server', forUserId?: number }) { 287 static removeNotificationsOf (options: { id: number, type: 'account' | 'server', forUserId?: number }) {
287 const id = parseInt(options.id + '', 10) 288 const id = forceNumber(options.id)
288 289
289 function buildAccountWhereQuery (base: string) { 290 function buildAccountWhereQuery (base: string) {
290 const whereSuffix = options.forUserId 291 const whereSuffix = options.forUserId
diff --git a/server/models/user/user.ts b/server/models/user/user.ts
index 1a7c84390..672728a2a 100644
--- a/server/models/user/user.ts
+++ b/server/models/user/user.ts
@@ -70,6 +70,7 @@ import { VideoImportModel } from '../video/video-import'
70import { VideoLiveModel } from '../video/video-live' 70import { VideoLiveModel } from '../video/video-live'
71import { VideoPlaylistModel } from '../video/video-playlist' 71import { VideoPlaylistModel } from '../video/video-playlist'
72import { UserNotificationSettingModel } from './user-notification-setting' 72import { UserNotificationSettingModel } from './user-notification-setting'
73import { forceNumber } from '@shared/core-utils'
73 74
74enum ScopeNames { 75enum ScopeNames {
75 FOR_ME_API = 'FOR_ME_API', 76 FOR_ME_API = 'FOR_ME_API',
@@ -403,6 +404,11 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> {
403 @Column 404 @Column
404 lastLoginDate: Date 405 lastLoginDate: Date
405 406
407 @AllowNull(true)
408 @Default(null)
409 @Column
410 otpSecret: string
411
406 @CreatedAt 412 @CreatedAt
407 createdAt: Date 413 createdAt: Date
408 414
@@ -886,34 +892,36 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> {
886 autoPlayNextVideoPlaylist: this.autoPlayNextVideoPlaylist, 892 autoPlayNextVideoPlaylist: this.autoPlayNextVideoPlaylist,
887 videoLanguages: this.videoLanguages, 893 videoLanguages: this.videoLanguages,
888 894
889 role: this.role, 895 role: {
890 roleLabel: USER_ROLE_LABELS[this.role], 896 id: this.role,
897 label: USER_ROLE_LABELS[this.role]
898 },
891 899
892 videoQuota: this.videoQuota, 900 videoQuota: this.videoQuota,
893 videoQuotaDaily: this.videoQuotaDaily, 901 videoQuotaDaily: this.videoQuotaDaily,
894 902
895 videoQuotaUsed: videoQuotaUsed !== undefined 903 videoQuotaUsed: videoQuotaUsed !== undefined
896 ? parseInt(videoQuotaUsed + '', 10) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id) 904 ? forceNumber(videoQuotaUsed) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id)
897 : undefined, 905 : undefined,
898 906
899 videoQuotaUsedDaily: videoQuotaUsedDaily !== undefined 907 videoQuotaUsedDaily: videoQuotaUsedDaily !== undefined
900 ? parseInt(videoQuotaUsedDaily + '', 10) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id) 908 ? forceNumber(videoQuotaUsedDaily) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id)
901 : undefined, 909 : undefined,
902 910
903 videosCount: videosCount !== undefined 911 videosCount: videosCount !== undefined
904 ? parseInt(videosCount + '', 10) 912 ? forceNumber(videosCount)
905 : undefined, 913 : undefined,
906 abusesCount: abusesCount 914 abusesCount: abusesCount
907 ? parseInt(abusesCount, 10) 915 ? forceNumber(abusesCount)
908 : undefined, 916 : undefined,
909 abusesAcceptedCount: abusesAcceptedCount 917 abusesAcceptedCount: abusesAcceptedCount
910 ? parseInt(abusesAcceptedCount, 10) 918 ? forceNumber(abusesAcceptedCount)
911 : undefined, 919 : undefined,
912 abusesCreatedCount: abusesCreatedCount !== undefined 920 abusesCreatedCount: abusesCreatedCount !== undefined
913 ? parseInt(abusesCreatedCount + '', 10) 921 ? forceNumber(abusesCreatedCount)
914 : undefined, 922 : undefined,
915 videoCommentsCount: videoCommentsCount !== undefined 923 videoCommentsCount: videoCommentsCount !== undefined
916 ? parseInt(videoCommentsCount + '', 10) 924 ? forceNumber(videoCommentsCount)
917 : undefined, 925 : undefined,
918 926
919 noInstanceConfigWarningModal: this.noInstanceConfigWarningModal, 927 noInstanceConfigWarningModal: this.noInstanceConfigWarningModal,
@@ -935,7 +943,9 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> {
935 943
936 pluginAuth: this.pluginAuth, 944 pluginAuth: this.pluginAuth,
937 945
938 lastLoginDate: this.lastLoginDate 946 lastLoginDate: this.lastLoginDate,
947
948 twoFactorEnabled: !!this.otpSecret
939 } 949 }
940 950
941 if (parameters.withAdminFlags) { 951 if (parameters.withAdminFlags) {
diff --git a/server/models/utils.ts b/server/models/utils.ts
index 1e168d419..3476799ce 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -1,5 +1,6 @@
1import { literal, Op, OrderItem, Sequelize } from 'sequelize' 1import { literal, Op, OrderItem, Sequelize } from 'sequelize'
2import validator from 'validator' 2import validator from 'validator'
3import { forceNumber } from '@shared/core-utils'
3 4
4type SortType = { sortModel: string, sortValue: string } 5type SortType = { sortModel: string, sortValue: string }
5 6
@@ -202,7 +203,7 @@ function buildBlockedAccountSQLOptimized (columnNameJoin: string, blockerIds: nu
202} 203}
203 204
204function buildServerIdsFollowedBy (actorId: any) { 205function buildServerIdsFollowedBy (actorId: any) {
205 const actorIdNumber = parseInt(actorId + '', 10) 206 const actorIdNumber = forceNumber(actorId)
206 207
207 return '(' + 208 return '(' +
208 'SELECT "actor"."serverId" FROM "actorFollow" ' + 209 'SELECT "actor"."serverId" FROM "actorFollow" ' +
@@ -218,7 +219,7 @@ function buildWhereIdOrUUID (id: number | string) {
218function parseAggregateResult (result: any) { 219function parseAggregateResult (result: any) {
219 if (!result) return 0 220 if (!result) return 0
220 221
221 const total = parseInt(result + '', 10) 222 const total = forceNumber(result)
222 if (isNaN(total)) return 0 223 if (isNaN(total)) return 0
223 224
224 return total 225 return total
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index e1b0eb610..f285db477 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -34,6 +34,7 @@ import {
34import { 34import {
35 MServer, 35 MServer,
36 MStreamingPlaylistRedundanciesOpt, 36 MStreamingPlaylistRedundanciesOpt,
37 MUserId,
37 MVideo, 38 MVideo,
38 MVideoAP, 39 MVideoAP,
39 MVideoFile, 40 MVideoFile,
@@ -57,7 +58,7 @@ export type VideoFormattingJSONOptions = {
57} 58}
58 59
59function guessAdditionalAttributesFromQuery (query: VideosCommonQueryAfterSanitize): VideoFormattingJSONOptions { 60function guessAdditionalAttributesFromQuery (query: VideosCommonQueryAfterSanitize): VideoFormattingJSONOptions {
60 if (!query || !query.include) return {} 61 if (!query?.include) return {}
61 62
62 return { 63 return {
63 additionalAttributes: { 64 additionalAttributes: {
@@ -102,6 +103,7 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm
102 }, 103 },
103 nsfw: video.nsfw, 104 nsfw: video.nsfw,
104 105
106 truncatedDescription: video.getTruncatedDescription(),
105 description: options && options.completeDescription === true 107 description: options && options.completeDescription === true
106 ? video.description 108 ? video.description
107 : video.getTruncatedDescription(), 109 : video.getTruncatedDescription(),
@@ -180,6 +182,7 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid
180 const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON') 182 const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON')
181 183
182 const videoJSON = video.toFormattedJSON({ 184 const videoJSON = video.toFormattedJSON({
185 completeDescription: true,
183 additionalAttributes: { 186 additionalAttributes: {
184 scheduledUpdate: true, 187 scheduledUpdate: true,
185 blacklistInfo: true, 188 blacklistInfo: true,
@@ -245,8 +248,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) {
245function videoFilesModelToFormattedJSON ( 248function videoFilesModelToFormattedJSON (
246 video: MVideoFormattable, 249 video: MVideoFormattable,
247 videoFiles: MVideoFileRedundanciesOpt[], 250 videoFiles: MVideoFileRedundanciesOpt[],
248 includeMagnet = true 251 options: {
252 includeMagnet?: boolean // default true
253 } = {}
249): VideoFile[] { 254): VideoFile[] {
255 const { includeMagnet = true } = options
256
250 const trackerUrls = includeMagnet 257 const trackerUrls = includeMagnet
251 ? video.getTrackerUrls() 258 ? video.getTrackerUrls()
252 : [] 259 : []
@@ -281,11 +288,14 @@ function videoFilesModelToFormattedJSON (
281 }) 288 })
282} 289}
283 290
284function addVideoFilesInAPAcc ( 291function addVideoFilesInAPAcc (options: {
285 acc: ActivityUrlObject[] | ActivityTagObject[], 292 acc: ActivityUrlObject[] | ActivityTagObject[]
286 video: MVideo, 293 video: MVideo
287 files: MVideoFile[] 294 files: MVideoFile[]
288) { 295 user?: MUserId
296}) {
297 const { acc, video, files } = options
298
289 const trackerUrls = video.getTrackerUrls() 299 const trackerUrls = video.getTrackerUrls()
290 300
291 const sortedFiles = (files || []) 301 const sortedFiles = (files || [])
@@ -370,7 +380,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
370 } 380 }
371 ] 381 ]
372 382
373 addVideoFilesInAPAcc(url, video, video.VideoFiles || []) 383 addVideoFilesInAPAcc({ acc: url, video, files: video.VideoFiles || [] })
374 384
375 for (const playlist of (video.VideoStreamingPlaylists || [])) { 385 for (const playlist of (video.VideoStreamingPlaylists || [])) {
376 const tag = playlist.p2pMediaLoaderInfohashes 386 const tag = playlist.p2pMediaLoaderInfohashes
@@ -382,7 +392,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
382 href: playlist.getSha256SegmentsUrl(video) 392 href: playlist.getSha256SegmentsUrl(video)
383 }) 393 })
384 394
385 addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) 395 addVideoFilesInAPAcc({ acc: tag, video, files: playlist.VideoFiles || [] })
386 396
387 url.push({ 397 url.push({
388 type: 'Link', 398 type: 'Link',
diff --git a/server/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/models/video/sql/video/shared/abstract-video-query-builder.ts
index 3c74b0ea6..f0ce69501 100644
--- a/server/models/video/sql/video/shared/abstract-video-query-builder.ts
+++ b/server/models/video/sql/video/shared/abstract-video-query-builder.ts
@@ -302,7 +302,7 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {
302 } 302 }
303 303
304 protected buildAttributesObject (prefixKey: string, attributeKeys: string[]) { 304 protected buildAttributesObject (prefixKey: string, attributeKeys: string[]) {
305 const result: { [id: string]: string} = {} 305 const result: { [id: string]: string } = {}
306 306
307 const prefixValue = prefixKey.replace(/->/g, '.') 307 const prefixValue = prefixKey.replace(/->/g, '.')
308 308
diff --git a/server/models/video/sql/video/videos-id-list-query-builder.ts b/server/models/video/sql/video/videos-id-list-query-builder.ts
index 14f903851..7c864bf27 100644
--- a/server/models/video/sql/video/videos-id-list-query-builder.ts
+++ b/server/models/video/sql/video/videos-id-list-query-builder.ts
@@ -6,6 +6,7 @@ import { buildDirectionAndField, createSafeIn, parseRowCountResult } from '@serv
6import { MUserAccountId, MUserId } from '@server/types/models' 6import { MUserAccountId, MUserId } from '@server/types/models'
7import { VideoInclude, VideoPrivacy, VideoState } from '@shared/models' 7import { VideoInclude, VideoPrivacy, VideoState } from '@shared/models'
8import { AbstractRunQuery } from '../../../shared/abstract-run-query' 8import { AbstractRunQuery } from '../../../shared/abstract-run-query'
9import { forceNumber } from '@shared/core-utils'
9 10
10/** 11/**
11 * 12 *
@@ -689,12 +690,12 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {
689 } 690 }
690 691
691 private setLimit (countArg: number) { 692 private setLimit (countArg: number) {
692 const count = parseInt(countArg + '', 10) 693 const count = forceNumber(countArg)
693 this.limit = `LIMIT ${count}` 694 this.limit = `LIMIT ${count}`
694 } 695 }
695 696
696 private setOffset (startArg: number) { 697 private setOffset (startArg: number) {
697 const start = parseInt(startArg + '', 10) 698 const start = forceNumber(startArg)
698 this.offset = `OFFSET ${start}` 699 this.offset = `OFFSET ${start}`
699 } 700 }
700} 701}
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts
index 91dafbcf1..9e461b6ca 100644
--- a/server/models/video/video-channel.ts
+++ b/server/models/video/video-channel.ts
@@ -19,7 +19,7 @@ import {
19} from 'sequelize-typescript' 19} from 'sequelize-typescript'
20import { CONFIG } from '@server/initializers/config' 20import { CONFIG } from '@server/initializers/config'
21import { MAccountActor } from '@server/types/models' 21import { MAccountActor } from '@server/types/models'
22import { pick } from '@shared/core-utils' 22import { forceNumber, pick } from '@shared/core-utils'
23import { AttributesOnly } from '@shared/typescript-utils' 23import { AttributesOnly } from '@shared/typescript-utils'
24import { ActivityPubActor } from '../../../shared/models/activitypub' 24import { ActivityPubActor } from '../../../shared/models/activitypub'
25import { VideoChannel, VideoChannelSummary } from '../../../shared/models/videos' 25import { VideoChannel, VideoChannelSummary } from '../../../shared/models/videos'
@@ -280,7 +280,7 @@ export type SummaryOptions = {
280 ] 280 ]
281 }, 281 },
282 [ScopeNames.WITH_STATS]: (options: AvailableWithStatsOptions = { daysPrior: 30 }) => { 282 [ScopeNames.WITH_STATS]: (options: AvailableWithStatsOptions = { daysPrior: 30 }) => {
283 const daysPrior = parseInt(options.daysPrior + '', 10) 283 const daysPrior = forceNumber(options.daysPrior)
284 284
285 return { 285 return {
286 attributes: { 286 attributes: {
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index d4f07f85f..9c4e6d078 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -22,8 +22,14 @@ import validator from 'validator'
22import { logger } from '@server/helpers/logger' 22import { logger } from '@server/helpers/logger'
23import { extractVideo } from '@server/helpers/video' 23import { extractVideo } from '@server/helpers/video'
24import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' 24import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
25import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' 25import {
26 getHLSPrivateFileUrl,
27 getHLSPublicFileUrl,
28 getWebTorrentPrivateFileUrl,
29 getWebTorrentPublicFileUrl
30} from '@server/lib/object-storage'
26import { getFSTorrentFilePath } from '@server/lib/paths' 31import { getFSTorrentFilePath } from '@server/lib/paths'
32import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
27import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' 33import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
28import { VideoResolution, VideoStorage } from '@shared/models' 34import { VideoResolution, VideoStorage } from '@shared/models'
29import { AttributesOnly } from '@shared/typescript-utils' 35import { AttributesOnly } from '@shared/typescript-utils'
@@ -48,6 +54,7 @@ import { doesExist } from '../shared'
48import { parseAggregateResult, throwIfNotValid } from '../utils' 54import { parseAggregateResult, throwIfNotValid } from '../utils'
49import { VideoModel } from './video' 55import { VideoModel } from './video'
50import { VideoStreamingPlaylistModel } from './video-streaming-playlist' 56import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
57import { CONFIG } from '@server/initializers/config'
51 58
52export enum ScopeNames { 59export enum ScopeNames {
53 WITH_VIDEO = 'WITH_VIDEO', 60 WITH_VIDEO = 'WITH_VIDEO',
@@ -295,6 +302,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
295 return VideoFileModel.findOne(query) 302 return VideoFileModel.findOne(query)
296 } 303 }
297 304
305 static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
306 const query = {
307 where: {
308 filename
309 }
310 }
311
312 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
313 }
314
298 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { 315 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
299 const query = { 316 const query = {
300 where: { 317 where: {
@@ -305,6 +322,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
305 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) 322 return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
306 } 323 }
307 324
325 static load (id: number): Promise<MVideoFile> {
326 return VideoFileModel.findByPk(id)
327 }
328
308 static loadWithMetadata (id: number) { 329 static loadWithMetadata (id: number) {
309 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) 330 return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
310 } 331 }
@@ -467,7 +488,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
467 } 488 }
468 489
469 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { 490 getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
470 if (this.videoId) return (this as MVideoFileVideo).Video 491 if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
471 492
472 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist 493 return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
473 } 494 }
@@ -488,7 +509,25 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
488 return !!this.videoStreamingPlaylistId 509 return !!this.videoStreamingPlaylistId
489 } 510 }
490 511
491 getObjectStorageUrl () { 512 // ---------------------------------------------------------------------------
513
514 getObjectStorageUrl (video: MVideo) {
515 if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
516 return this.getPrivateObjectStorageUrl(video)
517 }
518
519 return this.getPublicObjectStorageUrl()
520 }
521
522 private getPrivateObjectStorageUrl (video: MVideo) {
523 if (this.isHLS()) {
524 return getHLSPrivateFileUrl(video, this.filename)
525 }
526
527 return getWebTorrentPrivateFileUrl(this.filename)
528 }
529
530 private getPublicObjectStorageUrl () {
492 if (this.isHLS()) { 531 if (this.isHLS()) {
493 return getHLSPublicFileUrl(this.fileUrl) 532 return getHLSPublicFileUrl(this.fileUrl)
494 } 533 }
@@ -496,23 +535,46 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
496 return getWebTorrentPublicFileUrl(this.fileUrl) 535 return getWebTorrentPublicFileUrl(this.fileUrl)
497 } 536 }
498 537
538 // ---------------------------------------------------------------------------
539
499 getFileUrl (video: MVideo) { 540 getFileUrl (video: MVideo) {
500 if (this.storage === VideoStorage.OBJECT_STORAGE) { 541 if (video.isOwned()) {
501 return this.getObjectStorageUrl() 542 if (this.storage === VideoStorage.OBJECT_STORAGE) {
502 } 543 return this.getObjectStorageUrl(video)
544 }
503 545
504 if (!this.Video) this.Video = video as VideoModel 546 return WEBSERVER.URL + this.getFileStaticPath(video)
505 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) 547 }
506 548
507 return this.fileUrl 549 return this.fileUrl
508 } 550 }
509 551
552 // ---------------------------------------------------------------------------
553
510 getFileStaticPath (video: MVideo) { 554 getFileStaticPath (video: MVideo) {
511 if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) 555 if (this.isHLS()) return this.getHLSFileStaticPath(video)
556
557 return this.getWebTorrentFileStaticPath(video)
558 }
559
560 private getWebTorrentFileStaticPath (video: MVideo) {
561 if (isVideoInPrivateDirectory(video.privacy)) {
562 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
563 }
512 564
513 return join(STATIC_PATHS.WEBSEED, this.filename) 565 return join(STATIC_PATHS.WEBSEED, this.filename)
514 } 566 }
515 567
568 private getHLSFileStaticPath (video: MVideo) {
569 if (isVideoInPrivateDirectory(video.privacy)) {
570 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
571 }
572
573 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
574 }
575
576 // ---------------------------------------------------------------------------
577
516 getFileDownloadUrl (video: MVideoWithHost) { 578 getFileDownloadUrl (video: MVideoWithHost) {
517 const path = this.isHLS() 579 const path = this.isHLS()
518 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) 580 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
index 7497addf1..740f6b5c6 100644
--- a/server/models/video/video-job-info.ts
+++ b/server/models/video/video-job-info.ts
@@ -84,7 +84,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
84 static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { 84 static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> {
85 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } 85 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
86 86
87 const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` 87 const result = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
88 UPDATE 88 UPDATE
89 "videoJobInfo" 89 "videoJobInfo"
90 SET 90 SET
@@ -97,7 +97,9 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
97 "${column}"; 97 "${column}";
98 `, options) 98 `, options)
99 99
100 return pendingMove 100 if (result.length === 0) return undefined
101
102 return result[0].pendingMove
101 } 103 }
102 104
103 static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> { 105 static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> {
diff --git a/server/models/video/video-playlist-element.ts b/server/models/video/video-playlist-element.ts
index b45f15bd6..7181b5599 100644
--- a/server/models/video/video-playlist-element.ts
+++ b/server/models/video/video-playlist-element.ts
@@ -23,6 +23,7 @@ import {
23 MVideoPlaylistElementVideoUrlPlaylistPrivacy, 23 MVideoPlaylistElementVideoUrlPlaylistPrivacy,
24 MVideoPlaylistVideoThumbnail 24 MVideoPlaylistVideoThumbnail
25} from '@server/types/models/video/video-playlist-element' 25} from '@server/types/models/video/video-playlist-element'
26import { forceNumber } from '@shared/core-utils'
26import { AttributesOnly } from '@shared/typescript-utils' 27import { AttributesOnly } from '@shared/typescript-utils'
27import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' 28import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
28import { VideoPrivacy } from '../../../shared/models/videos' 29import { VideoPrivacy } from '../../../shared/models/videos'
@@ -185,7 +186,9 @@ export class VideoPlaylistElementModel extends Model<Partial<AttributesOnly<Vide
185 playlistId: number | string, 186 playlistId: number | string,
186 playlistElementId: number 187 playlistElementId: number
187 ): Promise<MVideoPlaylistElementVideoUrlPlaylistPrivacy> { 188 ): Promise<MVideoPlaylistElementVideoUrlPlaylistPrivacy> {
188 const playlistWhere = validator.isUUID('' + playlistId) ? { uuid: playlistId } : { id: playlistId } 189 const playlistWhere = validator.isUUID('' + playlistId)
190 ? { uuid: playlistId }
191 : { id: playlistId }
189 192
190 const query = { 193 const query = {
191 include: [ 194 include: [
@@ -262,13 +265,15 @@ export class VideoPlaylistElementModel extends Model<Partial<AttributesOnly<Vide
262 .then(position => position ? position + 1 : 1) 265 .then(position => position ? position + 1 : 1)
263 } 266 }
264 267
265 static reassignPositionOf ( 268 static reassignPositionOf (options: {
266 videoPlaylistId: number, 269 videoPlaylistId: number
267 firstPosition: number, 270 firstPosition: number
268 endPosition: number, 271 endPosition: number
269 newPosition: number, 272 newPosition: number
270 transaction?: Transaction 273 transaction?: Transaction
271 ) { 274 }) {
275 const { videoPlaylistId, firstPosition, endPosition, newPosition, transaction } = options
276
272 const query = { 277 const query = {
273 where: { 278 where: {
274 videoPlaylistId, 279 videoPlaylistId,
@@ -281,7 +286,7 @@ export class VideoPlaylistElementModel extends Model<Partial<AttributesOnly<Vide
281 validate: false // We use a literal to update the position 286 validate: false // We use a literal to update the position
282 } 287 }
283 288
284 const positionQuery = Sequelize.literal(`${newPosition} + "position" - ${firstPosition}`) 289 const positionQuery = Sequelize.literal(`${forceNumber(newPosition)} + "position" - ${forceNumber(firstPosition)}`)
285 return VideoPlaylistElementModel.update({ position: positionQuery }, query) 290 return VideoPlaylistElementModel.update({ position: positionQuery }, query)
286 } 291 }
287 292
diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts
index 81ce3dc9e..8bbe54c49 100644
--- a/server/models/video/video-playlist.ts
+++ b/server/models/video/video-playlist.ts
@@ -49,7 +49,7 @@ import {
49 MVideoPlaylistFormattable, 49 MVideoPlaylistFormattable,
50 MVideoPlaylistFull, 50 MVideoPlaylistFull,
51 MVideoPlaylistFullSummary, 51 MVideoPlaylistFullSummary,
52 MVideoPlaylistIdWithElements 52 MVideoPlaylistSummaryWithElements
53} from '../../types/models/video/video-playlist' 53} from '../../types/models/video/video-playlist'
54import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions } from '../account/account' 54import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions } from '../account/account'
55import { ActorModel } from '../actor/actor' 55import { ActorModel } from '../actor/actor'
@@ -470,9 +470,9 @@ export class VideoPlaylistModel extends Model<Partial<AttributesOnly<VideoPlayli
470 })) 470 }))
471 } 471 }
472 472
473 static listPlaylistIdsOf (accountId: number, videoIds: number[]): Promise<MVideoPlaylistIdWithElements[]> { 473 static listPlaylistSummariesOf (accountId: number, videoIds: number[]): Promise<MVideoPlaylistSummaryWithElements[]> {
474 const query = { 474 const query = {
475 attributes: [ 'id' ], 475 attributes: [ 'id', 'name', 'uuid' ],
476 where: { 476 where: {
477 ownerAccountId: accountId 477 ownerAccountId: accountId
478 }, 478 },
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts
index ca63bb2d9..f2190037e 100644
--- a/server/models/video/video-share.ts
+++ b/server/models/video/video-share.ts
@@ -1,5 +1,6 @@
1import { literal, Op, QueryTypes, Transaction } from 'sequelize' 1import { literal, Op, QueryTypes, Transaction } from 'sequelize'
2import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' 2import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript'
3import { forceNumber } from '@shared/core-utils'
3import { AttributesOnly } from '@shared/typescript-utils' 4import { AttributesOnly } from '@shared/typescript-utils'
4import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 5import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
5import { CONSTRAINTS_FIELDS } from '../../initializers/constants' 6import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
@@ -123,7 +124,7 @@ export class VideoShareModel extends Model<Partial<AttributesOnly<VideoShareMode
123 } 124 }
124 125
125 static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise<MActorDefault[]> { 126 static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise<MActorDefault[]> {
126 const safeOwnerId = parseInt(actorOwnerId + '', 10) 127 const safeOwnerId = forceNumber(actorOwnerId)
127 128
128 // /!\ On actor model 129 // /!\ On actor model
129 const query = { 130 const query = {
@@ -148,7 +149,7 @@ export class VideoShareModel extends Model<Partial<AttributesOnly<VideoShareMode
148 } 149 }
149 150
150 static loadActorsByVideoChannel (videoChannelId: number, t: Transaction): Promise<MActorDefault[]> { 151 static loadActorsByVideoChannel (videoChannelId: number, t: Transaction): Promise<MActorDefault[]> {
151 const safeChannelId = parseInt(videoChannelId + '', 10) 152 const safeChannelId = forceNumber(videoChannelId)
152 153
153 // /!\ On actor model 154 // /!\ On actor model
154 const query = { 155 const query = {
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index f587989dc..0386edf28 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -15,8 +15,10 @@ import {
15 Table, 15 Table,
16 UpdatedAt 16 UpdatedAt
17} from 'sequelize-typescript' 17} from 'sequelize-typescript'
18import { getHLSPublicFileUrl } from '@server/lib/object-storage' 18import { CONFIG } from '@server/initializers/config'
19import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage'
19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' 20import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
21import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
20import { VideoFileModel } from '@server/models/video/video-file' 22import { VideoFileModel } from '@server/models/video/video-file'
21import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' 23import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
22import { sha1 } from '@shared/extra-utils' 24import { sha1 } from '@shared/extra-utils'
@@ -244,26 +246,52 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
244 this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files) 246 this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
245 } 247 }
246 248
249 // ---------------------------------------------------------------------------
250
247 getMasterPlaylistUrl (video: MVideo) { 251 getMasterPlaylistUrl (video: MVideo) {
248 if (this.storage === VideoStorage.OBJECT_STORAGE) { 252 if (video.isOwned()) {
249 return getHLSPublicFileUrl(this.playlistUrl) 253 if (this.storage === VideoStorage.OBJECT_STORAGE) {
250 } 254 return this.getMasterPlaylistObjectStorageUrl(video)
255 }
251 256
252 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) 257 return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video)
258 }
253 259
254 return this.playlistUrl 260 return this.playlistUrl
255 } 261 }
256 262
257 getSha256SegmentsUrl (video: MVideo) { 263 private getMasterPlaylistObjectStorageUrl (video: MVideo) {
258 if (this.storage === VideoStorage.OBJECT_STORAGE) { 264 if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
259 return getHLSPublicFileUrl(this.segmentsSha256Url) 265 return getHLSPrivateFileUrl(video, this.playlistFilename)
260 } 266 }
261 267
262 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) 268 return getHLSPublicFileUrl(this.playlistUrl)
269 }
270
271 // ---------------------------------------------------------------------------
272
273 getSha256SegmentsUrl (video: MVideo) {
274 if (video.isOwned()) {
275 if (this.storage === VideoStorage.OBJECT_STORAGE) {
276 return this.getSha256SegmentsObjectStorageUrl(video)
277 }
278
279 return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video)
280 }
263 281
264 return this.segmentsSha256Url 282 return this.segmentsSha256Url
265 } 283 }
266 284
285 private getSha256SegmentsObjectStorageUrl (video: MVideo) {
286 if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) {
287 return getHLSPrivateFileUrl(video, this.segmentsSha256Filename)
288 }
289
290 return getHLSPublicFileUrl(this.segmentsSha256Url)
291 }
292
293 // ---------------------------------------------------------------------------
294
267 getStringType () { 295 getStringType () {
268 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' 296 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
269 297
@@ -283,13 +311,19 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
283 return Object.assign(this, { Video: video }) 311 return Object.assign(this, { Video: video })
284 } 312 }
285 313
286 private getMasterPlaylistStaticPath (videoUUID: string) { 314 private getMasterPlaylistStaticPath (video: MVideo) {
287 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) 315 if (isVideoInPrivateDirectory(video.privacy)) {
316 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename)
317 }
318
319 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename)
288 } 320 }
289 321
290 private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) { 322 private getSha256SegmentsStaticPath (video: MVideo) {
291 if (isLive) return join('/live', 'segments-sha256', videoUUID) 323 if (isVideoInPrivateDirectory(video.privacy)) {
324 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename)
325 }
292 326
293 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename) 327 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename)
294 } 328 }
295} 329}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 468117504..56cc45cfe 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -26,14 +26,15 @@ import {
26} from 'sequelize-typescript' 26} from 'sequelize-typescript'
27import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' 27import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
28import { LiveManager } from '@server/lib/live/live-manager' 28import { LiveManager } from '@server/lib/live/live-manager'
29import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' 29import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
30import { tracer } from '@server/lib/opentelemetry/tracing' 30import { tracer } from '@server/lib/opentelemetry/tracing'
31import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 31import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
32import { VideoPathManager } from '@server/lib/video-path-manager' 32import { VideoPathManager } from '@server/lib/video-path-manager'
33import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
33import { getServerActor } from '@server/models/application/application' 34import { getServerActor } from '@server/models/application/application'
34import { ModelCache } from '@server/models/model-cache' 35import { ModelCache } from '@server/models/model-cache'
35import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' 36import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
36import { ffprobePromise, getAudioStream, uuidToShort } from '@shared/extra-utils' 37import { ffprobePromise, getAudioStream, hasAudioStream, uuidToShort } from '@shared/extra-utils'
37import { 38import {
38 ResultList, 39 ResultList,
39 ThumbnailType, 40 ThumbnailType,
@@ -52,7 +53,7 @@ import {
52import { AttributesOnly } from '@shared/typescript-utils' 53import { AttributesOnly } from '@shared/typescript-utils'
53import { peertubeTruncate } from '../../helpers/core-utils' 54import { peertubeTruncate } from '../../helpers/core-utils'
54import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 55import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
55import { exists, isBooleanValid } from '../../helpers/custom-validators/misc' 56import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
56import { 57import {
57 isVideoDescriptionValid, 58 isVideoDescriptionValid,
58 isVideoDurationValid, 59 isVideoDurationValid,
@@ -784,9 +785,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
784 785
785 // Do not wait video deletion because we could be in a transaction 786 // Do not wait video deletion because we could be in a transaction
786 Promise.all(tasks) 787 Promise.all(tasks)
787 .catch(err => { 788 .then(() => logger.info('Removed files of video %s.', instance.url))
788 logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err }) 789 .catch(err => logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err }))
789 })
790 790
791 return undefined 791 return undefined
792 } 792 }
@@ -1458,6 +1458,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1458 const query = 'SELECT 1 FROM "videoShare" ' + 1458 const query = 'SELECT 1 FROM "videoShare" ' +
1459 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + 1459 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' +
1460 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' + 1460 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' +
1461 'UNION ' +
1462 'SELECT 1 FROM "video" ' +
1463 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' +
1464 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' +
1465 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "account"."actorId" ' +
1466 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "video"."id" = $videoId ' +
1461 'LIMIT 1' 1467 'LIMIT 1'
1462 1468
1463 const options = { 1469 const options = {
@@ -1696,12 +1702,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1696 let files: VideoFile[] = [] 1702 let files: VideoFile[] = []
1697 1703
1698 if (Array.isArray(this.VideoFiles)) { 1704 if (Array.isArray(this.VideoFiles)) {
1699 const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, includeMagnet) 1705 const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet })
1700 files = files.concat(result) 1706 files = files.concat(result)
1701 } 1707 }
1702 1708
1703 for (const p of (this.VideoStreamingPlaylists || [])) { 1709 for (const p of (this.VideoStreamingPlaylists || [])) {
1704 const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, includeMagnet) 1710 const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })
1705 files = files.concat(result) 1711 files = files.concat(result)
1706 } 1712 }
1707 1713
@@ -1745,9 +1751,11 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1745 const probe = await ffprobePromise(originalFilePath) 1751 const probe = await ffprobePromise(originalFilePath)
1746 1752
1747 const { audioStream } = await getAudioStream(originalFilePath, probe) 1753 const { audioStream } = await getAudioStream(originalFilePath, probe)
1754 const hasAudio = await hasAudioStream(originalFilePath, probe)
1748 1755
1749 return { 1756 return {
1750 audioStream, 1757 audioStream,
1758 hasAudio,
1751 1759
1752 ...await getVideoStreamDimensionsInfo(originalFilePath, probe) 1760 ...await getVideoStreamDimensionsInfo(originalFilePath, probe)
1753 } 1761 }
@@ -1764,9 +1772,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1764 const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) 1772 const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
1765 if (!playlist) return undefined 1773 if (!playlist) return undefined
1766 1774
1767 playlist.Video = this 1775 return playlist.withVideo(this)
1768
1769 return playlist
1770 } 1776 }
1771 1777
1772 setHLSPlaylist (playlist: MStreamingPlaylist) { 1778 setHLSPlaylist (playlist: MStreamingPlaylist) {
@@ -1832,8 +1838,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1832 await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) 1838 await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename))
1833 1839
1834 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { 1840 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
1835 await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename) 1841 await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), videoFile.filename)
1836 await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), resolutionFilename) 1842 await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), resolutionFilename)
1837 } 1843 }
1838 } 1844 }
1839 1845
@@ -1842,7 +1848,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1842 await remove(filePath) 1848 await remove(filePath)
1843 1849
1844 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { 1850 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
1845 await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), filename) 1851 await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), filename)
1846 } 1852 }
1847 } 1853 }
1848 1854
@@ -1868,24 +1874,39 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1868 return setAsUpdated('video', this.id, transaction) 1874 return setAsUpdated('video', this.id, transaction)
1869 } 1875 }
1870 1876
1871 requiresAuth () { 1877 // ---------------------------------------------------------------------------
1872 return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
1873 }
1874 1878
1875 setPrivacy (newPrivacy: VideoPrivacy) { 1879 requiresAuth (options: {
1876 if (this.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { 1880 urlParamId: string
1877 this.publishedAt = new Date() 1881 checkBlacklist: boolean
1882 }) {
1883 const { urlParamId, checkBlacklist } = options
1884
1885 if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) {
1886 return true
1887 }
1888
1889 if (this.privacy === VideoPrivacy.UNLISTED) {
1890 if (urlParamId && !isUUIDValid(urlParamId)) return true
1891
1892 return false
1893 }
1894
1895 if (checkBlacklist && this.VideoBlacklist) return true
1896
1897 if (this.privacy !== VideoPrivacy.PUBLIC) {
1898 throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`)
1878 } 1899 }
1879 1900
1880 this.privacy = newPrivacy 1901 return false
1881 } 1902 }
1882 1903
1883 isConfidential () { 1904 hasPrivateStaticPath () {
1884 return this.privacy === VideoPrivacy.PRIVATE || 1905 return isVideoInPrivateDirectory(this.privacy)
1885 this.privacy === VideoPrivacy.UNLISTED ||
1886 this.privacy === VideoPrivacy.INTERNAL
1887 } 1906 }
1888 1907
1908 // ---------------------------------------------------------------------------
1909
1889 async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { 1910 async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {
1890 if (this.state === newState) throw new Error('Cannot use same state ' + newState) 1911 if (this.state === newState) throw new Error('Cannot use same state ' + newState)
1891 1912
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index cd7a38459..961093bb5 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -2,6 +2,7 @@ import './abuses'
2import './accounts' 2import './accounts'
3import './blocklist' 3import './blocklist'
4import './bulk' 4import './bulk'
5import './channel-import-videos'
5import './config' 6import './config'
6import './contact-form' 7import './contact-form'
7import './custom-pages' 8import './custom-pages'
@@ -17,6 +18,7 @@ import './redundancy'
17import './search' 18import './search'
18import './services' 19import './services'
19import './transcoding' 20import './transcoding'
21import './two-factor'
20import './upload-quota' 22import './upload-quota'
21import './user-notifications' 23import './user-notifications'
22import './user-subscriptions' 24import './user-subscriptions'
@@ -24,15 +26,15 @@ import './users-admin'
24import './users' 26import './users'
25import './video-blacklist' 27import './video-blacklist'
26import './video-captions' 28import './video-captions'
29import './video-channel-syncs'
27import './video-channels' 30import './video-channels'
28import './video-comments' 31import './video-comments'
29import './video-files' 32import './video-files'
30import './video-imports' 33import './video-imports'
31import './video-channel-syncs'
32import './channel-import-videos'
33import './video-playlists' 34import './video-playlists'
34import './video-source' 35import './video-source'
35import './video-studio' 36import './video-studio'
37import './video-token'
36import './videos-common-filters' 38import './videos-common-filters'
37import './videos-history' 39import './videos-history'
38import './videos-overviews' 40import './videos-overviews'
diff --git a/server/tests/api/check-params/live.ts b/server/tests/api/check-params/live.ts
index 3f553c42b..2eff9414b 100644
--- a/server/tests/api/check-params/live.ts
+++ b/server/tests/api/check-params/live.ts
@@ -502,6 +502,23 @@ describe('Test video lives API validator', function () {
502 await stopFfmpeg(ffmpegCommand) 502 await stopFfmpeg(ffmpegCommand)
503 }) 503 })
504 504
505 it('Should fail to change live privacy if it has already started', async function () {
506 this.timeout(40000)
507
508 const live = await command.get({ videoId: video.id })
509
510 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
511
512 await command.waitUntilPublished({ videoId: video.id })
513 await server.videos.update({
514 id: video.id,
515 attributes: { privacy: VideoPrivacy.PUBLIC },
516 expectedStatus: HttpStatusCode.BAD_REQUEST_400
517 })
518
519 await stopFfmpeg(ffmpegCommand)
520 })
521
505 it('Should fail to stream twice in the save live', async function () { 522 it('Should fail to stream twice in the save live', async function () {
506 this.timeout(40000) 523 this.timeout(40000)
507 524
diff --git a/server/tests/api/check-params/two-factor.ts b/server/tests/api/check-params/two-factor.ts
new file mode 100644
index 000000000..f8365f1b5
--- /dev/null
+++ b/server/tests/api/check-params/two-factor.ts
@@ -0,0 +1,288 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { HttpStatusCode } from '@shared/models'
4import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands'
5
6describe('Test two factor API validators', function () {
7 let server: PeerTubeServer
8
9 let rootId: number
10 let rootPassword: string
11 let rootRequestToken: string
12 let rootOTPToken: string
13
14 let userId: number
15 let userToken = ''
16 let userPassword: string
17 let userRequestToken: string
18 let userOTPToken: string
19
20 // ---------------------------------------------------------------
21
22 before(async function () {
23 this.timeout(30000)
24
25 {
26 server = await createSingleServer(1)
27 await setAccessTokensToServers([ server ])
28 }
29
30 {
31 const result = await server.users.generate('user1')
32 userToken = result.token
33 userId = result.userId
34 userPassword = result.password
35 }
36
37 {
38 const { id } = await server.users.getMyInfo()
39 rootId = id
40 rootPassword = server.store.user.password
41 }
42 })
43
44 describe('When requesting two factor', function () {
45
46 it('Should fail with an unknown user id', async function () {
47 await server.twoFactor.request({ userId: 42, currentPassword: rootPassword, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
48 })
49
50 it('Should fail with an invalid user id', async function () {
51 await server.twoFactor.request({
52 userId: 'invalid' as any,
53 currentPassword: rootPassword,
54 expectedStatus: HttpStatusCode.BAD_REQUEST_400
55 })
56 })
57
58 it('Should fail to request another user two factor without the appropriate rights', async function () {
59 await server.twoFactor.request({
60 userId: rootId,
61 token: userToken,
62 currentPassword: userPassword,
63 expectedStatus: HttpStatusCode.FORBIDDEN_403
64 })
65 })
66
67 it('Should succeed to request another user two factor with the appropriate rights', async function () {
68 await server.twoFactor.request({ userId, currentPassword: rootPassword })
69 })
70
71 it('Should fail to request two factor without a password', async function () {
72 await server.twoFactor.request({
73 userId,
74 token: userToken,
75 currentPassword: undefined,
76 expectedStatus: HttpStatusCode.BAD_REQUEST_400
77 })
78 })
79
80 it('Should fail to request two factor with an incorrect password', async function () {
81 await server.twoFactor.request({
82 userId,
83 token: userToken,
84 currentPassword: rootPassword,
85 expectedStatus: HttpStatusCode.FORBIDDEN_403
86 })
87 })
88
89 it('Should succeed to request two factor without a password when targeting a remote user with an admin account', async function () {
90 await server.twoFactor.request({ userId })
91 })
92
93 it('Should fail to request two factor without a password when targeting myself with an admin account', async function () {
94 await server.twoFactor.request({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
95 await server.twoFactor.request({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
96 })
97
98 it('Should succeed to request my two factor auth', async function () {
99 {
100 const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword })
101 userRequestToken = otpRequest.requestToken
102 userOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate()
103 }
104
105 {
106 const { otpRequest } = await server.twoFactor.request({ userId: rootId, currentPassword: rootPassword })
107 rootRequestToken = otpRequest.requestToken
108 rootOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate()
109 }
110 })
111 })
112
113 describe('When confirming two factor request', function () {
114
115 it('Should fail with an unknown user id', async function () {
116 await server.twoFactor.confirmRequest({
117 userId: 42,
118 requestToken: rootRequestToken,
119 otpToken: rootOTPToken,
120 expectedStatus: HttpStatusCode.NOT_FOUND_404
121 })
122 })
123
124 it('Should fail with an invalid user id', async function () {
125 await server.twoFactor.confirmRequest({
126 userId: 'invalid' as any,
127 requestToken: rootRequestToken,
128 otpToken: rootOTPToken,
129 expectedStatus: HttpStatusCode.BAD_REQUEST_400
130 })
131 })
132
133 it('Should fail to confirm another user two factor request without the appropriate rights', async function () {
134 await server.twoFactor.confirmRequest({
135 userId: rootId,
136 token: userToken,
137 requestToken: rootRequestToken,
138 otpToken: rootOTPToken,
139 expectedStatus: HttpStatusCode.FORBIDDEN_403
140 })
141 })
142
143 it('Should fail without request token', async function () {
144 await server.twoFactor.confirmRequest({
145 userId,
146 requestToken: undefined,
147 otpToken: userOTPToken,
148 expectedStatus: HttpStatusCode.BAD_REQUEST_400
149 })
150 })
151
152 it('Should fail with an invalid request token', async function () {
153 await server.twoFactor.confirmRequest({
154 userId,
155 requestToken: 'toto',
156 otpToken: userOTPToken,
157 expectedStatus: HttpStatusCode.FORBIDDEN_403
158 })
159 })
160
161 it('Should fail with request token of another user', async function () {
162 await server.twoFactor.confirmRequest({
163 userId,
164 requestToken: rootRequestToken,
165 otpToken: userOTPToken,
166 expectedStatus: HttpStatusCode.FORBIDDEN_403
167 })
168 })
169
170 it('Should fail without an otp token', async function () {
171 await server.twoFactor.confirmRequest({
172 userId,
173 requestToken: userRequestToken,
174 otpToken: undefined,
175 expectedStatus: HttpStatusCode.BAD_REQUEST_400
176 })
177 })
178
179 it('Should fail with a bad otp token', async function () {
180 await server.twoFactor.confirmRequest({
181 userId,
182 requestToken: userRequestToken,
183 otpToken: '123456',
184 expectedStatus: HttpStatusCode.FORBIDDEN_403
185 })
186 })
187
188 it('Should succeed to confirm another user two factor request with the appropriate rights', async function () {
189 await server.twoFactor.confirmRequest({
190 userId,
191 requestToken: userRequestToken,
192 otpToken: userOTPToken
193 })
194
195 // Reinit
196 await server.twoFactor.disable({ userId, currentPassword: rootPassword })
197 })
198
199 it('Should succeed to confirm my two factor request', async function () {
200 await server.twoFactor.confirmRequest({
201 userId,
202 token: userToken,
203 requestToken: userRequestToken,
204 otpToken: userOTPToken
205 })
206 })
207
208 it('Should fail to confirm again two factor request', async function () {
209 await server.twoFactor.confirmRequest({
210 userId,
211 token: userToken,
212 requestToken: userRequestToken,
213 otpToken: userOTPToken,
214 expectedStatus: HttpStatusCode.BAD_REQUEST_400
215 })
216 })
217 })
218
219 describe('When disabling two factor', function () {
220
221 it('Should fail with an unknown user id', async function () {
222 await server.twoFactor.disable({
223 userId: 42,
224 currentPassword: rootPassword,
225 expectedStatus: HttpStatusCode.NOT_FOUND_404
226 })
227 })
228
229 it('Should fail with an invalid user id', async function () {
230 await server.twoFactor.disable({
231 userId: 'invalid' as any,
232 currentPassword: rootPassword,
233 expectedStatus: HttpStatusCode.BAD_REQUEST_400
234 })
235 })
236
237 it('Should fail to disable another user two factor without the appropriate rights', async function () {
238 await server.twoFactor.disable({
239 userId: rootId,
240 token: userToken,
241 currentPassword: userPassword,
242 expectedStatus: HttpStatusCode.FORBIDDEN_403
243 })
244 })
245
246 it('Should fail to disable two factor with an incorrect password', async function () {
247 await server.twoFactor.disable({
248 userId,
249 token: userToken,
250 currentPassword: rootPassword,
251 expectedStatus: HttpStatusCode.FORBIDDEN_403
252 })
253 })
254
255 it('Should succeed to disable two factor without a password when targeting a remote user with an admin account', async function () {
256 await server.twoFactor.disable({ userId })
257 await server.twoFactor.requestAndConfirm({ userId })
258 })
259
260 it('Should fail to disable two factor without a password when targeting myself with an admin account', async function () {
261 await server.twoFactor.disable({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
262 await server.twoFactor.disable({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
263 })
264
265 it('Should succeed to disable another user two factor with the appropriate rights', async function () {
266 await server.twoFactor.disable({ userId, currentPassword: rootPassword })
267
268 await server.twoFactor.requestAndConfirm({ userId })
269 })
270
271 it('Should succeed to update my two factor auth', async function () {
272 await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword })
273 })
274
275 it('Should fail to disable again two factor', async function () {
276 await server.twoFactor.disable({
277 userId,
278 token: userToken,
279 currentPassword: userPassword,
280 expectedStatus: HttpStatusCode.BAD_REQUEST_400
281 })
282 })
283 })
284
285 after(async function () {
286 await cleanupTests([ server ])
287 })
288})
diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts
index aa4de2c83..9dc59a1b5 100644
--- a/server/tests/api/check-params/video-files.ts
+++ b/server/tests/api/check-params/video-files.ts
@@ -1,10 +1,12 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { HttpStatusCode, UserRole } from '@shared/models' 3import { getAllFiles } from '@shared/core-utils'
4import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@shared/models'
4import { 5import {
5 cleanupTests, 6 cleanupTests,
6 createMultipleServers, 7 createMultipleServers,
7 doubleFollow, 8 doubleFollow,
9 makeRawRequest,
8 PeerTubeServer, 10 PeerTubeServer,
9 setAccessTokensToServers, 11 setAccessTokensToServers,
10 waitJobs 12 waitJobs
@@ -13,22 +15,9 @@ import {
13describe('Test videos files', function () { 15describe('Test videos files', function () {
14 let servers: PeerTubeServer[] 16 let servers: PeerTubeServer[]
15 17
16 let webtorrentId: string
17 let hlsId: string
18 let remoteId: string
19
20 let userToken: string 18 let userToken: string
21 let moderatorToken: string 19 let moderatorToken: string
22 20
23 let validId1: string
24 let validId2: string
25
26 let hlsFileId: number
27 let webtorrentFileId: number
28
29 let remoteHLSFileId: number
30 let remoteWebtorrentFileId: number
31
32 // --------------------------------------------------------------- 21 // ---------------------------------------------------------------
33 22
34 before(async function () { 23 before(async function () {
@@ -41,117 +30,163 @@ describe('Test videos files', function () {
41 30
42 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) 31 userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
43 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) 32 moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
33 })
44 34
45 { 35 describe('Getting metadata', function () {
46 const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) 36 let video: VideoDetails
47 await waitJobs(servers) 37
38 before(async function () {
39 const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
40 video = await servers[0].videos.getWithToken({ id: uuid })
41 })
42
43 it('Should not get metadata of private video without token', async function () {
44 for (const file of getAllFiles(video)) {
45 await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
46 }
47 })
48
49 it('Should not get metadata of private video without the appropriate token', async function () {
50 for (const file of getAllFiles(video)) {
51 await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
52 }
53 })
54
55 it('Should get metadata of private video with the appropriate token', async function () {
56 for (const file of getAllFiles(video)) {
57 await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 })
58 }
59 })
60 })
61
62 describe('Deleting files', function () {
63 let webtorrentId: string
64 let hlsId: string
65 let remoteId: string
66
67 let validId1: string
68 let validId2: string
48 69
49 const video = await servers[1].videos.get({ id: uuid }) 70 let hlsFileId: number
50 remoteId = video.uuid 71 let webtorrentFileId: number
51 remoteHLSFileId = video.streamingPlaylists[0].files[0].id
52 remoteWebtorrentFileId = video.files[0].id
53 }
54 72
55 { 73 let remoteHLSFileId: number
56 await servers[0].config.enableTranscoding(true, true) 74 let remoteWebtorrentFileId: number
75
76 before(async function () {
77 this.timeout(300_000)
57 78
58 { 79 {
59 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) 80 const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
60 await waitJobs(servers) 81 await waitJobs(servers)
61 82
62 const video = await servers[0].videos.get({ id: uuid }) 83 const video = await servers[1].videos.get({ id: uuid })
63 validId1 = video.uuid 84 remoteId = video.uuid
64 hlsFileId = video.streamingPlaylists[0].files[0].id 85 remoteHLSFileId = video.streamingPlaylists[0].files[0].id
65 webtorrentFileId = video.files[0].id 86 remoteWebtorrentFileId = video.files[0].id
66 } 87 }
67 88
68 { 89 {
69 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' }) 90 await servers[0].config.enableTranscoding(true, true)
70 validId2 = uuid 91
92 {
93 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
94 await waitJobs(servers)
95
96 const video = await servers[0].videos.get({ id: uuid })
97 validId1 = video.uuid
98 hlsFileId = video.streamingPlaylists[0].files[0].id
99 webtorrentFileId = video.files[0].id
100 }
101
102 {
103 const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' })
104 validId2 = uuid
105 }
71 } 106 }
72 }
73 107
74 await waitJobs(servers) 108 await waitJobs(servers)
75 109
76 { 110 {
77 await servers[0].config.enableTranscoding(false, true) 111 await servers[0].config.enableTranscoding(false, true)
78 const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) 112 const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
79 hlsId = uuid 113 hlsId = uuid
80 } 114 }
81 115
82 await waitJobs(servers) 116 await waitJobs(servers)
83 117
84 { 118 {
85 await servers[0].config.enableTranscoding(false, true) 119 await servers[0].config.enableTranscoding(false, true)
86 const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' }) 120 const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
87 webtorrentId = uuid 121 webtorrentId = uuid
88 } 122 }
89 123
90 await waitJobs(servers) 124 await waitJobs(servers)
91 }) 125 })
92 126
93 it('Should not delete files of a unknown video', async function () { 127 it('Should not delete files of a unknown video', async function () {
94 const expectedStatus = HttpStatusCode.NOT_FOUND_404 128 const expectedStatus = HttpStatusCode.NOT_FOUND_404
95 129
96 await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) 130 await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
97 await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus }) 131 await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
98 132
99 await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) 133 await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
100 await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus }) 134 await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
101 }) 135 })
102 136
103 it('Should not delete unknown files', async function () { 137 it('Should not delete unknown files', async function () {
104 const expectedStatus = HttpStatusCode.NOT_FOUND_404 138 const expectedStatus = HttpStatusCode.NOT_FOUND_404
105 139
106 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus }) 140 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
107 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) 141 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
108 }) 142 })
109 143
110 it('Should not delete files of a remote video', async function () { 144 it('Should not delete files of a remote video', async function () {
111 const expectedStatus = HttpStatusCode.BAD_REQUEST_400 145 const expectedStatus = HttpStatusCode.BAD_REQUEST_400
112 146
113 await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) 147 await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
114 await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus }) 148 await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
115 149
116 await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) 150 await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
117 await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus }) 151 await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
118 }) 152 })
119 153
120 it('Should not delete files by a non admin user', async function () { 154 it('Should not delete files by a non admin user', async function () {
121 const expectedStatus = HttpStatusCode.FORBIDDEN_403 155 const expectedStatus = HttpStatusCode.FORBIDDEN_403
122 156
123 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) 157 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
124 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) 158 await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
125 159
126 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) 160 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
127 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) 161 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
128 162
129 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) 163 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
130 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) 164 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
131 165
132 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus }) 166 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
133 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus }) 167 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
134 }) 168 })
135 169
136 it('Should not delete files if the files are not available', async function () { 170 it('Should not delete files if the files are not available', async function () {
137 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 171 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
138 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 172 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
139 173
140 await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 174 await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
141 await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 175 await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
142 }) 176 })
143 177
144 it('Should not delete files if no both versions are available', async function () { 178 it('Should not delete files if no both versions are available', async function () {
145 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 179 await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
146 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) 180 await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
147 }) 181 })
148 182
149 it('Should delete files if both versions are available', async function () { 183 it('Should delete files if both versions are available', async function () {
150 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) 184 await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
151 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId }) 185 await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
152 186
153 await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) 187 await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
154 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 }) 188 await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
189 })
155 }) 190 })
156 191
157 after(async function () { 192 after(async function () {
diff --git a/server/tests/api/check-params/video-token.ts b/server/tests/api/check-params/video-token.ts
new file mode 100644
index 000000000..7acb9d580
--- /dev/null
+++ b/server/tests/api/check-params/video-token.ts
@@ -0,0 +1,44 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { HttpStatusCode, VideoPrivacy } from '@shared/models'
4import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
5
6describe('Test video tokens', function () {
7 let server: PeerTubeServer
8 let videoId: string
9 let userToken: string
10
11 // ---------------------------------------------------------------
12
13 before(async function () {
14 this.timeout(300_000)
15
16 server = await createSingleServer(1)
17 await setAccessTokensToServers([ server ])
18
19 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
20 videoId = uuid
21
22 userToken = await server.users.generateUserAndToken('user1')
23 })
24
25 it('Should not generate tokens for unauthenticated user', async function () {
26 await server.videoToken.create({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
27 })
28
29 it('Should not generate tokens of unknown video', async function () {
30 await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
31 })
32
33 it('Should not generate tokens of a non owned video', async function () {
34 await server.videoToken.create({ videoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
35 })
36
37 it('Should generate token', async function () {
38 await server.videoToken.create({ videoId })
39 })
40
41 after(async function () {
42 await cleanupTests([ server ])
43 })
44})
diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts
index 64ef73a2a..c82585a9e 100644
--- a/server/tests/api/live/live-constraints.ts
+++ b/server/tests/api/live/live-constraints.ts
@@ -49,7 +49,7 @@ describe('Test live constraints', function () {
49 expect(video.duration).to.be.greaterThan(0) 49 expect(video.duration).to.be.greaterThan(0)
50 } 50 }
51 51
52 await checkLiveCleanup(servers[0], videoId, resolutions) 52 await checkLiveCleanup({ server: servers[0], permanent: false, videoUUID: videoId, savedResolutions: resolutions })
53 } 53 }
54 54
55 function updateQuota (options: { total: number, daily: number }) { 55 function updateQuota (options: { total: number, daily: number }) {
diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts
index 502959258..c0bb8d529 100644
--- a/server/tests/api/live/live-fast-restream.ts
+++ b/server/tests/api/live/live-fast-restream.ts
@@ -43,12 +43,31 @@ describe('Fast restream in live', function () {
43 // Streaming session #1 43 // Streaming session #1
44 let ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) 44 let ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions)
45 await server.live.waitUntilPublished({ videoId: liveVideoUUID }) 45 await server.live.waitUntilPublished({ videoId: liveVideoUUID })
46
47 const video = await server.videos.get({ id: liveVideoUUID })
48 const session1PlaylistId = video.streamingPlaylists[0].id
49
46 await stopFfmpeg(ffmpegCommand) 50 await stopFfmpeg(ffmpegCommand)
47 await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) 51 await server.live.waitUntilWaiting({ videoId: liveVideoUUID })
48 52
49 // Streaming session #2 53 // Streaming session #2
50 ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) 54 ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions)
51 await server.live.waitUntilSegmentGeneration({ videoUUID: liveVideoUUID, segment: 0, playlistNumber: 0, totalSessions: 2 }) 55
56 let hasNewPlaylist = false
57 do {
58 const video = await server.videos.get({ id: liveVideoUUID })
59 hasNewPlaylist = video.streamingPlaylists.length === 1 && video.streamingPlaylists[0].id !== session1PlaylistId
60
61 await wait(100)
62 } while (!hasNewPlaylist)
63
64 await server.live.waitUntilSegmentGeneration({
65 server,
66 videoUUID: liveVideoUUID,
67 segment: 1,
68 playlistNumber: 0,
69 objectStorage: false
70 })
52 71
53 return { ffmpegCommand, liveVideoUUID } 72 return { ffmpegCommand, liveVideoUUID }
54 } 73 }
@@ -59,9 +78,9 @@ describe('Fast restream in live', function () {
59 const video = await server.videos.get({ id: liveId }) 78 const video = await server.videos.get({ id: liveId })
60 expect(video.streamingPlaylists).to.have.lengthOf(1) 79 expect(video.streamingPlaylists).to.have.lengthOf(1)
61 80
62 await server.live.getSegment({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) 81 await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 })
63 await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) 82 await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
64 await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) 83 await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
65 84
66 await wait(100) 85 await wait(100)
67 } 86 }
@@ -111,7 +130,7 @@ describe('Fast restream in live', function () {
111 }) 130 })
112 131
113 it('Should correctly fast reastream in a permanent live with and without save replay', async function () { 132 it('Should correctly fast reastream in a permanent live with and without save replay', async function () {
114 this.timeout(240000) 133 this.timeout(480000)
115 134
116 // A test can take a long time, so prefer to run them in parallel 135 // A test can take a long time, so prefer to run them in parallel
117 await Promise.all([ 136 await Promise.all([
diff --git a/server/tests/api/live/live-permanent.ts b/server/tests/api/live/live-permanent.ts
index 5d227200e..4203b1bfc 100644
--- a/server/tests/api/live/live-permanent.ts
+++ b/server/tests/api/live/live-permanent.ts
@@ -1,6 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { checkLiveCleanup } from '@server/tests/shared'
4import { wait } from '@shared/core-utils' 5import { wait } from '@shared/core-utils'
5import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' 6import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
6import { 7import {
@@ -129,6 +130,8 @@ describe('Permanent live', function () {
129 130
130 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) 131 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
131 } 132 }
133
134 await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID })
132 }) 135 })
133 136
134 it('Should have set this live to waiting for live state', async function () { 137 it('Should have set this live to waiting for live state', async function () {
@@ -186,6 +189,15 @@ describe('Permanent live', function () {
186 } 189 }
187 }) 190 })
188 191
192 it('Should remove the live and have cleaned up the directory', async function () {
193 this.timeout(60000)
194
195 await servers[0].videos.remove({ id: videoUUID })
196 await waitJobs(servers)
197
198 await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID })
199 })
200
189 after(async function () { 201 after(async function () {
190 await cleanupTests(servers) 202 await cleanupTests(servers)
191 }) 203 })
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 7014292d0..8f17b4566 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -186,7 +186,7 @@ describe('Save replay setting', function () {
186 await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) 186 await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
187 187
188 // No resolutions saved since we did not save replay 188 // No resolutions saved since we did not save replay
189 await checkLiveCleanup(servers[0], liveVideoUUID, []) 189 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
190 }) 190 })
191 191
192 it('Should have appropriate ended session', async function () { 192 it('Should have appropriate ended session', async function () {
@@ -220,7 +220,7 @@ describe('Save replay setting', function () {
220 220
221 await wait(5000) 221 await wait(5000)
222 await waitJobs(servers) 222 await waitJobs(servers)
223 await checkLiveCleanup(servers[0], liveVideoUUID, []) 223 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
224 }) 224 })
225 225
226 it('Should have blacklisted session error', async function () { 226 it('Should have blacklisted session error', async function () {
@@ -238,7 +238,7 @@ describe('Save replay setting', function () {
238 await publishLiveAndDelete({ permanent: false, replay: false }) 238 await publishLiveAndDelete({ permanent: false, replay: false })
239 239
240 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 240 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
241 await checkLiveCleanup(servers[0], liveVideoUUID, []) 241 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
242 }) 242 })
243 }) 243 })
244 244
@@ -317,7 +317,7 @@ describe('Save replay setting', function () {
317 }) 317 })
318 318
319 it('Should have cleaned up the live files', async function () { 319 it('Should have cleaned up the live files', async function () {
320 await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) 320 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
321 }) 321 })
322 322
323 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { 323 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@@ -332,7 +332,7 @@ describe('Save replay setting', function () {
332 332
333 await wait(5000) 333 await wait(5000)
334 await waitJobs(servers) 334 await waitJobs(servers)
335 await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) 335 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
336 }) 336 })
337 337
338 it('Should correctly terminate the stream on delete and delete the video', async function () { 338 it('Should correctly terminate the stream on delete and delete the video', async function () {
@@ -341,7 +341,7 @@ describe('Save replay setting', function () {
341 await publishLiveAndDelete({ permanent: false, replay: true }) 341 await publishLiveAndDelete({ permanent: false, replay: true })
342 342
343 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 343 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
344 await checkLiveCleanup(servers[0], liveVideoUUID, []) 344 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
345 }) 345 })
346 }) 346 })
347 347
@@ -413,7 +413,7 @@ describe('Save replay setting', function () {
413 }) 413 })
414 414
415 it('Should have cleaned up the live files', async function () { 415 it('Should have cleaned up the live files', async function () {
416 await checkLiveCleanup(servers[0], liveVideoUUID, []) 416 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
417 }) 417 })
418 418
419 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { 419 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@@ -432,7 +432,7 @@ describe('Save replay setting', function () {
432 await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 432 await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
433 } 433 }
434 434
435 await checkLiveCleanup(servers[0], liveVideoUUID, []) 435 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
436 }) 436 })
437 437
438 it('Should correctly terminate the stream on delete and not save the video', async function () { 438 it('Should correctly terminate the stream on delete and not save the video', async function () {
@@ -444,7 +444,7 @@ describe('Save replay setting', function () {
444 expect(replay).to.not.exist 444 expect(replay).to.not.exist
445 445
446 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 446 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
447 await checkLiveCleanup(servers[0], liveVideoUUID, []) 447 await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
448 }) 448 })
449 }) 449 })
450 450
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index c436f0f01..003cc934f 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -3,7 +3,7 @@
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { basename, join } from 'path' 4import { basename, join } from 'path'
5import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg' 5import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
6import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared' 6import { testImage, testVideoResolutions } from '@server/tests/shared'
7import { getAllFiles, wait } from '@shared/core-utils' 7import { getAllFiles, wait } from '@shared/core-utils'
8import { 8import {
9 HttpStatusCode, 9 HttpStatusCode,
@@ -21,6 +21,7 @@ import {
21 doubleFollow, 21 doubleFollow,
22 killallServers, 22 killallServers,
23 LiveCommand, 23 LiveCommand,
24 makeGetRequest,
24 makeRawRequest, 25 makeRawRequest,
25 PeerTubeServer, 26 PeerTubeServer,
26 sendRTMPStream, 27 sendRTMPStream,
@@ -157,8 +158,8 @@ describe('Test live', function () {
157 expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) 158 expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
158 expect(video.nsfw).to.be.true 159 expect(video.nsfw).to.be.true
159 160
160 await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200) 161 await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
161 await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200) 162 await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
162 } 163 }
163 }) 164 })
164 165
@@ -372,46 +373,6 @@ describe('Test live', function () {
372 return uuid 373 return uuid
373 } 374 }
374 375
375 async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
376 for (const server of servers) {
377 const { data } = await server.videos.list()
378 expect(data.find(v => v.uuid === liveVideoId)).to.exist
379
380 const video = await server.videos.get({ id: liveVideoId })
381
382 expect(video.streamingPlaylists).to.have.lengthOf(1)
383
384 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
385 expect(hlsPlaylist).to.exist
386
387 // Only finite files are displayed
388 expect(hlsPlaylist.files).to.have.lengthOf(0)
389
390 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
391
392 for (let i = 0; i < resolutions.length; i++) {
393 const segmentNum = 3
394 const segmentName = `${i}-00000${segmentNum}.ts`
395 await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum })
396
397 const subPlaylist = await servers[0].streamingPlaylists.get({
398 url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`
399 })
400
401 expect(subPlaylist).to.contain(segmentName)
402
403 const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
404 await checkLiveSegmentHash({
405 server,
406 baseUrlSegment: baseUrlAndPath,
407 videoUUID: video.uuid,
408 segmentName,
409 hlsPlaylist
410 })
411 }
412 }
413 }
414
415 function updateConf (resolutions: number[]) { 376 function updateConf (resolutions: number[]) {
416 return servers[0].config.updateCustomSubConfig({ 377 return servers[0].config.updateCustomSubConfig({
417 newConfig: { 378 newConfig: {
@@ -449,7 +410,14 @@ describe('Test live', function () {
449 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 410 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
450 await waitJobs(servers) 411 await waitJobs(servers)
451 412
452 await testVideoResolutions(liveVideoId, [ 720 ]) 413 await testVideoResolutions({
414 originServer: servers[0],
415 servers,
416 liveVideoId,
417 resolutions: [ 720 ],
418 objectStorage: false,
419 transcoded: true
420 })
453 421
454 await stopFfmpeg(ffmpegCommand) 422 await stopFfmpeg(ffmpegCommand)
455 }) 423 })
@@ -477,7 +445,14 @@ describe('Test live', function () {
477 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 445 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
478 await waitJobs(servers) 446 await waitJobs(servers)
479 447
480 await testVideoResolutions(liveVideoId, resolutions.concat([ 720 ])) 448 await testVideoResolutions({
449 originServer: servers[0],
450 servers,
451 liveVideoId,
452 resolutions: resolutions.concat([ 720 ]),
453 objectStorage: false,
454 transcoded: true
455 })
481 456
482 await stopFfmpeg(ffmpegCommand) 457 await stopFfmpeg(ffmpegCommand)
483 }) 458 })
@@ -522,7 +497,14 @@ describe('Test live', function () {
522 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 497 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
523 await waitJobs(servers) 498 await waitJobs(servers)
524 499
525 await testVideoResolutions(liveVideoId, resolutions) 500 await testVideoResolutions({
501 originServer: servers[0],
502 servers,
503 liveVideoId,
504 resolutions,
505 objectStorage: false,
506 transcoded: true
507 })
526 508
527 await stopFfmpeg(ffmpegCommand) 509 await stopFfmpeg(ffmpegCommand)
528 await commands[0].waitUntilEnded({ videoId: liveVideoId }) 510 await commands[0].waitUntilEnded({ videoId: liveVideoId })
@@ -538,7 +520,7 @@ describe('Test live', function () {
538 } 520 }
539 521
540 const minBitrateLimits = { 522 const minBitrateLimits = {
541 720: 5500 * 1000, 523 720: 4800 * 1000,
542 360: 1000 * 1000, 524 360: 1000 * 1000,
543 240: 550 * 1000 525 240: 550 * 1000
544 } 526 }
@@ -551,8 +533,8 @@ describe('Test live', function () {
551 expect(video.files).to.have.lengthOf(0) 533 expect(video.files).to.have.lengthOf(0)
552 534
553 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) 535 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
554 await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) 536 await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
555 await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) 537 await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
556 538
557 // We should have generated random filenames 539 // We should have generated random filenames
558 expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') 540 expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
@@ -569,7 +551,7 @@ describe('Test live', function () {
569 if (resolution >= 720) { 551 if (resolution >= 720) {
570 expect(file.fps).to.be.approximately(60, 10) 552 expect(file.fps).to.be.approximately(60, 10)
571 } else { 553 } else {
572 expect(file.fps).to.be.approximately(30, 2) 554 expect(file.fps).to.be.approximately(30, 3)
573 } 555 }
574 556
575 const filename = basename(file.fileUrl) 557 const filename = basename(file.fileUrl)
@@ -583,8 +565,8 @@ describe('Test live', function () {
583 expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height]) 565 expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
584 expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height]) 566 expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
585 567
586 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) 568 await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
587 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 569 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
588 } 570 }
589 } 571 }
590 }) 572 })
@@ -611,7 +593,14 @@ describe('Test live', function () {
611 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 593 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
612 await waitJobs(servers) 594 await waitJobs(servers)
613 595
614 await testVideoResolutions(liveVideoId, resolutions) 596 await testVideoResolutions({
597 originServer: servers[0],
598 servers,
599 liveVideoId,
600 resolutions,
601 objectStorage: false,
602 transcoded: true
603 })
615 604
616 await stopFfmpeg(ffmpegCommand) 605 await stopFfmpeg(ffmpegCommand)
617 await commands[0].waitUntilEnded({ videoId: liveVideoId }) 606 await commands[0].waitUntilEnded({ videoId: liveVideoId })
@@ -640,7 +629,14 @@ describe('Test live', function () {
640 await waitUntilLivePublishedOnAllServers(servers, liveVideoId) 629 await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
641 await waitJobs(servers) 630 await waitJobs(servers)
642 631
643 await testVideoResolutions(liveVideoId, [ 720 ]) 632 await testVideoResolutions({
633 originServer: servers[0],
634 servers,
635 liveVideoId,
636 resolutions: [ 720 ],
637 objectStorage: false,
638 transcoded: true
639 })
644 640
645 await stopFfmpeg(ffmpegCommand) 641 await stopFfmpeg(ffmpegCommand)
646 await commands[0].waitUntilEnded({ videoId: liveVideoId }) 642 await commands[0].waitUntilEnded({ videoId: liveVideoId })
@@ -700,9 +696,15 @@ describe('Test live', function () {
700 commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) 696 commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
701 ]) 697 ])
702 698
703 await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoId, playlistNumber: 0, segment: 2 }) 699 for (const videoUUID of [ liveVideoId, liveVideoReplayId, permanentLiveVideoReplayId ]) {
704 await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoReplayId, playlistNumber: 0, segment: 2 }) 700 await commands[0].waitUntilSegmentGeneration({
705 await commands[0].waitUntilSegmentGeneration({ videoUUID: permanentLiveVideoReplayId, playlistNumber: 0, segment: 2 }) 701 server: servers[0],
702 videoUUID,
703 playlistNumber: 0,
704 segment: 2,
705 objectStorage: false
706 })
707 }
706 708
707 { 709 {
708 const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId }) 710 const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId })
diff --git a/server/tests/api/notifications/admin-notifications.ts b/server/tests/api/notifications/admin-notifications.ts
index b3bb4888e..07c981a37 100644
--- a/server/tests/api/notifications/admin-notifications.ts
+++ b/server/tests/api/notifications/admin-notifications.ts
@@ -37,7 +37,7 @@ describe('Test admin notifications', function () {
37 plugins: { 37 plugins: {
38 index: { 38 index: {
39 enabled: true, 39 enabled: true,
40 check_latest_versions_interval: '5 seconds' 40 check_latest_versions_interval: '3 seconds'
41 } 41 }
42 } 42 }
43 } 43 }
@@ -62,7 +62,7 @@ describe('Test admin notifications', function () {
62 62
63 describe('Latest PeerTube version notification', function () { 63 describe('Latest PeerTube version notification', function () {
64 64
65 it('Should not send a notification to admins if there is not a new version', async function () { 65 it('Should not send a notification to admins if there is no new version', async function () {
66 this.timeout(30000) 66 this.timeout(30000)
67 67
68 joinPeerTubeServer.setLatestVersion('1.4.2') 68 joinPeerTubeServer.setLatestVersion('1.4.2')
@@ -71,7 +71,7 @@ describe('Test admin notifications', function () {
71 await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' }) 71 await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' })
72 }) 72 })
73 73
74 it('Should send a notification to admins on new plugin version', async function () { 74 it('Should send a notification to admins on new version', async function () {
75 this.timeout(30000) 75 this.timeout(30000)
76 76
77 joinPeerTubeServer.setLatestVersion('15.4.2') 77 joinPeerTubeServer.setLatestVersion('15.4.2')
diff --git a/server/tests/api/notifications/moderation-notifications.ts b/server/tests/api/notifications/moderation-notifications.ts
index d8a7d576e..5a632fb22 100644
--- a/server/tests/api/notifications/moderation-notifications.ts
+++ b/server/tests/api/notifications/moderation-notifications.ts
@@ -382,7 +382,7 @@ describe('Test moderation notifications', function () {
382 }) 382 })
383 383
384 it('Should send a notification only to admin when there is a new instance follower', async function () { 384 it('Should send a notification only to admin when there is a new instance follower', async function () {
385 this.timeout(20000) 385 this.timeout(60000)
386 386
387 await servers[2].follows.follow({ hosts: [ servers[0].url ] }) 387 await servers[2].follows.follow({ hosts: [ servers[0].url ] })
388 388
@@ -545,7 +545,7 @@ describe('Test moderation notifications', function () {
545 }) 545 })
546 546
547 it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { 547 it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () {
548 this.timeout(40000) 548 this.timeout(120000)
549 549
550 const updateAt = new Date(new Date().getTime() + 1000000) 550 const updateAt = new Date(new Date().getTime() + 1000000)
551 551
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts
index f319d6ef5..1f4489fa3 100644
--- a/server/tests/api/object-storage/index.ts
+++ b/server/tests/api/object-storage/index.ts
@@ -1,3 +1,4 @@
1export * from './live' 1export * from './live'
2export * from './video-imports' 2export * from './video-imports'
3export * from './video-static-file-privacy'
3export * from './videos' 4export * from './videos'
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
index 0958ffe0f..ad2b554b7 100644
--- a/server/tests/api/object-storage/live.ts
+++ b/server/tests/api/object-storage/live.ts
@@ -1,9 +1,9 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectStartWith } from '@server/tests/shared' 4import { expectStartWith, testVideoResolutions } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models'
7import { 7import {
8 createMultipleServers, 8 createMultipleServers,
9 doubleFollow, 9 doubleFollow,
@@ -35,54 +35,56 @@ async function createLive (server: PeerTubeServer, permanent: boolean) {
35 return uuid 35 return uuid
36} 36}
37 37
38async function checkFiles (files: VideoFile[]) { 38async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) {
39 for (const file of files) { 39 for (const server of servers) {
40 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 40 const video = await server.videos.get({ id: videoUUID })
41 41
42 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 42 expect(video.files).to.have.lengthOf(0)
43 } 43 expect(video.streamingPlaylists).to.have.lengthOf(1)
44}
45 44
46async function getFiles (server: PeerTubeServer, videoUUID: string) { 45 const files = video.streamingPlaylists[0].files
47 const video = await server.videos.get({ id: videoUUID }) 46 expect(files).to.have.lengthOf(numberOfFiles)
48 47
49 expect(video.files).to.have.lengthOf(0) 48 for (const file of files) {
50 expect(video.streamingPlaylists).to.have.lengthOf(1) 49 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
51 50
52 return video.streamingPlaylists[0].files 51 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
52 }
53 }
53} 54}
54 55
55async function streamAndEnd (servers: PeerTubeServer[], liveUUID: string) { 56async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) {
56 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveUUID }) 57 const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`)
57 await waitUntilLivePublishedOnAllServers(servers, liveUUID)
58
59 const videoLiveDetails = await servers[0].videos.get({ id: liveUUID })
60 const liveDetails = await servers[0].live.get({ videoId: liveUUID })
61 58
62 await stopFfmpeg(ffmpegCommand) 59 for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) {
63 60 await server.live.getPlaylistFile({
64 if (liveDetails.permanentLive) { 61 videoUUID,
65 await waitUntilLiveWaitingOnAllServers(servers, liveUUID) 62 playlistName,
66 } else { 63 expectedStatus: HttpStatusCode.NOT_FOUND_404,
67 await waitUntilLiveReplacedByReplayOnAllServers(servers, liveUUID) 64 objectStorage: true
65 })
68 } 66 }
69 67
70 await waitJobs(servers) 68 await server.live.getSegmentFile({
71 69 videoUUID,
72 return { videoLiveDetails, liveDetails } 70 playlistNumber: 0,
71 segment: 0,
72 objectStorage: true,
73 expectedStatus: HttpStatusCode.NOT_FOUND_404
74 })
73} 75}
74 76
75describe('Object storage for lives', function () { 77describe('Object storage for lives', function () {
76 if (areObjectStorageTestsDisabled()) return 78 if (areMockObjectStorageTestsDisabled()) return
77 79
78 let servers: PeerTubeServer[] 80 let servers: PeerTubeServer[]
79 81
80 before(async function () { 82 before(async function () {
81 this.timeout(120000) 83 this.timeout(120000)
82 84
83 await ObjectStorageCommand.prepareDefaultBuckets() 85 await ObjectStorageCommand.prepareDefaultMockBuckets()
84 86
85 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) 87 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig())
86 88
87 await setAccessTokensToServers(servers) 89 await setAccessTokensToServers(servers)
88 await setDefaultVideoChannel(servers) 90 await setDefaultVideoChannel(servers)
@@ -100,57 +102,124 @@ describe('Object storage for lives', function () {
100 videoUUID = await createLive(servers[0], false) 102 videoUUID = await createLive(servers[0], false)
101 }) 103 })
102 104
103 it('Should create a live and save the replay on object storage', async function () { 105 it('Should create a live and publish it on object storage', async function () {
106 this.timeout(220000)
107
108 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
109 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
110
111 await testVideoResolutions({
112 originServer: servers[0],
113 servers,
114 liveVideoId: videoUUID,
115 resolutions: [ 720 ],
116 transcoded: false,
117 objectStorage: true
118 })
119
120 await stopFfmpeg(ffmpegCommand)
121 })
122
123 it('Should have saved the replay on object storage', async function () {
104 this.timeout(220000) 124 this.timeout(220000)
105 125
106 await streamAndEnd(servers, videoUUID) 126 await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID)
127 await waitJobs(servers)
107 128
108 for (const server of servers) { 129 await checkFilesExist(servers, videoUUID, 1)
109 const files = await getFiles(server, videoUUID) 130 })
110 expect(files).to.have.lengthOf(1)
111 131
112 await checkFiles(files) 132 it('Should have cleaned up live files from object storage', async function () {
113 } 133 await checkFilesCleanup(servers[0], videoUUID, [ 720 ])
114 }) 134 })
115 }) 135 })
116 136
117 describe('With live transcoding', async function () { 137 describe('With live transcoding', async function () {
118 let videoUUIDPermanent: string 138 const resolutions = [ 720, 480, 360, 240, 144 ]
119 let videoUUIDNonPermanent: string
120 139
121 before(async function () { 140 before(async function () {
122 await servers[0].config.enableLive({ transcoding: true }) 141 await servers[0].config.enableLive({ transcoding: true })
123
124 videoUUIDPermanent = await createLive(servers[0], true)
125 videoUUIDNonPermanent = await createLive(servers[0], false)
126 }) 142 })
127 143
128 it('Should create a live and save the replay on object storage', async function () { 144 describe('Normal replay', function () {
129 this.timeout(240000) 145 let videoUUIDNonPermanent: string
146
147 before(async function () {
148 videoUUIDNonPermanent = await createLive(servers[0], false)
149 })
150
151 it('Should create a live and publish it on object storage', async function () {
152 this.timeout(240000)
153
154 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent })
155 await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent)
156
157 await testVideoResolutions({
158 originServer: servers[0],
159 servers,
160 liveVideoId: videoUUIDNonPermanent,
161 resolutions,
162 transcoded: true,
163 objectStorage: true
164 })
165
166 await stopFfmpeg(ffmpegCommand)
167 })
130 168
131 await streamAndEnd(servers, videoUUIDNonPermanent) 169 it('Should have saved the replay on object storage', async function () {
170 this.timeout(220000)
132 171
133 for (const server of servers) { 172 await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent)
134 const files = await getFiles(server, videoUUIDNonPermanent) 173 await waitJobs(servers)
135 expect(files).to.have.lengthOf(5)
136 174
137 await checkFiles(files) 175 await checkFilesExist(servers, videoUUIDNonPermanent, 5)
138 } 176 })
177
178 it('Should have cleaned up live files from object storage', async function () {
179 await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions)
180 })
139 }) 181 })
140 182
141 it('Should create a live and save the replay of permanent live on object storage', async function () { 183 describe('Permanent replay', function () {
142 this.timeout(240000) 184 let videoUUIDPermanent: string
185
186 before(async function () {
187 videoUUIDPermanent = await createLive(servers[0], true)
188 })
189
190 it('Should create a live and publish it on object storage', async function () {
191 this.timeout(240000)
192
193 const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent })
194 await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent)
195
196 await testVideoResolutions({
197 originServer: servers[0],
198 servers,
199 liveVideoId: videoUUIDPermanent,
200 resolutions,
201 transcoded: true,
202 objectStorage: true
203 })
204
205 await stopFfmpeg(ffmpegCommand)
206 })
207
208 it('Should have saved the replay on object storage', async function () {
209 this.timeout(220000)
143 210
144 const { videoLiveDetails } = await streamAndEnd(servers, videoUUIDPermanent) 211 await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent)
212 await waitJobs(servers)
145 213
146 const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) 214 const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent })
215 const replay = await findExternalSavedVideo(servers[0], videoLiveDetails)
147 216
148 for (const server of servers) { 217 await checkFilesExist(servers, replay.uuid, 5)
149 const files = await getFiles(server, replay.uuid) 218 })
150 expect(files).to.have.lengthOf(5)
151 219
152 await checkFiles(files) 220 it('Should have cleaned up live files from object storage', async function () {
153 } 221 await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions)
222 })
154 }) 223 })
155 }) 224 })
156 225
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts
index f688c7018..11c866411 100644
--- a/server/tests/api/object-storage/video-imports.ts
+++ b/server/tests/api/object-storage/video-imports.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' 4import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { 7import {
8 createSingleServer, 8 createSingleServer,
@@ -29,16 +29,16 @@ async function importVideo (server: PeerTubeServer) {
29} 29}
30 30
31describe('Object storage for video import', function () { 31describe('Object storage for video import', function () {
32 if (areObjectStorageTestsDisabled()) return 32 if (areMockObjectStorageTestsDisabled()) return
33 33
34 let server: PeerTubeServer 34 let server: PeerTubeServer
35 35
36 before(async function () { 36 before(async function () {
37 this.timeout(120000) 37 this.timeout(120000)
38 38
39 await ObjectStorageCommand.prepareDefaultBuckets() 39 await ObjectStorageCommand.prepareDefaultMockBuckets()
40 40
41 server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) 41 server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig())
42 42
43 await setAccessTokensToServers([ server ]) 43 await setAccessTokensToServers([ server ])
44 await setDefaultVideoChannel([ server ]) 44 await setDefaultVideoChannel([ server ])
@@ -64,9 +64,9 @@ describe('Object storage for video import', function () {
64 expect(video.streamingPlaylists).to.have.lengthOf(0) 64 expect(video.streamingPlaylists).to.have.lengthOf(0)
65 65
66 const fileUrl = video.files[0].fileUrl 66 const fileUrl = video.files[0].fileUrl
67 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 67 expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
68 68
69 await makeRawRequest(fileUrl, HttpStatusCode.OK_200) 69 await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
70 }) 70 })
71 }) 71 })
72 72
@@ -89,15 +89,15 @@ describe('Object storage for video import', function () {
89 expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) 89 expect(video.streamingPlaylists[0].files).to.have.lengthOf(5)
90 90
91 for (const file of video.files) { 91 for (const file of video.files) {
92 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 92 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
93 93
94 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 94 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
95 } 95 }
96 96
97 for (const file of video.streamingPlaylists[0].files) { 97 for (const file of video.streamingPlaylists[0].files) {
98 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 98 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
99 99
100 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 100 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
101 } 101 }
102 }) 102 })
103 }) 103 })
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts
new file mode 100644
index 000000000..62edd10ba
--- /dev/null
+++ b/server/tests/api/object-storage/video-static-file-privacy.ts
@@ -0,0 +1,402 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { basename } from 'path'
5import { expectStartWith } from '@server/tests/shared'
6import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils'
7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
8import {
9 cleanupTests,
10 createSingleServer,
11 findExternalSavedVideo,
12 makeRawRequest,
13 ObjectStorageCommand,
14 PeerTubeServer,
15 sendRTMPStream,
16 setAccessTokensToServers,
17 setDefaultVideoChannel,
18 stopFfmpeg,
19 waitJobs
20} from '@shared/server-commands'
21
22function extractFilenameFromUrl (url: string) {
23 const parts = basename(url).split(':')
24
25 return parts[parts.length - 1]
26}
27
28describe('Object storage for video static file privacy', function () {
29 // We need real world object storage to check ACL
30 if (areScalewayObjectStorageTestsDisabled()) return
31
32 let server: PeerTubeServer
33 let userToken: string
34
35 // ---------------------------------------------------------------------------
36
37 async function checkPrivateVODFiles (uuid: string) {
38 const video = await server.videos.getWithToken({ id: uuid })
39
40 for (const file of video.files) {
41 expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
42
43 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
44 }
45
46 for (const file of getAllFiles(video)) {
47 const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
48 expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
49 await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
50 }
51
52 const hls = getHLS(video)
53
54 if (hls) {
55 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
56 expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
57 }
58
59 await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
60 await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
61
62 for (const file of hls.files) {
63 expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
64
65 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
66 }
67 }
68 }
69
70 async function checkPublicVODFiles (uuid: string) {
71 const video = await server.videos.getWithToken({ id: uuid })
72
73 for (const file of getAllFiles(video)) {
74 expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
75
76 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
77 }
78
79 const hls = getHLS(video)
80
81 if (hls) {
82 expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
83 expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
84
85 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
86 await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
87 }
88 }
89
90 // ---------------------------------------------------------------------------
91
92 before(async function () {
93 this.timeout(120000)
94
95 server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 }))
96 await setAccessTokensToServers([ server ])
97 await setDefaultVideoChannel([ server ])
98
99 await server.config.enableMinimumTranscoding()
100
101 userToken = await server.users.generateUserAndToken('user1')
102 })
103
104 describe('VOD', function () {
105 let privateVideoUUID: string
106 let publicVideoUUID: string
107 let userPrivateVideoUUID: string
108
109 // ---------------------------------------------------------------------------
110
111 async function getSampleFileUrls (videoId: string) {
112 const video = await server.videos.getWithToken({ id: videoId })
113
114 return {
115 webTorrentFile: video.files[0].fileUrl,
116 hlsFile: getHLS(video).files[0].fileUrl
117 }
118 }
119
120 // ---------------------------------------------------------------------------
121
122 it('Should upload a private video and have appropriate object storage ACL', async function () {
123 this.timeout(60000)
124
125 {
126 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
127 privateVideoUUID = uuid
128 }
129
130 {
131 const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE })
132 userPrivateVideoUUID = uuid
133 }
134
135 await waitJobs([ server ])
136
137 await checkPrivateVODFiles(privateVideoUUID)
138 })
139
140 it('Should upload a public video and have appropriate object storage ACL', async function () {
141 this.timeout(60000)
142
143 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED })
144 await waitJobs([ server ])
145
146 publicVideoUUID = uuid
147
148 await checkPublicVODFiles(publicVideoUUID)
149 })
150
151 it('Should not get files without appropriate OAuth token', async function () {
152 this.timeout(60000)
153
154 const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID)
155
156 await makeRawRequest({ url: webTorrentFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
157 await makeRawRequest({ url: webTorrentFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
158
159 await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
160 await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
161 })
162
163 it('Should not get HLS file of another video', async function () {
164 this.timeout(60000)
165
166 const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID })
167 const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl)
168
169 const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename
170 const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename
171
172 await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
173 await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
174 })
175
176 it('Should correctly check OAuth or video file token', async function () {
177 this.timeout(60000)
178
179 const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID })
180 const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID })
181
182 const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID)
183
184 for (const url of [ webTorrentFile, hlsFile ]) {
185 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
186 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
187 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
188
189 await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
190 await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 })
191 }
192 })
193
194 it('Should update public video to private', async function () {
195 this.timeout(60000)
196
197 await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } })
198
199 await checkPrivateVODFiles(publicVideoUUID)
200 })
201
202 it('Should update private video to public', async function () {
203 this.timeout(60000)
204
205 await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
206
207 await checkPublicVODFiles(publicVideoUUID)
208 })
209 })
210
211 describe('Live', function () {
212 let normalLiveId: string
213 let normalLive: LiveVideo
214
215 let permanentLiveId: string
216 let permanentLive: LiveVideo
217
218 let unrelatedFileToken: string
219
220 // ---------------------------------------------------------------------------
221
222 async function checkLiveFiles (live: LiveVideo, liveId: string) {
223 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
224 await server.live.waitUntilPublished({ videoId: liveId })
225
226 const video = await server.videos.getWithToken({ id: liveId })
227 const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
228
229 const hls = video.streamingPlaylists[0]
230
231 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
232 expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
233
234 await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
235 await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
236
237 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
238 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
239
240 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
241 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
242 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
243 }
244
245 await stopFfmpeg(ffmpegCommand)
246 }
247
248 async function checkReplay (replay: VideoDetails) {
249 const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
250
251 const hls = replay.streamingPlaylists[0]
252 expect(hls.files).to.not.have.lengthOf(0)
253
254 for (const file of hls.files) {
255 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
256 await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
257
258 await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
259 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
260 await makeRawRequest({
261 url: file.fileUrl,
262 query: { videoFileToken: unrelatedFileToken },
263 expectedStatus: HttpStatusCode.FORBIDDEN_403
264 })
265 }
266
267 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
268 expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
269
270 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
271 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
272
273 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
274 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
275 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
276 }
277 }
278
279 // ---------------------------------------------------------------------------
280
281 before(async function () {
282 await server.config.enableMinimumTranscoding()
283
284 const { uuid } = await server.videos.quickUpload({ name: 'another video' })
285 unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
286
287 await server.config.enableLive({
288 allowReplay: true,
289 transcoding: true,
290 resolutions: 'min'
291 })
292
293 {
294 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
295 normalLiveId = video.uuid
296 normalLive = live
297 }
298
299 {
300 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
301 permanentLiveId = video.uuid
302 permanentLive = live
303 }
304 })
305
306 it('Should create a private normal live and have a private static path', async function () {
307 this.timeout(240000)
308
309 await checkLiveFiles(normalLive, normalLiveId)
310 })
311
312 it('Should create a private permanent live and have a private static path', async function () {
313 this.timeout(240000)
314
315 await checkLiveFiles(permanentLive, permanentLiveId)
316 })
317
318 it('Should have created a replay of the normal live with a private static path', async function () {
319 this.timeout(240000)
320
321 await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
322
323 const replay = await server.videos.getWithToken({ id: normalLiveId })
324 await checkReplay(replay)
325 })
326
327 it('Should have created a replay of the permanent live with a private static path', async function () {
328 this.timeout(240000)
329
330 await server.live.waitUntilWaiting({ videoId: permanentLiveId })
331 await waitJobs([ server ])
332
333 const live = await server.videos.getWithToken({ id: permanentLiveId })
334 const replayFromList = await findExternalSavedVideo(server, live)
335 const replay = await server.videos.getWithToken({ id: replayFromList.id })
336
337 await checkReplay(replay)
338 })
339 })
340
341 describe('With private files proxy disabled and public ACL for private files', function () {
342 let videoUUID: string
343
344 before(async function () {
345 this.timeout(240000)
346
347 await server.kill()
348
349 const config = ObjectStorageCommand.getDefaultScalewayConfig({
350 serverNumber: 1,
351 enablePrivateProxy: false,
352 privateACL: 'public-read'
353 })
354 await server.run(config)
355
356 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
357 videoUUID = uuid
358
359 await waitJobs([ server ])
360 })
361
362 it('Should display object storage path for a private video and be able to access them', async function () {
363 this.timeout(60000)
364
365 await checkPublicVODFiles(videoUUID)
366 })
367
368 it('Should not be able to access object storage proxy', async function () {
369 const privateVideo = await server.videos.getWithToken({ id: videoUUID })
370 const webtorrentFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl)
371 const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl)
372
373 await makeRawRequest({
374 url: server.url + '/object-storage-proxy/webseed/private/' + webtorrentFilename,
375 token: server.accessToken,
376 expectedStatus: HttpStatusCode.BAD_REQUEST_400
377 })
378
379 await makeRawRequest({
380 url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename,
381 token: server.accessToken,
382 expectedStatus: HttpStatusCode.BAD_REQUEST_400
383 })
384 })
385 })
386
387 after(async function () {
388 this.timeout(240000)
389
390 const { data } = await server.videos.listAllForAdmin()
391
392 for (const v of data) {
393 await server.videos.remove({ id: v.uuid })
394 }
395
396 for (const v of data) {
397 await server.servers.waitUntilLog('Removed files of video ' + v.url)
398 }
399
400 await cleanupTests([ server ])
401 })
402})
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
index 3e65e1093..d1875febb 100644
--- a/server/tests/api/object-storage/videos.ts
+++ b/server/tests/api/object-storage/videos.ts
@@ -11,7 +11,7 @@ import {
11 generateHighBitrateVideo, 11 generateHighBitrateVideo,
12 MockObjectStorage 12 MockObjectStorage
13} from '@server/tests/shared' 13} from '@server/tests/shared'
14import { areObjectStorageTestsDisabled } from '@shared/core-utils' 14import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
15import { HttpStatusCode, VideoDetails } from '@shared/models' 15import { HttpStatusCode, VideoDetails } from '@shared/models'
16import { 16import {
17 cleanupTests, 17 cleanupTests,
@@ -52,18 +52,18 @@ async function checkFiles (options: {
52 for (const file of video.files) { 52 for (const file of video.files) {
53 const baseUrl = baseMockUrl 53 const baseUrl = baseMockUrl
54 ? `${baseMockUrl}/${webtorrentBucket}/` 54 ? `${baseMockUrl}/${webtorrentBucket}/`
55 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` 55 : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
56 56
57 const prefix = webtorrentPrefix || '' 57 const prefix = webtorrentPrefix || ''
58 const start = baseUrl + prefix 58 const start = baseUrl + prefix
59 59
60 expectStartWith(file.fileUrl, start) 60 expectStartWith(file.fileUrl, start)
61 61
62 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) 62 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
63 const location = res.headers['location'] 63 const location = res.headers['location']
64 expectStartWith(location, start) 64 expectStartWith(location, start)
65 65
66 await makeRawRequest(location, HttpStatusCode.OK_200) 66 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
67 } 67 }
68 68
69 const hls = video.streamingPlaylists[0] 69 const hls = video.streamingPlaylists[0]
@@ -73,7 +73,7 @@ async function checkFiles (options: {
73 73
74 const baseUrl = baseMockUrl 74 const baseUrl = baseMockUrl
75 ? `${baseMockUrl}/${playlistBucket}/` 75 ? `${baseMockUrl}/${playlistBucket}/`
76 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` 76 : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
77 77
78 const prefix = playlistPrefix || '' 78 const prefix = playlistPrefix || ''
79 const start = baseUrl + prefix 79 const start = baseUrl + prefix
@@ -81,19 +81,19 @@ async function checkFiles (options: {
81 expectStartWith(hls.playlistUrl, start) 81 expectStartWith(hls.playlistUrl, start)
82 expectStartWith(hls.segmentsSha256Url, start) 82 expectStartWith(hls.segmentsSha256Url, start)
83 83
84 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) 84 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
85 85
86 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) 86 const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
87 expect(JSON.stringify(resSha.body)).to.not.throw 87 expect(JSON.stringify(resSha.body)).to.not.throw
88 88
89 for (const file of hls.files) { 89 for (const file of hls.files) {
90 expectStartWith(file.fileUrl, start) 90 expectStartWith(file.fileUrl, start)
91 91
92 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) 92 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
93 const location = res.headers['location'] 93 const location = res.headers['location']
94 expectStartWith(location, start) 94 expectStartWith(location, start)
95 95
96 await makeRawRequest(location, HttpStatusCode.OK_200) 96 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
97 } 97 }
98 } 98 }
99 99
@@ -104,7 +104,7 @@ async function checkFiles (options: {
104 expect(torrent.files.length).to.equal(1) 104 expect(torrent.files.length).to.equal(1)
105 expect(torrent.files[0].path).to.exist.and.to.not.equal('') 105 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
106 106
107 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 107 const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
108 expect(res.body).to.have.length.above(100) 108 expect(res.body).to.have.length.above(100)
109 } 109 }
110 110
@@ -141,16 +141,16 @@ function runTestSuite (options: {
141 const port = await mockObjectStorage.initialize() 141 const port = await mockObjectStorage.initialize()
142 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined 142 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
143 143
144 await ObjectStorageCommand.createBucket(options.playlistBucket) 144 await ObjectStorageCommand.createMockBucket(options.playlistBucket)
145 await ObjectStorageCommand.createBucket(options.webtorrentBucket) 145 await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
146 146
147 const config = { 147 const config = {
148 object_storage: { 148 object_storage: {
149 enabled: true, 149 enabled: true,
150 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), 150 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
151 region: ObjectStorageCommand.getRegion(), 151 region: ObjectStorageCommand.getMockRegion(),
152 152
153 credentials: ObjectStorageCommand.getCredentialsConfig(), 153 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
154 154
155 max_upload_part: options.maxUploadPart || '5MB', 155 max_upload_part: options.maxUploadPart || '5MB',
156 156
@@ -220,7 +220,7 @@ function runTestSuite (options: {
220 220
221 it('Should fetch correctly all the files', async function () { 221 it('Should fetch correctly all the files', async function () {
222 for (const url of deletedUrls.concat(keptUrls)) { 222 for (const url of deletedUrls.concat(keptUrls)) {
223 await makeRawRequest(url, HttpStatusCode.OK_200) 223 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
224 } 224 }
225 }) 225 })
226 226
@@ -231,13 +231,13 @@ function runTestSuite (options: {
231 await waitJobs(servers) 231 await waitJobs(servers)
232 232
233 for (const url of deletedUrls) { 233 for (const url of deletedUrls) {
234 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) 234 await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
235 } 235 }
236 }) 236 })
237 237
238 it('Should have kept other files', async function () { 238 it('Should have kept other files', async function () {
239 for (const url of keptUrls) { 239 for (const url of keptUrls) {
240 await makeRawRequest(url, HttpStatusCode.OK_200) 240 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
241 } 241 }
242 }) 242 })
243 243
@@ -261,7 +261,7 @@ function runTestSuite (options: {
261} 261}
262 262
263describe('Object storage for videos', function () { 263describe('Object storage for videos', function () {
264 if (areObjectStorageTestsDisabled()) return 264 if (areMockObjectStorageTestsDisabled()) return
265 265
266 describe('Test config', function () { 266 describe('Test config', function () {
267 let server: PeerTubeServer 267 let server: PeerTubeServer
@@ -269,17 +269,17 @@ describe('Object storage for videos', function () {
269 const baseConfig = { 269 const baseConfig = {
270 object_storage: { 270 object_storage: {
271 enabled: true, 271 enabled: true,
272 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), 272 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
273 region: ObjectStorageCommand.getRegion(), 273 region: ObjectStorageCommand.getMockRegion(),
274 274
275 credentials: ObjectStorageCommand.getCredentialsConfig(), 275 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
276 276
277 streaming_playlists: { 277 streaming_playlists: {
278 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET 278 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
279 }, 279 },
280 280
281 videos: { 281 videos: {
282 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET 282 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
283 } 283 }
284 } 284 }
285 } 285 }
@@ -310,7 +310,7 @@ describe('Object storage for videos', function () {
310 it('Should fail with bad credentials', async function () { 310 it('Should fail with bad credentials', async function () {
311 this.timeout(60000) 311 this.timeout(60000)
312 312
313 await ObjectStorageCommand.prepareDefaultBuckets() 313 await ObjectStorageCommand.prepareDefaultMockBuckets()
314 314
315 const config = merge({}, baseConfig, { 315 const config = merge({}, baseConfig, {
316 object_storage: { 316 object_storage: {
@@ -323,7 +323,7 @@ describe('Object storage for videos', function () {
323 323
324 const { uuid } = await server.videos.quickUpload({ name: 'video' }) 324 const { uuid } = await server.videos.quickUpload({ name: 'video' })
325 325
326 await waitJobs([ server ], true) 326 await waitJobs([ server ], { skipDelayed: true })
327 const video = await server.videos.get({ id: uuid }) 327 const video = await server.videos.get({ id: uuid })
328 328
329 expectStartWith(video.files[0].fileUrl, server.url) 329 expectStartWith(video.files[0].fileUrl, server.url)
@@ -334,7 +334,7 @@ describe('Object storage for videos', function () {
334 it('Should succeed with credentials from env', async function () { 334 it('Should succeed with credentials from env', async function () {
335 this.timeout(60000) 335 this.timeout(60000)
336 336
337 await ObjectStorageCommand.prepareDefaultBuckets() 337 await ObjectStorageCommand.prepareDefaultMockBuckets()
338 338
339 const config = merge({}, baseConfig, { 339 const config = merge({}, baseConfig, {
340 object_storage: { 340 object_storage: {
@@ -345,7 +345,7 @@ describe('Object storage for videos', function () {
345 } 345 }
346 }) 346 })
347 347
348 const goodCredentials = ObjectStorageCommand.getCredentialsConfig() 348 const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
349 349
350 server = await createSingleServer(1, config, { 350 server = await createSingleServer(1, config, {
351 env: { 351 env: {
@@ -358,10 +358,10 @@ describe('Object storage for videos', function () {
358 358
359 const { uuid } = await server.videos.quickUpload({ name: 'video' }) 359 const { uuid } = await server.videos.quickUpload({ name: 'video' })
360 360
361 await waitJobs([ server ], true) 361 await waitJobs([ server ], { skipDelayed: true })
362 const video = await server.videos.get({ id: uuid }) 362 const video = await server.videos.get({ id: uuid })
363 363
364 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 364 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
365 }) 365 })
366 366
367 after(async function () { 367 after(async function () {
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 5abed358f..fb2e6e91c 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -5,7 +5,7 @@ import { readdir } from 'fs-extra'
5import magnetUtil from 'magnet-uri' 5import magnetUtil from 'magnet-uri'
6import { basename, join } from 'path' 6import { basename, join } from 'path'
7import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared' 7import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared'
8import { root, wait } from '@shared/core-utils' 8import { wait } from '@shared/core-utils'
9import { 9import {
10 HttpStatusCode, 10 HttpStatusCode,
11 VideoDetails, 11 VideoDetails,
@@ -39,7 +39,7 @@ async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], ser
39 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) 39 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
40 40
41 for (const url of parsed.urlList) { 41 for (const url of parsed.urlList) {
42 await makeRawRequest(url, HttpStatusCode.OK_200) 42 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
43 } 43 }
44} 44}
45 45
@@ -125,7 +125,7 @@ async function check1WebSeed (videoUUID?: string) {
125 if (!videoUUID) videoUUID = video1Server2.uuid 125 if (!videoUUID) videoUUID = video1Server2.uuid
126 126
127 const webseeds = [ 127 const webseeds = [
128 `http://localhost:${servers[1].port}/static/webseed/` 128 `${servers[1].url}/static/webseed/`
129 ] 129 ]
130 130
131 for (const server of servers) { 131 for (const server of servers) {
@@ -144,8 +144,8 @@ async function check2Webseeds (videoUUID?: string) {
144 if (!videoUUID) videoUUID = video1Server2.uuid 144 if (!videoUUID) videoUUID = video1Server2.uuid
145 145
146 const webseeds = [ 146 const webseeds = [
147 `http://localhost:${servers[0].port}/static/redundancy/`, 147 `${servers[0].url}/static/redundancy/`,
148 `http://localhost:${servers[1].port}/static/webseed/` 148 `${servers[1].url}/static/webseed/`
149 ] 149 ]
150 150
151 for (const server of servers) { 151 for (const server of servers) {
@@ -159,12 +159,12 @@ async function check2Webseeds (videoUUID?: string) {
159 const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) 159 const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
160 160
161 const directories = [ 161 const directories = [
162 'test' + servers[0].internalServerNumber + '/redundancy', 162 servers[0].getDirectoryPath('redundancy'),
163 'test' + servers[1].internalServerNumber + '/videos' 163 servers[1].getDirectoryPath('videos')
164 ] 164 ]
165 165
166 for (const directory of directories) { 166 for (const directory of directories) {
167 const files = await readdir(join(root(), directory)) 167 const files = await readdir(directory)
168 expect(files).to.have.length.at.least(4) 168 expect(files).to.have.length.at.least(4)
169 169
170 // Ensure we files exist on disk 170 // Ensure we files exist on disk
@@ -214,12 +214,12 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
214 const { hlsFilenames } = await ensureSameFilenames(videoUUID) 214 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
215 215
216 const directories = [ 216 const directories = [
217 'test' + servers[0].internalServerNumber + '/redundancy/hls', 217 servers[0].getDirectoryPath('redundancy/hls'),
218 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' 218 servers[1].getDirectoryPath('streaming-playlists/hls')
219 ] 219 ]
220 220
221 for (const directory of directories) { 221 for (const directory of directories) {
222 const files = await readdir(join(root(), directory, videoUUID)) 222 const files = await readdir(join(directory, videoUUID))
223 expect(files).to.have.length.at.least(4) 223 expect(files).to.have.length.at.least(4)
224 224
225 // Ensure we files exist on disk 225 // Ensure we files exist on disk
diff --git a/server/tests/api/server/follow-constraints.ts b/server/tests/api/server/follow-constraints.ts
index 5998f58cc..e1ec2b069 100644
--- a/server/tests/api/server/follow-constraints.ts
+++ b/server/tests/api/server/follow-constraints.ts
@@ -1,8 +1,15 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
5import { HttpStatusCode, PeerTubeProblemDocument, ServerErrorCode } from '@shared/models' 4import { HttpStatusCode, PeerTubeProblemDocument, ServerErrorCode } from '@shared/models'
5import {
6 cleanupTests,
7 createMultipleServers,
8 doubleFollow,
9 PeerTubeServer,
10 setAccessTokensToServers,
11 waitJobs
12} from '@shared/server-commands'
6 13
7describe('Test follow constraints', function () { 14describe('Test follow constraints', function () {
8 let servers: PeerTubeServer[] = [] 15 let servers: PeerTubeServer[] = []
@@ -189,6 +196,7 @@ describe('Test follow constraints', function () {
189 }) 196 })
190 197
191 describe('With a logged user', function () { 198 describe('With a logged user', function () {
199
192 it('Should get the local video', async function () { 200 it('Should get the local video', async function () {
193 await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) 201 await servers[0].videos.getWithToken({ token: userToken, id: video1UUID })
194 }) 202 })
@@ -229,6 +237,84 @@ describe('Test follow constraints', function () {
229 }) 237 })
230 }) 238 })
231 239
240 describe('When following a remote account', function () {
241
242 before(async function () {
243 this.timeout(60000)
244
245 await servers[0].follows.follow({ handles: [ 'root@' + servers[1].host ] })
246 await waitJobs(servers)
247 })
248
249 it('Should get the remote video with an unlogged user', async function () {
250 await servers[0].videos.get({ id: video2UUID })
251 })
252
253 it('Should get the remote video with a logged in user', async function () {
254 await servers[0].videos.getWithToken({ token: userToken, id: video2UUID })
255 })
256 })
257
258 describe('When unfollowing a remote account', function () {
259
260 before(async function () {
261 this.timeout(60000)
262
263 await servers[0].follows.unfollow({ target: 'root@' + servers[1].host })
264 await waitJobs(servers)
265 })
266
267 it('Should not get the remote video with an unlogged user', async function () {
268 const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
269
270 const error = body as unknown as PeerTubeProblemDocument
271 expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS)
272 })
273
274 it('Should get the remote video with a logged in user', async function () {
275 await servers[0].videos.getWithToken({ token: userToken, id: video2UUID })
276 })
277 })
278
279 describe('When following a remote channel', function () {
280
281 before(async function () {
282 this.timeout(60000)
283
284 await servers[0].follows.follow({ handles: [ 'root_channel@' + servers[1].host ] })
285 await waitJobs(servers)
286 })
287
288 it('Should get the remote video with an unlogged user', async function () {
289 await servers[0].videos.get({ id: video2UUID })
290 })
291
292 it('Should get the remote video with a logged in user', async function () {
293 await servers[0].videos.getWithToken({ token: userToken, id: video2UUID })
294 })
295 })
296
297 describe('When unfollowing a remote channel', function () {
298
299 before(async function () {
300 this.timeout(60000)
301
302 await servers[0].follows.unfollow({ target: 'root_channel@' + servers[1].host })
303 await waitJobs(servers)
304 })
305
306 it('Should not get the remote video with an unlogged user', async function () {
307 const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
308
309 const error = body as unknown as PeerTubeProblemDocument
310 expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS)
311 })
312
313 it('Should get the remote video with a logged in user', async function () {
314 await servers[0].videos.getWithToken({ token: userToken, id: video2UUID })
315 })
316 })
317
232 after(async function () { 318 after(async function () {
233 await cleanupTests(servers) 319 await cleanupTests(servers)
234 }) 320 })
diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts
index 43a27cc32..7a294be82 100644
--- a/server/tests/api/server/open-telemetry.ts
+++ b/server/tests/api/server/open-telemetry.ts
@@ -18,7 +18,7 @@ describe('Open Telemetry', function () {
18 18
19 let hasError = false 19 let hasError = false
20 try { 20 try {
21 await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404) 21 await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
22 } catch (err) { 22 } catch (err) {
23 hasError = err.message.includes('ECONNREFUSED') 23 hasError = err.message.includes('ECONNREFUSED')
24 } 24 }
@@ -37,7 +37,7 @@ describe('Open Telemetry', function () {
37 } 37 }
38 }) 38 })
39 39
40 const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) 40 const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
41 expect(res.text).to.contain('peertube_job_queue_total{') 41 expect(res.text).to.contain('peertube_job_queue_total{')
42 }) 42 })
43 43
@@ -60,7 +60,7 @@ describe('Open Telemetry', function () {
60 } 60 }
61 }) 61 })
62 62
63 const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) 63 const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
64 expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') 64 expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{')
65 }) 65 })
66 66
diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts
index a4151ebdd..71c444efd 100644
--- a/server/tests/api/server/proxy.ts
+++ b/server/tests/api/server/proxy.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' 4import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { 7import {
8 cleanupTests, 8 cleanupTests,
@@ -120,40 +120,40 @@ describe('Test proxy', function () {
120 }) 120 })
121 121
122 describe('Object storage', function () { 122 describe('Object storage', function () {
123 if (areObjectStorageTestsDisabled()) return 123 if (areMockObjectStorageTestsDisabled()) return
124 124
125 before(async function () { 125 before(async function () {
126 this.timeout(30000) 126 this.timeout(30000)
127 127
128 await ObjectStorageCommand.prepareDefaultBuckets() 128 await ObjectStorageCommand.prepareDefaultMockBuckets()
129 }) 129 })
130 130
131 it('Should succeed to upload to object storage with the appropriate proxy config', async function () { 131 it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
132 this.timeout(120000) 132 this.timeout(120000)
133 133
134 await servers[0].kill() 134 await servers[0].kill()
135 await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv }) 135 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv })
136 136
137 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) 137 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
138 await waitJobs(servers) 138 await waitJobs(servers)
139 139
140 const video = await servers[0].videos.get({ id: uuid }) 140 const video = await servers[0].videos.get({ id: uuid })
141 141
142 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 142 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
143 }) 143 })
144 144
145 it('Should fail to upload to object storage with a wrong proxy config', async function () { 145 it('Should fail to upload to object storage with a wrong proxy config', async function () {
146 this.timeout(120000) 146 this.timeout(120000)
147 147
148 await servers[0].kill() 148 await servers[0].kill()
149 await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv }) 149 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv })
150 150
151 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) 151 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
152 await waitJobs(servers) 152 await waitJobs(servers, { skipDelayed: true })
153 153
154 const video = await servers[0].videos.get({ id: uuid }) 154 const video = await servers[0].videos.get({ id: uuid })
155 155
156 expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 156 expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
157 }) 157 })
158 }) 158 })
159 159
diff --git a/server/tests/api/transcoding/audio-only.ts b/server/tests/api/transcoding/audio-only.ts
index 1897c6d6d..b72f5fdbe 100644
--- a/server/tests/api/transcoding/audio-only.ts
+++ b/server/tests/api/transcoding/audio-only.ts
@@ -89,7 +89,12 @@ describe('Test audio only video transcoding', function () {
89 expect(audioStream['bit_rate']).to.be.at.most(384 * 8000) 89 expect(audioStream['bit_rate']).to.be.at.most(384 * 8000)
90 90
91 const size = await getVideoStreamDimensionsInfo(path) 91 const size = await getVideoStreamDimensionsInfo(path)
92 expect(size).to.not.exist 92
93 expect(size.height).to.equal(0)
94 expect(size.width).to.equal(0)
95 expect(size.isPortraitMode).to.be.false
96 expect(size.ratio).to.equal(0)
97 expect(size.resolution).to.equal(0)
93 } 98 }
94 }) 99 })
95 100
diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts
index a50bf7654..85389a949 100644
--- a/server/tests/api/transcoding/create-transcoding.ts
+++ b/server/tests/api/transcoding/create-transcoding.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared' 4import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoDetails } from '@shared/models' 6import { HttpStatusCode, VideoDetails } from '@shared/models'
7import { 7import {
8 cleanupTests, 8 cleanupTests,
@@ -19,23 +19,23 @@ import {
19 19
20async function checkFilesInObjectStorage (video: VideoDetails) { 20async function checkFilesInObjectStorage (video: VideoDetails) {
21 for (const file of video.files) { 21 for (const file of video.files) {
22 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 22 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
23 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 23 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
24 } 24 }
25 25
26 if (video.streamingPlaylists.length === 0) return 26 if (video.streamingPlaylists.length === 0) return
27 27
28 const hlsPlaylist = video.streamingPlaylists[0] 28 const hlsPlaylist = video.streamingPlaylists[0]
29 for (const file of hlsPlaylist.files) { 29 for (const file of hlsPlaylist.files) {
30 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 30 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
31 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 31 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
32 } 32 }
33 33
34 expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 34 expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
35 await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) 35 await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
36 36
37 expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl()) 37 expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl())
38 await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) 38 await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
39} 39}
40 40
41function runTests (objectStorage: boolean) { 41function runTests (objectStorage: boolean) {
@@ -49,7 +49,7 @@ function runTests (objectStorage: boolean) {
49 this.timeout(120000) 49 this.timeout(120000)
50 50
51 const config = objectStorage 51 const config = objectStorage
52 ? ObjectStorageCommand.getDefaultConfig() 52 ? ObjectStorageCommand.getDefaultMockConfig()
53 : {} 53 : {}
54 54
55 // Run server 2 to have transcoding enabled 55 // Run server 2 to have transcoding enabled
@@ -60,7 +60,7 @@ function runTests (objectStorage: boolean) {
60 60
61 await doubleFollow(servers[0], servers[1]) 61 await doubleFollow(servers[0], servers[1])
62 62
63 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() 63 if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
64 64
65 const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) 65 const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
66 videoUUID = shortUUID 66 videoUUID = shortUUID
@@ -234,7 +234,7 @@ function runTests (objectStorage: boolean) {
234 234
235 it('Should have correctly deleted previous files', async function () { 235 it('Should have correctly deleted previous files', async function () {
236 for (const fileUrl of shouldBeDeleted) { 236 for (const fileUrl of shouldBeDeleted) {
237 await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404) 237 await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
238 } 238 }
239 }) 239 })
240 240
@@ -256,7 +256,7 @@ describe('Test create transcoding jobs from API', function () {
256 }) 256 })
257 257
258 describe('On object storage', function () { 258 describe('On object storage', function () {
259 if (areObjectStorageTestsDisabled()) return 259 if (areMockObjectStorageTestsDisabled()) return
260 260
261 runTests(true) 261 runTests(true)
262 }) 262 })
diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts
index 252422e5d..84a53c0bd 100644
--- a/server/tests/api/transcoding/hls.ts
+++ b/server/tests/api/transcoding/hls.ts
@@ -1,168 +1,48 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { join } from 'path'
4import { basename, join } from 'path' 4import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
5import { 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6 checkDirectoryIsEmpty, 6import { HttpStatusCode } from '@shared/models'
7 checkResolutionsInMasterPlaylist,
8 checkSegmentHash,
9 checkTmpIsEmpty,
10 expectStartWith,
11 hlsInfohashExist
12} from '@server/tests/shared'
13import { areObjectStorageTestsDisabled, removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
14import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
15import { 7import {
16 cleanupTests, 8 cleanupTests,
17 createMultipleServers, 9 createMultipleServers,
18 doubleFollow, 10 doubleFollow,
19 makeRawRequest,
20 ObjectStorageCommand, 11 ObjectStorageCommand,
21 PeerTubeServer, 12 PeerTubeServer,
22 setAccessTokensToServers, 13 setAccessTokensToServers,
23 waitJobs, 14 waitJobs
24 webtorrentAdd
25} from '@shared/server-commands' 15} from '@shared/server-commands'
26import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' 16import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
27 17
28async function checkHlsPlaylist (options: {
29 servers: PeerTubeServer[]
30 videoUUID: string
31 hlsOnly: boolean
32
33 resolutions?: number[]
34 objectStorageBaseUrl: string
35}) {
36 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
37
38 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
39
40 for (const server of options.servers) {
41 const videoDetails = await server.videos.get({ id: videoUUID })
42 const baseUrl = `http://${videoDetails.account.host}`
43
44 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
45
46 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
47 expect(hlsPlaylist).to.not.be.undefined
48
49 const hlsFiles = hlsPlaylist.files
50 expect(hlsFiles).to.have.lengthOf(resolutions.length)
51
52 if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
53 else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
54
55 // Check JSON files
56 for (const resolution of resolutions) {
57 const file = hlsFiles.find(f => f.resolution.id === resolution)
58 expect(file).to.not.be.undefined
59
60 expect(file.magnetUri).to.have.lengthOf.above(2)
61 expect(file.torrentUrl).to.match(
62 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
63 )
64
65 if (objectStorageBaseUrl) {
66 expectStartWith(file.fileUrl, objectStorageBaseUrl)
67 } else {
68 expect(file.fileUrl).to.match(
69 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
70 )
71 }
72
73 expect(file.resolution.label).to.equal(resolution + 'p')
74
75 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
76 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
77
78 const torrent = await webtorrentAdd(file.magnetUri, true)
79 expect(torrent.files).to.be.an('array')
80 expect(torrent.files.length).to.equal(1)
81 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
82 }
83
84 // Check master playlist
85 {
86 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
87
88 const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
89
90 let i = 0
91 for (const resolution of resolutions) {
92 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
93 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
94
95 const url = 'http://' + videoDetails.account.host
96 await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
97
98 i++
99 }
100 }
101
102 // Check resolution playlists
103 {
104 for (const resolution of resolutions) {
105 const file = hlsFiles.find(f => f.resolution.id === resolution)
106 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
107
108 const url = objectStorageBaseUrl
109 ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
110 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
111
112 const subPlaylist = await server.streamingPlaylists.get({ url })
113
114 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
115 expect(subPlaylist).to.contain(basename(file.fileUrl))
116 }
117 }
118
119 {
120 const baseUrlAndPath = objectStorageBaseUrl
121 ? objectStorageBaseUrl + 'hls/' + videoUUID
122 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
123
124 for (const resolution of resolutions) {
125 await checkSegmentHash({
126 server,
127 baseUrlPlaylist: baseUrlAndPath,
128 baseUrlSegment: baseUrlAndPath,
129 resolution,
130 hlsPlaylist
131 })
132 }
133 }
134 }
135}
136
137describe('Test HLS videos', function () { 18describe('Test HLS videos', function () {
138 let servers: PeerTubeServer[] = [] 19 let servers: PeerTubeServer[] = []
139 let videoUUID = ''
140 let videoAudioUUID = ''
141 20
142 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { 21 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
22 const videoUUIDs: string[] = []
143 23
144 it('Should upload a video and transcode it to HLS', async function () { 24 it('Should upload a video and transcode it to HLS', async function () {
145 this.timeout(120000) 25 this.timeout(120000)
146 26
147 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } }) 27 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } })
148 videoUUID = uuid 28 videoUUIDs.push(uuid)
149 29
150 await waitJobs(servers) 30 await waitJobs(servers)
151 31
152 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) 32 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
153 }) 33 })
154 34
155 it('Should upload an audio file and transcode it to HLS', async function () { 35 it('Should upload an audio file and transcode it to HLS', async function () {
156 this.timeout(120000) 36 this.timeout(120000)
157 37
158 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } }) 38 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } })
159 videoAudioUUID = uuid 39 videoUUIDs.push(uuid)
160 40
161 await waitJobs(servers) 41 await waitJobs(servers)
162 42
163 await checkHlsPlaylist({ 43 await completeCheckHlsPlaylist({
164 servers, 44 servers,
165 videoUUID: videoAudioUUID, 45 videoUUID: uuid,
166 hlsOnly, 46 hlsOnly,
167 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], 47 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
168 objectStorageBaseUrl 48 objectStorageBaseUrl
@@ -172,31 +52,36 @@ describe('Test HLS videos', function () {
172 it('Should update the video', async function () { 52 it('Should update the video', async function () {
173 this.timeout(30000) 53 this.timeout(30000)
174 54
175 await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video 1 updated' } }) 55 await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } })
176 56
177 await waitJobs(servers) 57 await waitJobs(servers)
178 58
179 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) 59 await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl })
180 }) 60 })
181 61
182 it('Should delete videos', async function () { 62 it('Should delete videos', async function () {
183 this.timeout(10000) 63 this.timeout(10000)
184 64
185 await servers[0].videos.remove({ id: videoUUID }) 65 for (const uuid of videoUUIDs) {
186 await servers[0].videos.remove({ id: videoAudioUUID }) 66 await servers[0].videos.remove({ id: uuid })
67 }
187 68
188 await waitJobs(servers) 69 await waitJobs(servers)
189 70
190 for (const server of servers) { 71 for (const server of servers) {
191 await server.videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 72 for (const uuid of videoUUIDs) {
192 await server.videos.get({ id: videoAudioUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) 73 await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
74 }
193 } 75 }
194 }) 76 })
195 77
196 it('Should have the playlists/segment deleted from the disk', async function () { 78 it('Should have the playlists/segment deleted from the disk', async function () {
197 for (const server of servers) { 79 for (const server of servers) {
198 await checkDirectoryIsEmpty(server, 'videos') 80 await checkDirectoryIsEmpty(server, 'videos', [ 'private' ])
199 await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls')) 81 await checkDirectoryIsEmpty(server, join('videos', 'private'))
82
83 await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ])
84 await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private'))
200 } 85 }
201 }) 86 })
202 87
@@ -265,19 +150,19 @@ describe('Test HLS videos', function () {
265 }) 150 })
266 151
267 describe('With object storage enabled', function () { 152 describe('With object storage enabled', function () {
268 if (areObjectStorageTestsDisabled()) return 153 if (areMockObjectStorageTestsDisabled()) return
269 154
270 before(async function () { 155 before(async function () {
271 this.timeout(120000) 156 this.timeout(120000)
272 157
273 const configOverride = ObjectStorageCommand.getDefaultConfig() 158 const configOverride = ObjectStorageCommand.getDefaultMockConfig()
274 await ObjectStorageCommand.prepareDefaultBuckets() 159 await ObjectStorageCommand.prepareDefaultMockBuckets()
275 160
276 await servers[0].kill() 161 await servers[0].kill()
277 await servers[0].run(configOverride) 162 await servers[0].run(configOverride)
278 }) 163 })
279 164
280 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) 165 runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
281 }) 166 })
282 167
283 after(async function () { 168 after(async function () {
diff --git a/server/tests/api/transcoding/index.ts b/server/tests/api/transcoding/index.ts
index 0cc28b4a4..9866418d6 100644
--- a/server/tests/api/transcoding/index.ts
+++ b/server/tests/api/transcoding/index.ts
@@ -2,4 +2,5 @@ export * from './audio-only'
2export * from './create-transcoding' 2export * from './create-transcoding'
3export * from './hls' 3export * from './hls'
4export * from './transcoder' 4export * from './transcoder'
5export * from './update-while-transcoding'
5export * from './video-studio' 6export * from './video-studio'
diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts
new file mode 100644
index 000000000..8e32ea069
--- /dev/null
+++ b/server/tests/api/transcoding/update-while-transcoding.ts
@@ -0,0 +1,151 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { completeCheckHlsPlaylist } from '@server/tests/shared'
4import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils'
5import { VideoPrivacy } from '@shared/models'
6import {
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 ObjectStorageCommand,
11 PeerTubeServer,
12 setAccessTokensToServers,
13 waitJobs
14} from '@shared/server-commands'
15
16describe('Test update video privacy while transcoding', function () {
17 let servers: PeerTubeServer[] = []
18
19 const videoUUIDs: string[] = []
20
21 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
22
23 it('Should not have an error while quickly updating a private video to public after upload #1', async function () {
24 this.timeout(360_000)
25
26 const attributes = {
27 name: 'quick update',
28 privacy: VideoPrivacy.PRIVATE
29 }
30
31 const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false })
32 await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
33 videoUUIDs.push(uuid)
34
35 await waitJobs(servers)
36
37 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
38 })
39
40 it('Should not have an error while quickly updating a private video to public after upload #2', async function () {
41
42 {
43 const attributes = {
44 name: 'quick update 2',
45 privacy: VideoPrivacy.PRIVATE
46 }
47
48 const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
49 await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
50 videoUUIDs.push(uuid)
51
52 await waitJobs(servers)
53
54 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
55 }
56 })
57
58 it('Should not have an error while quickly updating a private video to public after upload #3', async function () {
59 const attributes = {
60 name: 'quick update 3',
61 privacy: VideoPrivacy.PRIVATE
62 }
63
64 const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
65 await wait(1000)
66 await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
67 videoUUIDs.push(uuid)
68
69 await waitJobs(servers)
70
71 await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
72 })
73 }
74
75 before(async function () {
76 this.timeout(120000)
77
78 const configOverride = {
79 transcoding: {
80 enabled: true,
81 allow_audio_files: true,
82 hls: {
83 enabled: true
84 }
85 }
86 }
87 servers = await createMultipleServers(2, configOverride)
88
89 // Get the access tokens
90 await setAccessTokensToServers(servers)
91
92 // Server 1 and server 2 follow each other
93 await doubleFollow(servers[0], servers[1])
94 })
95
96 describe('With WebTorrent & HLS enabled', function () {
97 runTestSuite(false)
98 })
99
100 describe('With only HLS enabled', function () {
101
102 before(async function () {
103 await servers[0].config.updateCustomSubConfig({
104 newConfig: {
105 transcoding: {
106 enabled: true,
107 allowAudioFiles: true,
108 resolutions: {
109 '144p': false,
110 '240p': true,
111 '360p': true,
112 '480p': true,
113 '720p': true,
114 '1080p': true,
115 '1440p': true,
116 '2160p': true
117 },
118 hls: {
119 enabled: true
120 },
121 webtorrent: {
122 enabled: false
123 }
124 }
125 }
126 })
127 })
128
129 runTestSuite(true)
130 })
131
132 describe('With object storage enabled', function () {
133 if (areMockObjectStorageTestsDisabled()) return
134
135 before(async function () {
136 this.timeout(120000)
137
138 const configOverride = ObjectStorageCommand.getDefaultMockConfig()
139 await ObjectStorageCommand.prepareDefaultMockBuckets()
140
141 await servers[0].kill()
142 await servers[0].run(configOverride)
143 })
144
145 runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
146 })
147
148 after(async function () {
149 await cleanupTests(servers)
150 })
151})
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts
index 9613111b5..ab08e8fb6 100644
--- a/server/tests/api/transcoding/video-studio.ts
+++ b/server/tests/api/transcoding/video-studio.ts
@@ -1,6 +1,6 @@
1import { expect } from 'chai' 1import { expect } from 'chai'
2import { expectStartWith } from '@server/tests/shared' 2import { expectStartWith } from '@server/tests/shared'
3import { areObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' 3import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
4import { VideoStudioTask } from '@shared/models' 4import { VideoStudioTask } from '@shared/models'
5import { 5import {
6 cleanupTests, 6 cleanupTests,
@@ -315,13 +315,13 @@ describe('Test video studio', function () {
315 }) 315 })
316 316
317 describe('Object storage video edition', function () { 317 describe('Object storage video edition', function () {
318 if (areObjectStorageTestsDisabled()) return 318 if (areMockObjectStorageTestsDisabled()) return
319 319
320 before(async function () { 320 before(async function () {
321 await ObjectStorageCommand.prepareDefaultBuckets() 321 await ObjectStorageCommand.prepareDefaultMockBuckets()
322 322
323 await servers[0].kill() 323 await servers[0].kill()
324 await servers[0].run(ObjectStorageCommand.getDefaultConfig()) 324 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
325 325
326 await servers[0].config.enableMinimumTranscoding() 326 await servers[0].config.enableMinimumTranscoding()
327 }) 327 })
@@ -344,11 +344,11 @@ describe('Test video studio', function () {
344 } 344 }
345 345
346 for (const webtorrentFile of video.files) { 346 for (const webtorrentFile of video.files) {
347 expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 347 expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
348 } 348 }
349 349
350 for (const hlsFile of video.streamingPlaylists[0].files) { 350 for (const hlsFile of video.streamingPlaylists[0].files) {
351 expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 351 expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
352 } 352 }
353 353
354 await checkDuration(server, 9) 354 await checkDuration(server, 9)
diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts
index c65152c6f..643f1a531 100644
--- a/server/tests/api/users/index.ts
+++ b/server/tests/api/users/index.ts
@@ -1,3 +1,4 @@
1import './two-factor'
1import './user-subscriptions' 2import './user-subscriptions'
2import './user-videos' 3import './user-videos'
3import './users' 4import './users'
diff --git a/server/tests/api/users/two-factor.ts b/server/tests/api/users/two-factor.ts
new file mode 100644
index 000000000..0dcab9e17
--- /dev/null
+++ b/server/tests/api/users/two-factor.ts
@@ -0,0 +1,200 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { expectStartWith } from '@server/tests/shared'
5import { HttpStatusCode } from '@shared/models'
6import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands'
7
8async function login (options: {
9 server: PeerTubeServer
10 username: string
11 password: string
12 otpToken?: string
13 expectedStatus?: HttpStatusCode
14}) {
15 const { server, username, password, otpToken, expectedStatus } = options
16
17 const user = { username, password }
18 const { res, body: { access_token: token } } = await server.login.loginAndGetResponse({ user, otpToken, expectedStatus })
19
20 return { res, token }
21}
22
23describe('Test users', function () {
24 let server: PeerTubeServer
25 let otpSecret: string
26 let requestToken: string
27
28 const userUsername = 'user1'
29 let userId: number
30 let userPassword: string
31 let userToken: string
32
33 before(async function () {
34 this.timeout(30000)
35
36 server = await createSingleServer(1)
37
38 await setAccessTokensToServers([ server ])
39 const res = await server.users.generate(userUsername)
40 userId = res.userId
41 userPassword = res.password
42 userToken = res.token
43 })
44
45 it('Should not add the header on login if two factor is not enabled', async function () {
46 const { res, token } = await login({ server, username: userUsername, password: userPassword })
47
48 expect(res.header['x-peertube-otp']).to.not.exist
49
50 await server.users.getMyInfo({ token })
51 })
52
53 it('Should request two factor and get the secret and uri', async function () {
54 const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword })
55
56 expect(otpRequest.requestToken).to.exist
57
58 expect(otpRequest.secret).to.exist
59 expect(otpRequest.secret).to.have.lengthOf(32)
60
61 expect(otpRequest.uri).to.exist
62 expectStartWith(otpRequest.uri, 'otpauth://')
63 expect(otpRequest.uri).to.include(otpRequest.secret)
64
65 requestToken = otpRequest.requestToken
66 otpSecret = otpRequest.secret
67 })
68
69 it('Should not have two factor confirmed yet', async function () {
70 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
71 expect(twoFactorEnabled).to.be.false
72 })
73
74 it('Should confirm two factor', async function () {
75 await server.twoFactor.confirmRequest({
76 userId,
77 token: userToken,
78 otpToken: TwoFactorCommand.buildOTP({ secret: otpSecret }).generate(),
79 requestToken
80 })
81 })
82
83 it('Should not add the header on login if two factor is enabled and password is incorrect', async function () {
84 const { res, token } = await login({ server, username: userUsername, password: 'fake', expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
85
86 expect(res.header['x-peertube-otp']).to.not.exist
87 expect(token).to.not.exist
88 })
89
90 it('Should add the header on login if two factor is enabled and password is correct', async function () {
91 const { res, token } = await login({
92 server,
93 username: userUsername,
94 password: userPassword,
95 expectedStatus: HttpStatusCode.UNAUTHORIZED_401
96 })
97
98 expect(res.header['x-peertube-otp']).to.exist
99 expect(token).to.not.exist
100
101 await server.users.getMyInfo({ token })
102 })
103
104 it('Should not login with correct password and incorrect otp secret', async function () {
105 const otp = TwoFactorCommand.buildOTP({ secret: 'a'.repeat(32) })
106
107 const { res, token } = await login({
108 server,
109 username: userUsername,
110 password: userPassword,
111 otpToken: otp.generate(),
112 expectedStatus: HttpStatusCode.BAD_REQUEST_400
113 })
114
115 expect(res.header['x-peertube-otp']).to.not.exist
116 expect(token).to.not.exist
117 })
118
119 it('Should not login with correct password and incorrect otp code', async function () {
120 const { res, token } = await login({
121 server,
122 username: userUsername,
123 password: userPassword,
124 otpToken: '123456',
125 expectedStatus: HttpStatusCode.BAD_REQUEST_400
126 })
127
128 expect(res.header['x-peertube-otp']).to.not.exist
129 expect(token).to.not.exist
130 })
131
132 it('Should not login with incorrect password and correct otp code', async function () {
133 const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate()
134
135 const { res, token } = await login({
136 server,
137 username: userUsername,
138 password: 'fake',
139 otpToken,
140 expectedStatus: HttpStatusCode.BAD_REQUEST_400
141 })
142
143 expect(res.header['x-peertube-otp']).to.not.exist
144 expect(token).to.not.exist
145 })
146
147 it('Should correctly login with correct password and otp code', async function () {
148 const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate()
149
150 const { res, token } = await login({ server, username: userUsername, password: userPassword, otpToken })
151
152 expect(res.header['x-peertube-otp']).to.not.exist
153 expect(token).to.exist
154
155 await server.users.getMyInfo({ token })
156 })
157
158 it('Should have two factor enabled when getting my info', async function () {
159 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
160 expect(twoFactorEnabled).to.be.true
161 })
162
163 it('Should disable two factor and be able to login without otp token', async function () {
164 await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword })
165
166 const { res, token } = await login({ server, username: userUsername, password: userPassword })
167 expect(res.header['x-peertube-otp']).to.not.exist
168
169 await server.users.getMyInfo({ token })
170 })
171
172 it('Should have two factor disabled when getting my info', async function () {
173 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
174 expect(twoFactorEnabled).to.be.false
175 })
176
177 it('Should enable two factor auth without password from an admin', async function () {
178 const { otpRequest } = await server.twoFactor.request({ userId })
179
180 await server.twoFactor.confirmRequest({
181 userId,
182 otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate(),
183 requestToken: otpRequest.requestToken
184 })
185
186 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
187 expect(twoFactorEnabled).to.be.true
188 })
189
190 it('Should disable two factor auth without password from an admin', async function () {
191 await server.twoFactor.disable({ userId })
192
193 const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken })
194 expect(twoFactorEnabled).to.be.false
195 })
196
197 after(async function () {
198 await cleanupTests([ server ])
199 })
200})
diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts
index 62d668d1e..188e6f137 100644
--- a/server/tests/api/users/users-multiple-servers.ts
+++ b/server/tests/api/users/users-multiple-servers.ts
@@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () {
197 it('Should not have actor files', async () => { 197 it('Should not have actor files', async () => {
198 for (const server of servers) { 198 for (const server of servers) {
199 for (const userAvatarFilename of userAvatarFilenames) { 199 for (const userAvatarFilename of userAvatarFilenames) {
200 await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber) 200 await checkActorFilesWereRemoved(userAvatarFilename, server)
201 } 201 }
202 } 202 }
203 }) 203 })
diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts
index 9e657b387..421b3ce16 100644
--- a/server/tests/api/users/users.ts
+++ b/server/tests/api/users/users.ts
@@ -181,7 +181,7 @@ describe('Test users', function () {
181 }) 181 })
182 182
183 it('Should refresh the token', async function () { 183 it('Should refresh the token', async function () {
184 this.timeout(15000) 184 this.timeout(50000)
185 185
186 const futureDate = new Date(new Date().getTime() + 1000 * 60).toISOString() 186 const futureDate = new Date(new Date().getTime() + 1000 * 60).toISOString()
187 await server.sql.setTokenField(server.accessToken, 'refreshTokenExpiresAt', futureDate) 187 await server.sql.setTokenField(server.accessToken, 'refreshTokenExpiresAt', futureDate)
@@ -219,7 +219,7 @@ describe('Test users', function () {
219 expect(user.email).to.equal('user_1@example.com') 219 expect(user.email).to.equal('user_1@example.com')
220 expect(user.nsfwPolicy).to.equal('display') 220 expect(user.nsfwPolicy).to.equal('display')
221 expect(user.videoQuota).to.equal(2 * 1024 * 1024) 221 expect(user.videoQuota).to.equal(2 * 1024 * 1024)
222 expect(user.roleLabel).to.equal('User') 222 expect(user.role.label).to.equal('User')
223 expect(user.id).to.be.a('number') 223 expect(user.id).to.be.a('number')
224 expect(user.account.displayName).to.equal('user_1') 224 expect(user.account.displayName).to.equal('user_1')
225 expect(user.account.description).to.be.null 225 expect(user.account.description).to.be.null
@@ -277,7 +277,7 @@ describe('Test users', function () {
277 const user = data[0] 277 const user = data[0]
278 expect(user.username).to.equal('root') 278 expect(user.username).to.equal('root')
279 expect(user.email).to.equal('admin' + server.internalServerNumber + '@example.com') 279 expect(user.email).to.equal('admin' + server.internalServerNumber + '@example.com')
280 expect(user.roleLabel).to.equal('Administrator') 280 expect(user.role.label).to.equal('Administrator')
281 expect(user.nsfwPolicy).to.equal('display') 281 expect(user.nsfwPolicy).to.equal('display')
282 }) 282 })
283 283
@@ -531,7 +531,7 @@ describe('Test users', function () {
531 expect(user.emailVerified).to.be.true 531 expect(user.emailVerified).to.be.true
532 expect(user.nsfwPolicy).to.equal('do_not_list') 532 expect(user.nsfwPolicy).to.equal('do_not_list')
533 expect(user.videoQuota).to.equal(42) 533 expect(user.videoQuota).to.equal(42)
534 expect(user.roleLabel).to.equal('Moderator') 534 expect(user.role.label).to.equal('Moderator')
535 expect(user.id).to.be.a('number') 535 expect(user.id).to.be.a('number')
536 expect(user.adminFlags).to.equal(UserAdminFlag.NONE) 536 expect(user.adminFlags).to.equal(UserAdminFlag.NONE)
537 expect(user.pluginAuth).to.equal('toto') 537 expect(user.pluginAuth).to.equal('toto')
diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts
index 7cfd02fbb..a66f88a0e 100644
--- a/server/tests/api/videos/channel-import-videos.ts
+++ b/server/tests/api/videos/channel-import-videos.ts
@@ -109,6 +109,45 @@ describe('Test videos import in a channel', function () {
109 } 109 }
110 }) 110 })
111 111
112 it('Should limit max amount of videos synced on full sync', async function () {
113 this.timeout(240_000)
114
115 await server.kill()
116 await server.run({
117 import: {
118 video_channel_synchronization: {
119 full_sync_videos_limit: 1
120 }
121 }
122 })
123
124 const { id } = await server.channels.create({ attributes: { name: 'channel3' } })
125 const channel3Id = id
126
127 const { videoChannelSync } = await server.channelSyncs.create({
128 attributes: {
129 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
130 videoChannelId: channel3Id
131 }
132 })
133 const syncId = videoChannelSync.id
134
135 await waitJobs(server)
136
137 await server.channels.importVideos({
138 channelName: 'channel3',
139 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
140 videoChannelSyncId: syncId
141 })
142
143 await waitJobs(server)
144
145 const { total, data } = await server.videos.listByChannel({ handle: 'channel3' })
146
147 expect(total).to.equal(1)
148 expect(data).to.have.lengthOf(1)
149 })
150
112 after(async function () { 151 after(async function () {
113 await server?.kill() 152 await server?.kill()
114 }) 153 })
@@ -116,5 +155,7 @@ describe('Test videos import in a channel', function () {
116 } 155 }
117 156
118 runSuite('yt-dlp') 157 runSuite('yt-dlp')
119 runSuite('youtube-dl') 158
159 // FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails
160 // runSuite('youtube-dl')
120}) 161})
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 266155297..357c08199 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -19,3 +19,4 @@ import './videos-common-filters'
19import './videos-history' 19import './videos-history'
20import './videos-overview' 20import './videos-overview'
21import './video-source' 21import './video-source'
22import './video-static-file-privacy'
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index d47807a79..2ad749fd4 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -156,7 +156,7 @@ describe('Test multiple servers', function () {
156 }) 156 })
157 157
158 it('Should upload the video on server 2 and propagate on each server', async function () { 158 it('Should upload the video on server 2 and propagate on each server', async function () {
159 this.timeout(100000) 159 this.timeout(240000)
160 160
161 const user = { 161 const user = {
162 username: 'user1', 162 username: 'user1',
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts
index 865b25f04..91291524d 100644
--- a/server/tests/api/videos/video-channel-syncs.ts
+++ b/server/tests/api/videos/video-channel-syncs.ts
@@ -220,7 +220,7 @@ describe('Test channel synchronizations', function () {
220 expect(total).to.equal(0) 220 expect(total).to.equal(0)
221 }) 221 })
222 222
223 // FIXME: youtube-dl doesn't work when speicifying a port after the hostname 223 // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
224 // it('Should import a remote PeerTube channel', async function () { 224 // it('Should import a remote PeerTube channel', async function () {
225 // this.timeout(240_000) 225 // this.timeout(240_000)
226 226
diff --git a/server/tests/api/videos/video-description.ts b/server/tests/api/videos/video-description.ts
index a4b3ff6e7..c4185882a 100644
--- a/server/tests/api/videos/video-description.ts
+++ b/server/tests/api/videos/video-description.ts
@@ -14,8 +14,12 @@ describe('Test video description', function () {
14 let servers: PeerTubeServer[] = [] 14 let servers: PeerTubeServer[] = []
15 let videoUUID = '' 15 let videoUUID = ''
16 let videoId: number 16 let videoId: number
17
17 const longDescription = 'my super description for server 1'.repeat(50) 18 const longDescription = 'my super description for server 1'.repeat(50)
18 19
20 // 30 characters * 6 -> 240 characters
21 const truncatedDescription = 'my super description for server 1'.repeat(7) + 'my super descrip...'
22
19 before(async function () { 23 before(async function () {
20 this.timeout(40000) 24 this.timeout(40000)
21 25
@@ -45,15 +49,22 @@ describe('Test video description', function () {
45 videoUUID = data[0].uuid 49 videoUUID = data[0].uuid
46 }) 50 })
47 51
48 it('Should have a truncated description on each server', async function () { 52 it('Should have a truncated description on each server when listing videos', async function () {
49 for (const server of servers) { 53 for (const server of servers) {
50 const video = await server.videos.get({ id: videoUUID }) 54 const { data } = await server.videos.list()
51 55 const video = data.find(v => v.uuid === videoUUID)
52 // 30 characters * 6 -> 240 characters
53 const truncatedDescription = 'my super description for server 1'.repeat(7) +
54 'my super descrip...'
55 56
56 expect(video.description).to.equal(truncatedDescription) 57 expect(video.description).to.equal(truncatedDescription)
58 expect(video.truncatedDescription).to.equal(truncatedDescription)
59 }
60 })
61
62 it('Should not have a truncated description on each server when getting videos', async function () {
63 for (const server of servers) {
64 const video = await server.videos.get({ id: videoUUID })
65
66 expect(video.description).to.equal(longDescription)
67 expect(video.truncatedDescription).to.equal(truncatedDescription)
57 } 68 }
58 }) 69 })
59 70
diff --git a/server/tests/api/videos/video-files.ts b/server/tests/api/videos/video-files.ts
index 10277b9cf..8c913bf31 100644
--- a/server/tests/api/videos/video-files.ts
+++ b/server/tests/api/videos/video-files.ts
@@ -33,7 +33,7 @@ describe('Test videos files', function () {
33 let validId2: string 33 let validId2: string
34 34
35 before(async function () { 35 before(async function () {
36 this.timeout(120_000) 36 this.timeout(360_000)
37 37
38 { 38 {
39 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) 39 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
@@ -153,7 +153,7 @@ describe('Test videos files', function () {
153 expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) 153 expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
154 expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist 154 expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist
155 155
156 const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl) 156 const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
157 157
158 expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false 158 expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
159 expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true 159 expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts
index 47b8c7b1e..a3de73ba5 100644
--- a/server/tests/api/videos/video-playlists.ts
+++ b/server/tests/api/videos/video-playlists.ts
@@ -23,6 +23,7 @@ import {
23 setDefaultVideoChannel, 23 setDefaultVideoChannel,
24 waitJobs 24 waitJobs
25} from '@shared/server-commands' 25} from '@shared/server-commands'
26import { uuidToShort } from '@shared/extra-utils'
26 27
27async function checkPlaylistElementType ( 28async function checkPlaylistElementType (
28 servers: PeerTubeServer[], 29 servers: PeerTubeServer[],
@@ -56,6 +57,7 @@ describe('Test video playlists', function () {
56 let playlistServer2UUID2: string 57 let playlistServer2UUID2: string
57 58
58 let playlistServer1Id: number 59 let playlistServer1Id: number
60 let playlistServer1DisplayName: string
59 let playlistServer1UUID: string 61 let playlistServer1UUID: string
60 let playlistServer1UUID2: string 62 let playlistServer1UUID2: string
61 63
@@ -70,7 +72,7 @@ describe('Test video playlists', function () {
70 let commands: PlaylistsCommand[] 72 let commands: PlaylistsCommand[]
71 73
72 before(async function () { 74 before(async function () {
73 this.timeout(120000) 75 this.timeout(240000)
74 76
75 servers = await createMultipleServers(3) 77 servers = await createMultipleServers(3)
76 78
@@ -489,15 +491,17 @@ describe('Test video playlists', function () {
489 return commands[0].addElement({ playlistId: playlistServer1Id, attributes }) 491 return commands[0].addElement({ playlistId: playlistServer1Id, attributes })
490 } 492 }
491 493
494 const playlistDisplayName = 'playlist 4'
492 const playlist = await commands[0].create({ 495 const playlist = await commands[0].create({
493 attributes: { 496 attributes: {
494 displayName: 'playlist 4', 497 displayName: playlistDisplayName,
495 privacy: VideoPlaylistPrivacy.PUBLIC, 498 privacy: VideoPlaylistPrivacy.PUBLIC,
496 videoChannelId: servers[0].store.channel.id 499 videoChannelId: servers[0].store.channel.id
497 } 500 }
498 }) 501 })
499 502
500 playlistServer1Id = playlist.id 503 playlistServer1Id = playlist.id
504 playlistServer1DisplayName = playlistDisplayName
501 playlistServer1UUID = playlist.uuid 505 playlistServer1UUID = playlist.uuid
502 506
503 await addVideo({ videoId: servers[0].store.videos[0].uuid, startTimestamp: 15, stopTimestamp: 28 }) 507 await addVideo({ videoId: servers[0].store.videos[0].uuid, startTimestamp: 15, stopTimestamp: 28 })
@@ -908,6 +912,8 @@ describe('Test video playlists', function () {
908 const elem = obj[servers[0].store.videos[0].id] 912 const elem = obj[servers[0].store.videos[0].id]
909 expect(elem).to.have.lengthOf(1) 913 expect(elem).to.have.lengthOf(1)
910 expect(elem[0].playlistElementId).to.exist 914 expect(elem[0].playlistElementId).to.exist
915 expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName)
916 expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID))
911 expect(elem[0].playlistId).to.equal(playlistServer1Id) 917 expect(elem[0].playlistId).to.equal(playlistServer1Id)
912 expect(elem[0].startTimestamp).to.equal(15) 918 expect(elem[0].startTimestamp).to.equal(15)
913 expect(elem[0].stopTimestamp).to.equal(28) 919 expect(elem[0].stopTimestamp).to.equal(28)
@@ -917,6 +923,8 @@ describe('Test video playlists', function () {
917 const elem = obj[servers[0].store.videos[3].id] 923 const elem = obj[servers[0].store.videos[3].id]
918 expect(elem).to.have.lengthOf(1) 924 expect(elem).to.have.lengthOf(1)
919 expect(elem[0].playlistElementId).to.equal(playlistElementServer1Video4) 925 expect(elem[0].playlistElementId).to.equal(playlistElementServer1Video4)
926 expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName)
927 expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID))
920 expect(elem[0].playlistId).to.equal(playlistServer1Id) 928 expect(elem[0].playlistId).to.equal(playlistServer1Id)
921 expect(elem[0].startTimestamp).to.equal(1) 929 expect(elem[0].startTimestamp).to.equal(1)
922 expect(elem[0].stopTimestamp).to.equal(35) 930 expect(elem[0].stopTimestamp).to.equal(35)
@@ -926,6 +934,8 @@ describe('Test video playlists', function () {
926 const elem = obj[servers[0].store.videos[4].id] 934 const elem = obj[servers[0].store.videos[4].id]
927 expect(elem).to.have.lengthOf(1) 935 expect(elem).to.have.lengthOf(1)
928 expect(elem[0].playlistId).to.equal(playlistServer1Id) 936 expect(elem[0].playlistId).to.equal(playlistServer1Id)
937 expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName)
938 expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID))
929 expect(elem[0].startTimestamp).to.equal(45) 939 expect(elem[0].startTimestamp).to.equal(45)
930 expect(elem[0].stopTimestamp).to.equal(null) 940 expect(elem[0].stopTimestamp).to.equal(null)
931 } 941 }
@@ -1049,7 +1059,7 @@ describe('Test video playlists', function () {
1049 this.timeout(30000) 1059 this.timeout(30000)
1050 1060
1051 for (const server of servers) { 1061 for (const server of servers) {
1052 await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber) 1062 await checkPlaylistFilesWereRemoved(playlistServer1UUID, server)
1053 } 1063 }
1054 }) 1064 })
1055 1065
diff --git a/server/tests/api/videos/video-privacy.ts b/server/tests/api/videos/video-privacy.ts
index b18c71c94..264a05d3f 100644
--- a/server/tests/api/videos/video-privacy.ts
+++ b/server/tests/api/videos/video-privacy.ts
@@ -45,7 +45,7 @@ describe('Test video privacy', function () {
45 describe('Private and internal videos', function () { 45 describe('Private and internal videos', function () {
46 46
47 it('Should upload a private and internal videos on server 1', async function () { 47 it('Should upload a private and internal videos on server 1', async function () {
48 this.timeout(10000) 48 this.timeout(50000)
49 49
50 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { 50 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
51 const attributes = { privacy } 51 const attributes = { privacy }
@@ -128,7 +128,7 @@ describe('Test video privacy', function () {
128 describe('Unlisted videos', function () { 128 describe('Unlisted videos', function () {
129 129
130 it('Should upload an unlisted video on server 2', async function () { 130 it('Should upload an unlisted video on server 2', async function () {
131 this.timeout(60000) 131 this.timeout(120000)
132 132
133 const attributes = { 133 const attributes = {
134 name: 'unlisted video', 134 name: 'unlisted video',
diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts
new file mode 100644
index 000000000..eaaed5aad
--- /dev/null
+++ b/server/tests/api/videos/video-static-file-privacy.ts
@@ -0,0 +1,422 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { decode } from 'magnet-uri'
5import { expectStartWith } from '@server/tests/shared'
6import { getAllFiles, wait } from '@shared/core-utils'
7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
8import {
9 cleanupTests,
10 createSingleServer,
11 findExternalSavedVideo,
12 makeRawRequest,
13 parseTorrentVideo,
14 PeerTubeServer,
15 sendRTMPStream,
16 setAccessTokensToServers,
17 setDefaultVideoChannel,
18 stopFfmpeg,
19 waitJobs
20} from '@shared/server-commands'
21
22describe('Test video static file privacy', function () {
23 let server: PeerTubeServer
24 let userToken: string
25
26 before(async function () {
27 this.timeout(50000)
28
29 server = await createSingleServer(1)
30 await setAccessTokensToServers([ server ])
31 await setDefaultVideoChannel([ server ])
32
33 userToken = await server.users.generateUserAndToken('user1')
34 })
35
36 describe('VOD static file path', function () {
37
38 function runSuite () {
39
40 async function checkPrivateFiles (uuid: string) {
41 const video = await server.videos.getWithToken({ id: uuid })
42
43 for (const file of video.files) {
44 expect(file.fileDownloadUrl).to.not.include('/private/')
45 expectStartWith(file.fileUrl, server.url + '/static/webseed/private/')
46
47 const torrent = await parseTorrentVideo(server, file)
48 expect(torrent.urlList).to.have.lengthOf(0)
49
50 const magnet = decode(file.magnetUri)
51 expect(magnet.urlList).to.have.lengthOf(0)
52
53 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
54 }
55
56 const hls = video.streamingPlaylists[0]
57 if (hls) {
58 expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/')
59 expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/')
60
61 await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
62 await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
63 }
64 }
65
66 async function checkPublicFiles (uuid: string) {
67 const video = await server.videos.get({ id: uuid })
68
69 for (const file of getAllFiles(video)) {
70 expect(file.fileDownloadUrl).to.not.include('/private/')
71 expect(file.fileUrl).to.not.include('/private/')
72
73 const torrent = await parseTorrentVideo(server, file)
74 expect(torrent.urlList[0]).to.not.include('private')
75
76 const magnet = decode(file.magnetUri)
77 expect(magnet.urlList[0]).to.not.include('private')
78
79 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
80 await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
81 await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
82 }
83
84 const hls = video.streamingPlaylists[0]
85 if (hls) {
86 expect(hls.playlistUrl).to.not.include('private')
87 expect(hls.segmentsSha256Url).to.not.include('private')
88
89 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
90 await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
91 }
92 }
93
94 it('Should upload a private/internal video and have a private static path', async function () {
95 this.timeout(120000)
96
97 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
98 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy })
99 await waitJobs([ server ])
100
101 await checkPrivateFiles(uuid)
102 }
103 })
104
105 it('Should upload a public video and update it as private/internal to have a private static path', async function () {
106 this.timeout(120000)
107
108 for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
109 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC })
110 await waitJobs([ server ])
111
112 await server.videos.update({ id: uuid, attributes: { privacy } })
113 await waitJobs([ server ])
114
115 await checkPrivateFiles(uuid)
116 }
117 })
118
119 it('Should upload a private video and update it to unlisted to have a public static path', async function () {
120 this.timeout(120000)
121
122 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
123 await waitJobs([ server ])
124
125 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } })
126 await waitJobs([ server ])
127
128 await checkPublicFiles(uuid)
129 })
130
131 it('Should upload an internal video and update it to public to have a public static path', async function () {
132 this.timeout(120000)
133
134 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
135 await waitJobs([ server ])
136
137 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
138 await waitJobs([ server ])
139
140 await checkPublicFiles(uuid)
141 })
142
143 it('Should upload an internal video and schedule a public publish', async function () {
144 this.timeout(120000)
145
146 const attributes = {
147 name: 'video',
148 privacy: VideoPrivacy.PRIVATE,
149 scheduleUpdate: {
150 updateAt: new Date(Date.now() + 1000).toISOString(),
151 privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC
152 }
153 }
154
155 const { uuid } = await server.videos.upload({ attributes })
156
157 await waitJobs([ server ])
158 await wait(1000)
159 await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } })
160
161 await waitJobs([ server ])
162
163 await checkPublicFiles(uuid)
164 })
165 }
166
167 describe('Without transcoding', function () {
168 runSuite()
169 })
170
171 describe('With transcoding', function () {
172
173 before(async function () {
174 await server.config.enableMinimumTranscoding()
175 })
176
177 runSuite()
178 })
179 })
180
181 describe('VOD static file right check', function () {
182 let unrelatedFileToken: string
183
184 async function checkVideoFiles (options: {
185 id: string
186 expectedStatus: HttpStatusCode
187 token: string
188 videoFileToken: string
189 }) {
190 const { id, expectedStatus, token, videoFileToken } = options
191
192 const video = await server.videos.getWithToken({ id })
193
194 for (const file of getAllFiles(video)) {
195 await makeRawRequest({ url: file.fileUrl, token, expectedStatus })
196 await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus })
197
198 await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus })
199 await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus })
200 }
201
202 const hls = video.streamingPlaylists[0]
203 await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus })
204 await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus })
205
206 await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus })
207 await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus })
208 }
209
210 before(async function () {
211 await server.config.enableMinimumTranscoding()
212
213 const { uuid } = await server.videos.quickUpload({ name: 'another video' })
214 unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
215 })
216
217 it('Should not be able to access a private video files without OAuth token and file token', async function () {
218 this.timeout(120000)
219
220 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
221 await waitJobs([ server ])
222
223 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null })
224 })
225
226 it('Should not be able to access an internal video files without appropriate OAuth token and file token', async function () {
227 this.timeout(120000)
228
229 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
230 await waitJobs([ server ])
231
232 await checkVideoFiles({
233 id: uuid,
234 expectedStatus: HttpStatusCode.FORBIDDEN_403,
235 token: userToken,
236 videoFileToken: unrelatedFileToken
237 })
238 })
239
240 it('Should be able to access a private video files with appropriate OAuth token or file token', async function () {
241 this.timeout(120000)
242
243 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
244 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
245
246 await waitJobs([ server ])
247
248 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
249 })
250
251 it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () {
252 this.timeout(120000)
253
254 const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE })
255 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
256
257 await waitJobs([ server ])
258
259 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
260 })
261 })
262
263 describe('Live static file path and check', function () {
264 let normalLiveId: string
265 let normalLive: LiveVideo
266
267 let permanentLiveId: string
268 let permanentLive: LiveVideo
269
270 let unrelatedFileToken: string
271
272 async function checkLiveFiles (live: LiveVideo, liveId: string) {
273 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
274 await server.live.waitUntilPublished({ videoId: liveId })
275
276 const video = await server.videos.getWithToken({ id: liveId })
277 const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
278
279 const hls = video.streamingPlaylists[0]
280
281 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
282 expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
283
284 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
285 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
286
287 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
288 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
289 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
290 }
291
292 await stopFfmpeg(ffmpegCommand)
293 }
294
295 async function checkReplay (replay: VideoDetails) {
296 const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
297
298 const hls = replay.streamingPlaylists[0]
299 expect(hls.files).to.not.have.lengthOf(0)
300
301 for (const file of hls.files) {
302 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
303 await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
304
305 await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
306 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
307 await makeRawRequest({
308 url: file.fileUrl,
309 query: { videoFileToken: unrelatedFileToken },
310 expectedStatus: HttpStatusCode.FORBIDDEN_403
311 })
312 }
313
314 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
315 expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
316
317 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
318 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
319
320 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
321 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
322 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
323 }
324 }
325
326 before(async function () {
327 await server.config.enableMinimumTranscoding()
328
329 const { uuid } = await server.videos.quickUpload({ name: 'another video' })
330 unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
331
332 await server.config.enableLive({
333 allowReplay: true,
334 transcoding: true,
335 resolutions: 'min'
336 })
337
338 {
339 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
340 normalLiveId = video.uuid
341 normalLive = live
342 }
343
344 {
345 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
346 permanentLiveId = video.uuid
347 permanentLive = live
348 }
349 })
350
351 it('Should create a private normal live and have a private static path', async function () {
352 this.timeout(240000)
353
354 await checkLiveFiles(normalLive, normalLiveId)
355 })
356
357 it('Should create a private permanent live and have a private static path', async function () {
358 this.timeout(240000)
359
360 await checkLiveFiles(permanentLive, permanentLiveId)
361 })
362
363 it('Should have created a replay of the normal live with a private static path', async function () {
364 this.timeout(240000)
365
366 await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
367
368 const replay = await server.videos.getWithToken({ id: normalLiveId })
369 await checkReplay(replay)
370 })
371
372 it('Should have created a replay of the permanent live with a private static path', async function () {
373 this.timeout(240000)
374
375 await server.live.waitUntilWaiting({ videoId: permanentLiveId })
376 await waitJobs([ server ])
377
378 const live = await server.videos.getWithToken({ id: permanentLiveId })
379 const replayFromList = await findExternalSavedVideo(server, live)
380 const replay = await server.videos.getWithToken({ id: replayFromList.id })
381
382 await checkReplay(replay)
383 })
384 })
385
386 describe('With static file right check disabled', function () {
387 let videoUUID: string
388
389 before(async function () {
390 this.timeout(240000)
391
392 await server.kill()
393
394 await server.run({
395 static_files: {
396 private_files_require_auth: false
397 }
398 })
399
400 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
401 videoUUID = uuid
402
403 await waitJobs([ server ])
404 })
405
406 it('Should not check auth for private static files', async function () {
407 const video = await server.videos.getWithToken({ id: videoUUID })
408
409 for (const file of getAllFiles(video)) {
410 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
411 }
412
413 const hls = video.streamingPlaylists[0]
414 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
415 await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
416 })
417 })
418
419 after(async function () {
420 await cleanupTests([ server ])
421 })
422})
diff --git a/server/tests/api/videos/videos-common-filters.ts b/server/tests/api/videos/videos-common-filters.ts
index e7fc15e42..b176d90ab 100644
--- a/server/tests/api/videos/videos-common-filters.ts
+++ b/server/tests/api/videos/videos-common-filters.ts
@@ -232,7 +232,7 @@ describe('Test videos filter', function () {
232 }) 232 })
233 233
234 it('Should display only remote videos', async function () { 234 it('Should display only remote videos', async function () {
235 this.timeout(40000) 235 this.timeout(120000)
236 236
237 await servers[1].videos.upload({ attributes: { name: 'remote video' } }) 237 await servers[1].videos.upload({ attributes: { name: 'remote video' } })
238 238
diff --git a/server/tests/api/views/video-views-counter.ts b/server/tests/api/views/video-views-counter.ts
index ca33ff9cd..0c1b7859c 100644
--- a/server/tests/api/views/video-views-counter.ts
+++ b/server/tests/api/views/video-views-counter.ts
@@ -76,7 +76,7 @@ describe('Test video views/viewers counters', function () {
76 let command: FfmpegCommand 76 let command: FfmpegCommand
77 77
78 before(async function () { 78 before(async function () {
79 this.timeout(120000); 79 this.timeout(240000);
80 80
81 ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) 81 ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true }))
82 }) 82 })
diff --git a/server/tests/api/views/video-views-overall-stats.ts b/server/tests/api/views/video-views-overall-stats.ts
index bb00684ef..3aadc9689 100644
--- a/server/tests/api/views/video-views-overall-stats.ts
+++ b/server/tests/api/views/video-views-overall-stats.ts
@@ -20,7 +20,7 @@ describe('Test views overall stats', function () {
20 let command: FfmpegCommand 20 let command: FfmpegCommand
21 21
22 before(async function () { 22 before(async function () {
23 this.timeout(120000); 23 this.timeout(240000);
24 24
25 ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) 25 ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true }))
26 }) 26 })
@@ -179,7 +179,7 @@ describe('Test views overall stats', function () {
179 let before2Watchers: Date 179 let before2Watchers: Date
180 180
181 before(async function () { 181 before(async function () {
182 this.timeout(120000); 182 this.timeout(240000);
183 183
184 ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) 184 ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true }))
185 }) 185 })
diff --git a/server/tests/api/views/video-views-retention-stats.ts b/server/tests/api/views/video-views-retention-stats.ts
index 621b05110..5b9ce4c92 100644
--- a/server/tests/api/views/video-views-retention-stats.ts
+++ b/server/tests/api/views/video-views-retention-stats.ts
@@ -17,7 +17,7 @@ describe('Test views retention stats', function () {
17 let vodVideoId: string 17 let vodVideoId: string
18 18
19 before(async function () { 19 before(async function () {
20 this.timeout(120000); 20 this.timeout(240000);
21 21
22 ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) 22 ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true }))
23 }) 23 })
diff --git a/server/tests/api/views/video-views-timeserie-stats.ts b/server/tests/api/views/video-views-timeserie-stats.ts
index e8cb34ad6..2d991d7ea 100644
--- a/server/tests/api/views/video-views-timeserie-stats.ts
+++ b/server/tests/api/views/video-views-timeserie-stats.ts
@@ -30,7 +30,7 @@ describe('Test views timeserie stats', function () {
30 let vodVideoId: string 30 let vodVideoId: string
31 31
32 before(async function () { 32 before(async function () {
33 this.timeout(120000); 33 this.timeout(240000);
34 34
35 ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) 35 ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true }))
36 }) 36 })
@@ -81,7 +81,7 @@ describe('Test views timeserie stats', function () {
81 } 81 }
82 82
83 before(async function () { 83 before(async function () {
84 this.timeout(120000); 84 this.timeout(240000);
85 85
86 ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) 86 ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true }))
87 }) 87 })
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
index 2cf2dd8f8..43f53035b 100644
--- a/server/tests/cli/create-import-video-file-job.ts
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -1,7 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { areObjectStorageTestsDisabled } from '@shared/core-utils' 4import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
5import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models' 5import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models'
6import { 6import {
7 cleanupTests, 7 cleanupTests,
@@ -27,9 +27,9 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s
27 27
28async function checkFiles (video: VideoDetails, objectStorage: boolean) { 28async function checkFiles (video: VideoDetails, objectStorage: boolean) {
29 for (const file of video.files) { 29 for (const file of video.files) {
30 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 30 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
31 31
32 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 32 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
33 } 33 }
34} 34}
35 35
@@ -43,7 +43,7 @@ function runTests (objectStorage: boolean) {
43 this.timeout(90000) 43 this.timeout(90000)
44 44
45 const config = objectStorage 45 const config = objectStorage
46 ? ObjectStorageCommand.getDefaultConfig() 46 ? ObjectStorageCommand.getDefaultMockConfig()
47 : {} 47 : {}
48 48
49 // Run server 2 to have transcoding enabled 49 // Run server 2 to have transcoding enabled
@@ -52,7 +52,7 @@ function runTests (objectStorage: boolean) {
52 52
53 await doubleFollow(servers[0], servers[1]) 53 await doubleFollow(servers[0], servers[1])
54 54
55 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() 55 if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
56 56
57 // Upload two videos for our needs 57 // Upload two videos for our needs
58 { 58 {
@@ -157,7 +157,7 @@ describe('Test create import video jobs', function () {
157 }) 157 })
158 158
159 describe('On object storage', function () { 159 describe('On object storage', function () {
160 if (areObjectStorageTestsDisabled()) return 160 if (areMockObjectStorageTestsDisabled()) return
161 161
162 runTests(true) 162 runTests(true)
163 }) 163 })
diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts
index 6a12a2c6c..c357f501b 100644
--- a/server/tests/cli/create-move-video-storage-job.ts
+++ b/server/tests/cli/create-move-video-storage-job.ts
@@ -1,6 +1,6 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { areObjectStorageTestsDisabled } from '@shared/core-utils' 3import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
4import { HttpStatusCode, VideoDetails } from '@shared/models' 4import { HttpStatusCode, VideoDetails } from '@shared/models'
5import { 5import {
6 cleanupTests, 6 cleanupTests,
@@ -17,16 +17,16 @@ import { expectStartWith } from '../shared'
17async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) { 17async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
18 for (const file of video.files) { 18 for (const file of video.files) {
19 const start = inObjectStorage 19 const start = inObjectStorage
20 ? ObjectStorageCommand.getWebTorrentBaseUrl() 20 ? ObjectStorageCommand.getMockWebTorrentBaseUrl()
21 : origin.url 21 : origin.url
22 22
23 expectStartWith(file.fileUrl, start) 23 expectStartWith(file.fileUrl, start)
24 24
25 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 25 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
26 } 26 }
27 27
28 const start = inObjectStorage 28 const start = inObjectStorage
29 ? ObjectStorageCommand.getPlaylistBaseUrl() 29 ? ObjectStorageCommand.getMockPlaylistBaseUrl()
30 : origin.url 30 : origin.url
31 31
32 const hls = video.streamingPlaylists[0] 32 const hls = video.streamingPlaylists[0]
@@ -36,12 +36,12 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
36 for (const file of hls.files) { 36 for (const file of hls.files) {
37 expectStartWith(file.fileUrl, start) 37 expectStartWith(file.fileUrl, start)
38 38
39 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 39 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
40 } 40 }
41} 41}
42 42
43describe('Test create move video storage job', function () { 43describe('Test create move video storage job', function () {
44 if (areObjectStorageTestsDisabled()) return 44 if (areMockObjectStorageTestsDisabled()) return
45 45
46 let servers: PeerTubeServer[] = [] 46 let servers: PeerTubeServer[] = []
47 const uuids: string[] = [] 47 const uuids: string[] = []
@@ -55,7 +55,7 @@ describe('Test create move video storage job', function () {
55 55
56 await doubleFollow(servers[0], servers[1]) 56 await doubleFollow(servers[0], servers[1])
57 57
58 await ObjectStorageCommand.prepareDefaultBuckets() 58 await ObjectStorageCommand.prepareDefaultMockBuckets()
59 59
60 await servers[0].config.enableTranscoding() 60 await servers[0].config.enableTranscoding()
61 61
@@ -67,14 +67,14 @@ describe('Test create move video storage job', function () {
67 await waitJobs(servers) 67 await waitJobs(servers)
68 68
69 await servers[0].kill() 69 await servers[0].kill()
70 await servers[0].run(ObjectStorageCommand.getDefaultConfig()) 70 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
71 }) 71 })
72 72
73 it('Should move only one file', async function () { 73 it('Should move only one file', async function () {
74 this.timeout(120000) 74 this.timeout(120000)
75 75
76 const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` 76 const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
77 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) 77 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
78 await waitJobs(servers) 78 await waitJobs(servers)
79 79
80 for (const server of servers) { 80 for (const server of servers) {
@@ -94,7 +94,7 @@ describe('Test create move video storage job', function () {
94 this.timeout(120000) 94 this.timeout(120000)
95 95
96 const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` 96 const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
97 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) 97 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
98 await waitJobs(servers) 98 await waitJobs(servers)
99 99
100 for (const server of servers) { 100 for (const server of servers) {
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index 8897d8c23..38b737829 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -1,7 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { areObjectStorageTestsDisabled } from '@shared/core-utils' 4import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
5import { HttpStatusCode, VideoFile } from '@shared/models' 5import { HttpStatusCode, VideoFile } from '@shared/models'
6import { 6import {
7 cleanupTests, 7 cleanupTests,
@@ -18,12 +18,12 @@ import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared'
18async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { 18async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') {
19 for (const file of files) { 19 for (const file of files) {
20 const shouldStartWith = type === 'webtorrent' 20 const shouldStartWith = type === 'webtorrent'
21 ? ObjectStorageCommand.getWebTorrentBaseUrl() 21 ? ObjectStorageCommand.getMockWebTorrentBaseUrl()
22 : ObjectStorageCommand.getPlaylistBaseUrl() 22 : ObjectStorageCommand.getMockPlaylistBaseUrl()
23 23
24 expectStartWith(file.fileUrl, shouldStartWith) 24 expectStartWith(file.fileUrl, shouldStartWith)
25 25
26 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) 26 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
27 } 27 }
28} 28}
29 29
@@ -36,7 +36,7 @@ function runTests (objectStorage: boolean) {
36 this.timeout(120000) 36 this.timeout(120000)
37 37
38 const config = objectStorage 38 const config = objectStorage
39 ? ObjectStorageCommand.getDefaultConfig() 39 ? ObjectStorageCommand.getDefaultMockConfig()
40 : {} 40 : {}
41 41
42 // Run server 2 to have transcoding enabled 42 // Run server 2 to have transcoding enabled
@@ -47,7 +47,7 @@ function runTests (objectStorage: boolean) {
47 47
48 await doubleFollow(servers[0], servers[1]) 48 await doubleFollow(servers[0], servers[1])
49 49
50 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() 50 if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
51 51
52 for (let i = 1; i <= 5; i++) { 52 for (let i = 1; i <= 5; i++) {
53 const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) 53 const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
@@ -255,7 +255,7 @@ describe('Test create transcoding jobs', function () {
255 }) 255 })
256 256
257 describe('On object storage', function () { 257 describe('On object storage', function () {
258 if (areObjectStorageTestsDisabled()) return 258 if (areMockObjectStorageTestsDisabled()) return
259 259
260 runTests(true) 260 runTests(true)
261 }) 261 })
diff --git a/server/tests/cli/peertube.ts b/server/tests/cli/peertube.ts
index 9b17cdd1b..a39bcfebe 100644
--- a/server/tests/cli/peertube.ts
+++ b/server/tests/cli/peertube.ts
@@ -24,7 +24,13 @@ describe('Test CLI wrapper', function () {
24 before(async function () { 24 before(async function () {
25 this.timeout(30000) 25 this.timeout(30000)
26 26
27 server = await createSingleServer(1) 27 server = await createSingleServer(1, {
28 rates_limit: {
29 login: {
30 max: 30
31 }
32 }
33 })
28 await setAccessTokensToServers([ server ]) 34 await setAccessTokensToServers([ server ])
29 35
30 await server.users.create({ username: 'user_1', password: 'super_password' }) 36 await server.users.create({ username: 'user_1', password: 'super_password' })
@@ -240,6 +246,19 @@ describe('Test CLI wrapper', function () {
240 246
241 expect(res).to.not.contain('peertube-plugin-hello-world') 247 expect(res).to.not.contain('peertube-plugin-hello-world')
242 }) 248 })
249
250 it('Should install a plugin in requested beta version', async function () {
251 this.timeout(60000)
252
253 await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world --plugin-version 0.0.21-beta.1`)
254
255 const res = await cliCommand.execWithEnv(`${cmd} plugins list`)
256
257 expect(res).to.contain('peertube-plugin-hello-world')
258 expect(res).to.contain('0.0.21-beta.1')
259
260 await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`)
261 })
243 }) 262 })
244 263
245 describe('Manage video redundancies', function () { 264 describe('Manage video redundancies', function () {
diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts
index a89e17e3c..ba0fa1f86 100644
--- a/server/tests/cli/prune-storage.ts
+++ b/server/tests/cli/prune-storage.ts
@@ -5,7 +5,7 @@ import { createFile, readdir } from 'fs-extra'
5import { join } from 'path' 5import { join } from 'path'
6import { wait } from '@shared/core-utils' 6import { wait } from '@shared/core-utils'
7import { buildUUID } from '@shared/extra-utils' 7import { buildUUID } from '@shared/extra-utils'
8import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' 8import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
9import { 9import {
10 cleanupTests, 10 cleanupTests,
11 CLICommand, 11 CLICommand,
@@ -36,22 +36,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
36async function assertCountAreOkay (servers: PeerTubeServer[]) { 36async function assertCountAreOkay (servers: PeerTubeServer[]) {
37 for (const server of servers) { 37 for (const server of servers) {
38 const videosCount = await countFiles(server, 'videos') 38 const videosCount = await countFiles(server, 'videos')
39 expect(videosCount).to.equal(8) 39 expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
40
41 const privateVideosCount = await countFiles(server, 'videos/private')
42 expect(privateVideosCount).to.equal(4)
40 43
41 const torrentsCount = await countFiles(server, 'torrents') 44 const torrentsCount = await countFiles(server, 'torrents')
42 expect(torrentsCount).to.equal(16) 45 expect(torrentsCount).to.equal(24)
43 46
44 const previewsCount = await countFiles(server, 'previews') 47 const previewsCount = await countFiles(server, 'previews')
45 expect(previewsCount).to.equal(2) 48 expect(previewsCount).to.equal(3)
46 49
47 const thumbnailsCount = await countFiles(server, 'thumbnails') 50 const thumbnailsCount = await countFiles(server, 'thumbnails')
48 expect(thumbnailsCount).to.equal(6) 51 expect(thumbnailsCount).to.equal(7) // 3 local videos, 1 local playlist, 2 remotes videos and 1 remote playlist
49 52
50 const avatarsCount = await countFiles(server, 'avatars') 53 const avatarsCount = await countFiles(server, 'avatars')
51 expect(avatarsCount).to.equal(4) 54 expect(avatarsCount).to.equal(4)
52 55
53 const hlsRootCount = await countFiles(server, 'streaming-playlists/hls') 56 const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls'))
54 expect(hlsRootCount).to.equal(2) 57 expect(hlsRootCount).to.equal(3) // 2 videos + private directory
58
59 const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private'))
60 expect(hlsPrivateRootCount).to.equal(1)
55 } 61 }
56} 62}
57 63
@@ -67,8 +73,10 @@ describe('Test prune storage scripts', function () {
67 await setDefaultVideoChannel(servers) 73 await setDefaultVideoChannel(servers)
68 74
69 for (const server of servers) { 75 for (const server of servers) {
70 await server.videos.upload({ attributes: { name: 'video 1' } }) 76 await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
71 await server.videos.upload({ attributes: { name: 'video 2' } }) 77 await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
78
79 await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
72 80
73 await server.users.updateMyAvatar({ fixture: 'avatar.png' }) 81 await server.users.updateMyAvatar({ fixture: 'avatar.png' })
74 82
@@ -123,13 +131,16 @@ describe('Test prune storage scripts', function () {
123 it('Should create some dirty files', async function () { 131 it('Should create some dirty files', async function () {
124 for (let i = 0; i < 2; i++) { 132 for (let i = 0; i < 2; i++) {
125 { 133 {
126 const base = servers[0].servers.buildDirectory('videos') 134 const basePublic = servers[0].servers.buildDirectory('videos')
135 const basePrivate = servers[0].servers.buildDirectory(join('videos', 'private'))
127 136
128 const n1 = buildUUID() + '.mp4' 137 const n1 = buildUUID() + '.mp4'
129 const n2 = buildUUID() + '.webm' 138 const n2 = buildUUID() + '.webm'
130 139
131 await createFile(join(base, n1)) 140 await createFile(join(basePublic, n1))
132 await createFile(join(base, n2)) 141 await createFile(join(basePublic, n2))
142 await createFile(join(basePrivate, n1))
143 await createFile(join(basePrivate, n2))
133 144
134 badNames['videos'] = [ n1, n2 ] 145 badNames['videos'] = [ n1, n2 ]
135 } 146 }
@@ -184,10 +195,12 @@ describe('Test prune storage scripts', function () {
184 195
185 { 196 {
186 const directory = join('streaming-playlists', 'hls') 197 const directory = join('streaming-playlists', 'hls')
187 const base = servers[0].servers.buildDirectory(directory) 198 const basePublic = servers[0].servers.buildDirectory(directory)
199 const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
188 200
189 const n1 = buildUUID() 201 const n1 = buildUUID()
190 await createFile(join(base, n1)) 202 await createFile(join(basePublic, n1))
203 await createFile(join(basePrivate, n1))
191 badNames[directory] = [ n1 ] 204 badNames[directory] = [ n1 ]
192 } 205 }
193 } 206 }
diff --git a/server/tests/cli/regenerate-thumbnails.ts b/server/tests/cli/regenerate-thumbnails.ts
index f459b11b8..16a8adcda 100644
--- a/server/tests/cli/regenerate-thumbnails.ts
+++ b/server/tests/cli/regenerate-thumbnails.ts
@@ -6,7 +6,7 @@ import {
6 cleanupTests, 6 cleanupTests,
7 createMultipleServers, 7 createMultipleServers,
8 doubleFollow, 8 doubleFollow,
9 makeRawRequest, 9 makeGetRequest,
10 PeerTubeServer, 10 PeerTubeServer,
11 setAccessTokensToServers, 11 setAccessTokensToServers,
12 waitJobs 12 waitJobs
@@ -16,8 +16,8 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string)
16 const video = await server.videos.get({ id: videoId }) 16 const video = await server.videos.get({ id: videoId })
17 17
18 const requests = [ 18 const requests = [
19 makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200), 19 makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }),
20 makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200) 20 makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
21 ] 21 ]
22 22
23 for (const req of requests) { 23 for (const req of requests) {
@@ -69,17 +69,17 @@ describe('Test regenerate thumbnails script', function () {
69 69
70 it('Should have empty thumbnails', async function () { 70 it('Should have empty thumbnails', async function () {
71 { 71 {
72 const res = await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.OK_200) 72 const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
73 expect(res.body).to.have.lengthOf(0) 73 expect(res.body).to.have.lengthOf(0)
74 } 74 }
75 75
76 { 76 {
77 const res = await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.OK_200) 77 const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
78 expect(res.body).to.not.have.lengthOf(0) 78 expect(res.body).to.not.have.lengthOf(0)
79 } 79 }
80 80
81 { 81 {
82 const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) 82 const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
83 expect(res.body).to.have.lengthOf(0) 83 expect(res.body).to.have.lengthOf(0)
84 } 84 }
85 }) 85 })
@@ -94,21 +94,21 @@ describe('Test regenerate thumbnails script', function () {
94 await testThumbnail(servers[0], video1.uuid) 94 await testThumbnail(servers[0], video1.uuid)
95 await testThumbnail(servers[0], video2.uuid) 95 await testThumbnail(servers[0], video2.uuid)
96 96
97 const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) 97 const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
98 expect(res.body).to.have.lengthOf(0) 98 expect(res.body).to.have.lengthOf(0)
99 }) 99 })
100 100
101 it('Should have deleted old thumbnail files', async function () { 101 it('Should have deleted old thumbnail files', async function () {
102 { 102 {
103 await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.NOT_FOUND_404) 103 await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
104 } 104 }
105 105
106 { 106 {
107 await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.NOT_FOUND_404) 107 await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
108 } 108 }
109 109
110 { 110 {
111 const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) 111 const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
112 expect(res.body).to.have.lengthOf(0) 112 expect(res.body).to.have.lengthOf(0)
113 } 113 }
114 }) 114 })
diff --git a/server/tests/external-plugins/akismet.ts b/server/tests/external-plugins/akismet.ts
new file mode 100644
index 000000000..974bf0011
--- /dev/null
+++ b/server/tests/external-plugins/akismet.ts
@@ -0,0 +1,160 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { HttpStatusCode } from '@shared/models'
5import {
6 cleanupTests,
7 createMultipleServers,
8 doubleFollow,
9 PeerTubeServer,
10 setAccessTokensToServers,
11 waitJobs
12} from '@shared/server-commands'
13
14describe('Official plugin Akismet', function () {
15 let servers: PeerTubeServer[]
16 let videoUUID: string
17
18 before(async function () {
19 this.timeout(30000)
20
21 servers = await createMultipleServers(2)
22 await setAccessTokensToServers(servers)
23
24 await servers[0].plugins.install({
25 npmName: 'peertube-plugin-akismet'
26 })
27
28 if (!process.env.AKISMET_KEY) throw new Error('Missing AKISMET_KEY from env')
29
30 await servers[0].plugins.updateSettings({
31 npmName: 'peertube-plugin-akismet',
32 settings: {
33 'akismet-api-key': process.env.AKISMET_KEY
34 }
35 })
36
37 await doubleFollow(servers[0], servers[1])
38 })
39
40 describe('Local threads/replies', function () {
41
42 before(async function () {
43 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
44 videoUUID = uuid
45 })
46
47 it('Should not detect a thread as spam', async function () {
48 await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' })
49 })
50
51 it('Should not detect a reply as spam', async function () {
52 await servers[0].comments.addReplyToLastThread({ text: 'reply' })
53 })
54
55 it('Should detect a thread as spam', async function () {
56 await servers[0].comments.createThread({
57 videoId: videoUUID,
58 text: 'akismet-guaranteed-spam',
59 expectedStatus: HttpStatusCode.FORBIDDEN_403
60 })
61 })
62
63 it('Should detect a thread as spam', async function () {
64 await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' })
65 await servers[0].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
66 })
67 })
68
69 describe('Remote threads/replies', function () {
70
71 before(async function () {
72 this.timeout(60000)
73
74 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
75 videoUUID = uuid
76
77 await waitJobs(servers)
78 })
79
80 it('Should not detect a thread as spam', async function () {
81 this.timeout(30000)
82
83 await servers[1].comments.createThread({ videoId: videoUUID, text: 'remote comment 1' })
84 await waitJobs(servers)
85
86 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
87 expect(data).to.have.lengthOf(1)
88 })
89
90 it('Should not detect a reply as spam', async function () {
91 this.timeout(30000)
92
93 await servers[1].comments.addReplyToLastThread({ text: 'I agree with you' })
94 await waitJobs(servers)
95
96 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
97 expect(data).to.have.lengthOf(1)
98
99 const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: data[0].id })
100 expect(tree.children).to.have.lengthOf(1)
101 })
102
103 it('Should detect a thread as spam', async function () {
104 this.timeout(30000)
105
106 await servers[1].comments.createThread({ videoId: videoUUID, text: 'akismet-guaranteed-spam' })
107 await waitJobs(servers)
108
109 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
110 expect(data).to.have.lengthOf(1)
111 })
112
113 it('Should detect a thread as spam', async function () {
114 this.timeout(30000)
115
116 await servers[1].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam' })
117 await waitJobs(servers)
118
119 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID })
120 expect(data).to.have.lengthOf(1)
121
122 const thread = data[0]
123 const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: thread.id })
124 expect(tree.children).to.have.lengthOf(1)
125 })
126 })
127
128 describe('Signup', function () {
129
130 before(async function () {
131 await servers[0].config.updateExistingSubConfig({
132 newConfig: {
133 signup: {
134 enabled: true
135 }
136 }
137 })
138 })
139
140 it('Should allow signup', async function () {
141 await servers[0].users.register({
142 username: 'user1',
143 displayName: 'user 1'
144 })
145 })
146
147 it('Should detect a signup as SPAM', async function () {
148 await servers[0].users.register({
149 username: 'user2',
150 displayName: 'user 2',
151 email: 'akismet-guaranteed-spam@example.com',
152 expectedStatus: HttpStatusCode.FORBIDDEN_403
153 })
154 })
155 })
156
157 after(async function () {
158 await cleanupTests(servers)
159 })
160})
diff --git a/server/tests/external-plugins/auth-ldap.ts b/server/tests/external-plugins/auth-ldap.ts
index d7f155d2a..6f6a574a0 100644
--- a/server/tests/external-plugins/auth-ldap.ts
+++ b/server/tests/external-plugins/auth-ldap.ts
@@ -94,6 +94,14 @@ describe('Official plugin auth-ldap', function () {
94 await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } }) 94 await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } })
95 }) 95 })
96 96
97 it('Should not be able to ask password reset', async function () {
98 await server.users.askResetPassword({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 })
99 })
100
101 it('Should not be able to ask email verification', async function () {
102 await server.users.askSendVerifyEmail({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 })
103 })
104
97 it('Should not login if the plugin is uninstalled', async function () { 105 it('Should not login if the plugin is uninstalled', async function () {
98 await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' }) 106 await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' })
99 107
diff --git a/server/tests/external-plugins/index.ts b/server/tests/external-plugins/index.ts
index 31d818b51..815bbf1da 100644
--- a/server/tests/external-plugins/index.ts
+++ b/server/tests/external-plugins/index.ts
@@ -1,3 +1,4 @@
1import './akismet'
1import './auth-ldap' 2import './auth-ldap'
2import './auto-block-videos' 3import './auto-block-videos'
3import './auto-mute' 4import './auto-mute'
diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts
index 1d3c03d67..906dab1a3 100644
--- a/server/tests/feeds/feeds.ts
+++ b/server/tests/feeds/feeds.ts
@@ -9,6 +9,7 @@ import {
9 createSingleServer, 9 createSingleServer,
10 doubleFollow, 10 doubleFollow,
11 makeGetRequest, 11 makeGetRequest,
12 makeRawRequest,
12 PeerTubeServer, 13 PeerTubeServer,
13 setAccessTokensToServers, 14 setAccessTokensToServers,
14 setDefaultChannelAvatar, 15 setDefaultChannelAvatar,
@@ -306,6 +307,15 @@ describe('Test syndication feeds', () => {
306 307
307 await stopFfmpeg(ffmpeg) 308 await stopFfmpeg(ffmpeg)
308 }) 309 })
310
311 it('Should have the channel avatar as feed icon', async function () {
312 const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: rootChannelId }, ignoreCache: true })
313
314 const jsonObj = JSON.parse(json)
315 const imageUrl = jsonObj.icon
316 expect(imageUrl).to.include('/lazy-static/avatars/')
317 await makeRawRequest({ url: imageUrl, expectedStatus: HttpStatusCode.OK_200 })
318 })
309 }) 319 })
310 320
311 describe('Video comments feed', function () { 321 describe('Video comments feed', function () {
diff --git a/server/tests/fixtures/peertube-plugin-test-four/main.js b/server/tests/fixtures/peertube-plugin-test-four/main.js
index 5194e3e02..3e848c49e 100644
--- a/server/tests/fixtures/peertube-plugin-test-four/main.js
+++ b/server/tests/fixtures/peertube-plugin-test-four/main.js
@@ -128,6 +128,22 @@ async function register ({
128 128
129 return res.json(result) 129 return res.json(result)
130 }) 130 })
131
132 router.post('/send-notification', async (req, res) => {
133 peertubeHelpers.socket.sendNotification(req.body.userId, {
134 type: 1,
135 userId: req.body.userId
136 })
137
138 return res.sendStatus(201)
139 })
140
141 router.post('/send-video-live-new-state/:uuid', async (req, res) => {
142 const video = await peertubeHelpers.videos.loadByIdOrUUID(req.params.uuid)
143 peertubeHelpers.socket.sendVideoLiveNewState(video)
144
145 return res.sendStatus(201)
146 })
131 } 147 }
132 148
133} 149}
diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/main.js b/server/tests/fixtures/peertube-plugin-test-websocket/main.js
new file mode 100644
index 000000000..3fde76cfe
--- /dev/null
+++ b/server/tests/fixtures/peertube-plugin-test-websocket/main.js
@@ -0,0 +1,36 @@
1const WebSocketServer = require('ws').WebSocketServer
2
3async function register ({
4 registerWebSocketRoute
5}) {
6 const wss = new WebSocketServer({ noServer: true })
7
8 wss.on('connection', function connection(ws) {
9 ws.on('message', function message(data) {
10 if (data.toString() === 'ping') {
11 ws.send('pong')
12 }
13 })
14 })
15
16 registerWebSocketRoute({
17 route: '/toto',
18
19 handler: (request, socket, head) => {
20 wss.handleUpgrade(request, socket, head, ws => {
21 wss.emit('connection', ws, request)
22 })
23 }
24 })
25}
26
27async function unregister () {
28 return
29}
30
31module.exports = {
32 register,
33 unregister
34}
35
36// ###########################################################################
diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/package.json b/server/tests/fixtures/peertube-plugin-test-websocket/package.json
new file mode 100644
index 000000000..89c8baa04
--- /dev/null
+++ b/server/tests/fixtures/peertube-plugin-test-websocket/package.json
@@ -0,0 +1,20 @@
1{
2 "name": "peertube-plugin-test-websocket",
3 "version": "0.0.1",
4 "description": "Plugin test websocket",
5 "engine": {
6 "peertube": ">=1.3.0"
7 },
8 "keywords": [
9 "peertube",
10 "plugin"
11 ],
12 "homepage": "https://github.com/Chocobozzz/PeerTube",
13 "author": "Chocobozzz",
14 "bugs": "https://github.com/Chocobozzz/PeerTube/issues",
15 "library": "./main.js",
16 "staticDirs": {},
17 "css": [],
18 "clientScripts": [],
19 "translations": {}
20}
diff --git a/server/tests/fixtures/peertube-plugin-test/main.js b/server/tests/fixtures/peertube-plugin-test/main.js
index 813482a27..19dccf26e 100644
--- a/server/tests/fixtures/peertube-plugin-test/main.js
+++ b/server/tests/fixtures/peertube-plugin-test/main.js
@@ -178,6 +178,8 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
178 } 178 }
179 }) 179 })
180 180
181 // ---------------------------------------------------------------------------
182
181 registerHook({ 183 registerHook({
182 target: 'filter:api.video-thread.create.accept.result', 184 target: 'filter:api.video-thread.create.accept.result',
183 handler: ({ accepted }, { commentBody }) => checkCommentBadWord(accepted, commentBody) 185 handler: ({ accepted }, { commentBody }) => checkCommentBadWord(accepted, commentBody)
@@ -189,6 +191,13 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
189 }) 191 })
190 192
191 registerHook({ 193 registerHook({
194 target: 'filter:activity-pub.remote-video-comment.create.accept.result',
195 handler: ({ accepted }, { comment }) => checkCommentBadWord(accepted, comment)
196 })
197
198 // ---------------------------------------------------------------------------
199
200 registerHook({
192 target: 'filter:api.video-threads.list.params', 201 target: 'filter:api.video-threads.list.params',
193 handler: obj => addToCount(obj) 202 handler: obj => addToCount(obj)
194 }) 203 })
diff --git a/server/tests/helpers/crypto.ts b/server/tests/helpers/crypto.ts
new file mode 100644
index 000000000..b508c715b
--- /dev/null
+++ b/server/tests/helpers/crypto.ts
@@ -0,0 +1,33 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { decrypt, encrypt } from '@server/helpers/peertube-crypto'
5
6describe('Encrypt/Descrypt', function () {
7
8 it('Should encrypt and decrypt the string', async function () {
9 const secret = 'my_secret'
10 const str = 'my super string'
11
12 const encrypted = await encrypt(str, secret)
13 const decrypted = await decrypt(encrypted, secret)
14
15 expect(str).to.equal(decrypted)
16 })
17
18 it('Should not decrypt without the same secret', async function () {
19 const str = 'my super string'
20
21 const encrypted = await encrypt(str, 'my_secret')
22
23 let error = false
24
25 try {
26 await decrypt(encrypted, 'my_sicret')
27 } catch (err) {
28 error = true
29 }
30
31 expect(error).to.be.true
32 })
33})
diff --git a/server/tests/helpers/index.ts b/server/tests/helpers/index.ts
index 951208842..1b5c6d15b 100644
--- a/server/tests/helpers/index.ts
+++ b/server/tests/helpers/index.ts
@@ -1,6 +1,8 @@
1import './image' 1import './comment-model'
2import './core-utils' 2import './core-utils'
3import './crypto'
3import './dns' 4import './dns'
4import './comment-model' 5import './image'
5import './markdown' 6import './markdown'
6import './request' 7import './request'
8import './validator'
diff --git a/server/tests/helpers/validator.ts b/server/tests/helpers/validator.ts
new file mode 100644
index 000000000..f40a3aaae
--- /dev/null
+++ b/server/tests/helpers/validator.ts
@@ -0,0 +1,32 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { isPluginStableOrUnstableVersionValid, isPluginStableVersionValid } from '@server/helpers/custom-validators/plugins'
5
6describe('Validators', function () {
7
8 it('Should correctly check stable plugin versions', async function () {
9 expect(isPluginStableVersionValid('3.4.0')).to.be.true
10 expect(isPluginStableVersionValid('0.4.0')).to.be.true
11 expect(isPluginStableVersionValid('0.1.0')).to.be.true
12
13 expect(isPluginStableVersionValid('0.1.0-beta-1')).to.be.false
14 expect(isPluginStableVersionValid('hello')).to.be.false
15 expect(isPluginStableVersionValid('0.x.a')).to.be.false
16 })
17
18 it('Should correctly check unstable plugin versions', async function () {
19 expect(isPluginStableOrUnstableVersionValid('3.4.0')).to.be.true
20 expect(isPluginStableOrUnstableVersionValid('0.4.0')).to.be.true
21 expect(isPluginStableOrUnstableVersionValid('0.1.0')).to.be.true
22
23 expect(isPluginStableOrUnstableVersionValid('0.1.0-beta.1')).to.be.true
24 expect(isPluginStableOrUnstableVersionValid('0.1.0-alpha.45')).to.be.true
25 expect(isPluginStableOrUnstableVersionValid('0.1.0-rc.45')).to.be.true
26
27 expect(isPluginStableOrUnstableVersionValid('hello')).to.be.false
28 expect(isPluginStableOrUnstableVersionValid('0.x.a')).to.be.false
29 expect(isPluginStableOrUnstableVersionValid('0.1.0-rc-45')).to.be.false
30 expect(isPluginStableOrUnstableVersionValid('0.1.0-rc.45d')).to.be.false
31 })
32})
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 663ac044a..d2072342e 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -1,18 +1,24 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' 4import { writeJson } from 'fs-extra'
5import { join } from 'path'
5import { HttpStatusCode, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
6import { expectLogDoesNotContain } from './shared' 8import { expectLogDoesNotContain } from './shared'
7 9
8describe('Test misc endpoints', function () { 10describe('Test misc endpoints', function () {
9 let server: PeerTubeServer 11 let server: PeerTubeServer
12 let wellKnownPath: string
10 13
11 before(async function () { 14 before(async function () {
12 this.timeout(120000) 15 this.timeout(120000)
13 16
14 server = await createSingleServer(1) 17 server = await createSingleServer(1)
18
15 await setAccessTokensToServers([ server ]) 19 await setAccessTokensToServers([ server ])
20
21 wellKnownPath = server.getDirectoryPath('well-known')
16 }) 22 })
17 23
18 describe('Test a well known endpoints', function () { 24 describe('Test a well known endpoints', function () {
@@ -93,6 +99,28 @@ describe('Test misc endpoints', function () {
93 expect(remoteInteract).to.exist 99 expect(remoteInteract).to.exist
94 expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') 100 expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}')
95 }) 101 })
102
103 it('Should return 404 for non-existing files in /.well-known', async function () {
104 await makeGetRequest({
105 url: server.url,
106 path: '/.well-known/non-existing-file',
107 expectedStatus: HttpStatusCode.NOT_FOUND_404
108 })
109 })
110
111 it('Should return custom file from /.well-known', async function () {
112 const filename = 'existing-file.json'
113
114 await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' })
115
116 const { body } = await makeGetRequest({
117 url: server.url,
118 path: '/.well-known/' + filename,
119 expectedStatus: HttpStatusCode.OK_200
120 })
121
122 expect(body.iThink).to.equal('therefore I am')
123 })
96 }) 124 })
97 125
98 describe('Test classic static endpoints', function () { 126 describe('Test classic static endpoints', function () {
diff --git a/server/tests/plugins/action-hooks.ts b/server/tests/plugins/action-hooks.ts
index af65166d9..36f8052c0 100644
--- a/server/tests/plugins/action-hooks.ts
+++ b/server/tests/plugins/action-hooks.ts
@@ -21,7 +21,7 @@ describe('Test plugin action hooks', function () {
21 } 21 }
22 22
23 before(async function () { 23 before(async function () {
24 this.timeout(30000) 24 this.timeout(120000)
25 25
26 servers = await createMultipleServers(2) 26 servers = await createMultipleServers(2)
27 await setAccessTokensToServers(servers) 27 await setAccessTokensToServers(servers)
diff --git a/server/tests/plugins/external-auth.ts b/server/tests/plugins/external-auth.ts
index e08b83791..437777e90 100644
--- a/server/tests/plugins/external-auth.ts
+++ b/server/tests/plugins/external-auth.ts
@@ -155,7 +155,7 @@ describe('Test external auth plugins', function () {
155 expect(body.username).to.equal('cyan') 155 expect(body.username).to.equal('cyan')
156 expect(body.account.displayName).to.equal('cyan') 156 expect(body.account.displayName).to.equal('cyan')
157 expect(body.email).to.equal('cyan@example.com') 157 expect(body.email).to.equal('cyan@example.com')
158 expect(body.role).to.equal(UserRole.USER) 158 expect(body.role.id).to.equal(UserRole.USER)
159 } 159 }
160 }) 160 })
161 161
@@ -177,7 +177,7 @@ describe('Test external auth plugins', function () {
177 expect(body.username).to.equal('kefka') 177 expect(body.username).to.equal('kefka')
178 expect(body.account.displayName).to.equal('Kefka Palazzo') 178 expect(body.account.displayName).to.equal('Kefka Palazzo')
179 expect(body.email).to.equal('kefka@example.com') 179 expect(body.email).to.equal('kefka@example.com')
180 expect(body.role).to.equal(UserRole.ADMINISTRATOR) 180 expect(body.role.id).to.equal(UserRole.ADMINISTRATOR)
181 } 181 }
182 }) 182 })
183 183
@@ -237,7 +237,7 @@ describe('Test external auth plugins', function () {
237 expect(body.username).to.equal('cyan') 237 expect(body.username).to.equal('cyan')
238 expect(body.account.displayName).to.equal('Cyan Garamonde') 238 expect(body.account.displayName).to.equal('Cyan Garamonde')
239 expect(body.account.description).to.equal('Retainer to the king of Doma') 239 expect(body.account.description).to.equal('Retainer to the king of Doma')
240 expect(body.role).to.equal(UserRole.USER) 240 expect(body.role.id).to.equal(UserRole.USER)
241 }) 241 })
242 242
243 it('Should not update an external auth email', async function () { 243 it('Should not update an external auth email', async function () {
diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts
index 026c7e856..083fd43ca 100644
--- a/server/tests/plugins/filter-hooks.ts
+++ b/server/tests/plugins/filter-hooks.ts
@@ -6,6 +6,7 @@ import {
6 cleanupTests, 6 cleanupTests,
7 createMultipleServers, 7 createMultipleServers,
8 doubleFollow, 8 doubleFollow,
9 makeGetRequest,
9 makeRawRequest, 10 makeRawRequest,
10 PeerTubeServer, 11 PeerTubeServer,
11 PluginsCommand, 12 PluginsCommand,
@@ -64,232 +65,289 @@ describe('Test plugin filter hooks', function () {
64 }) 65 })
65 }) 66 })
66 67
67 it('Should run filter:api.videos.list.params', async function () { 68 describe('Videos', function () {
68 const { data } = await servers[0].videos.list({ start: 0, count: 2 })
69 69
70 // 2 plugins do +1 to the count parameter 70 it('Should run filter:api.videos.list.params', async function () {
71 expect(data).to.have.lengthOf(4) 71 const { data } = await servers[0].videos.list({ start: 0, count: 2 })
72 })
73 72
74 it('Should run filter:api.videos.list.result', async function () { 73 // 2 plugins do +1 to the count parameter
75 const { total } = await servers[0].videos.list({ start: 0, count: 0 }) 74 expect(data).to.have.lengthOf(4)
75 })
76 76
77 // Plugin do +1 to the total result 77 it('Should run filter:api.videos.list.result', async function () {
78 expect(total).to.equal(11) 78 const { total } = await servers[0].videos.list({ start: 0, count: 0 })
79 })
80 79
81 it('Should run filter:api.video-playlist.videos.list.params', async function () { 80 // Plugin do +1 to the total result
82 const { data } = await servers[0].playlists.listVideos({ 81 expect(total).to.equal(11)
83 count: 2,
84 playlistId: videoPlaylistUUID
85 }) 82 })
86 83
87 // 1 plugin do +1 to the count parameter 84 it('Should run filter:api.video-playlist.videos.list.params', async function () {
88 expect(data).to.have.lengthOf(3) 85 const { data } = await servers[0].playlists.listVideos({
89 }) 86 count: 2,
87 playlistId: videoPlaylistUUID
88 })
90 89
91 it('Should run filter:api.video-playlist.videos.list.result', async function () { 90 // 1 plugin do +1 to the count parameter
92 const { total } = await servers[0].playlists.listVideos({ 91 expect(data).to.have.lengthOf(3)
93 count: 0,
94 playlistId: videoPlaylistUUID
95 }) 92 })
96 93
97 // Plugin do +1 to the total result 94 it('Should run filter:api.video-playlist.videos.list.result', async function () {
98 expect(total).to.equal(11) 95 const { total } = await servers[0].playlists.listVideos({
99 }) 96 count: 0,
97 playlistId: videoPlaylistUUID
98 })
100 99
101 it('Should run filter:api.accounts.videos.list.params', async function () { 100 // Plugin do +1 to the total result
102 const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) 101 expect(total).to.equal(11)
102 })
103 103
104 // 1 plugin do +1 to the count parameter 104 it('Should run filter:api.accounts.videos.list.params', async function () {
105 expect(data).to.have.lengthOf(3) 105 const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 })
106 })
107 106
108 it('Should run filter:api.accounts.videos.list.result', async function () { 107 // 1 plugin do +1 to the count parameter
109 const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) 108 expect(data).to.have.lengthOf(3)
109 })
110 110
111 // Plugin do +2 to the total result 111 it('Should run filter:api.accounts.videos.list.result', async function () {
112 expect(total).to.equal(12) 112 const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 })
113 })
114 113
115 it('Should run filter:api.video-channels.videos.list.params', async function () { 114 // Plugin do +2 to the total result
116 const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) 115 expect(total).to.equal(12)
116 })
117 117
118 // 1 plugin do +3 to the count parameter 118 it('Should run filter:api.video-channels.videos.list.params', async function () {
119 expect(data).to.have.lengthOf(5) 119 const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 })
120 })
121 120
122 it('Should run filter:api.video-channels.videos.list.result', async function () { 121 // 1 plugin do +3 to the count parameter
123 const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) 122 expect(data).to.have.lengthOf(5)
123 })
124 124
125 // Plugin do +3 to the total result 125 it('Should run filter:api.video-channels.videos.list.result', async function () {
126 expect(total).to.equal(13) 126 const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 })
127 })
128 127
129 it('Should run filter:api.user.me.videos.list.params', async function () { 128 // Plugin do +3 to the total result
130 const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) 129 expect(total).to.equal(13)
130 })
131 131
132 // 1 plugin do +4 to the count parameter 132 it('Should run filter:api.user.me.videos.list.params', async function () {
133 expect(data).to.have.lengthOf(6) 133 const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 })
134 })
135 134
136 it('Should run filter:api.user.me.videos.list.result', async function () { 135 // 1 plugin do +4 to the count parameter
137 const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) 136 expect(data).to.have.lengthOf(6)
137 })
138 138
139 // Plugin do +4 to the total result 139 it('Should run filter:api.user.me.videos.list.result', async function () {
140 expect(total).to.equal(14) 140 const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 })
141 })
142 141
143 it('Should run filter:api.video.get.result', async function () { 142 // Plugin do +4 to the total result
144 const video = await servers[0].videos.get({ id: videoUUID }) 143 expect(total).to.equal(14)
145 expect(video.name).to.contain('<3') 144 })
146 })
147 145
148 it('Should run filter:api.video.upload.accept.result', async function () { 146 it('Should run filter:api.video.get.result', async function () {
149 await servers[0].videos.upload({ attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 147 const video = await servers[0].videos.get({ id: videoUUID })
148 expect(video.name).to.contain('<3')
149 })
150 }) 150 })
151 151
152 it('Should run filter:api.live-video.create.accept.result', async function () { 152 describe('Video/live/import accept', function () {
153 const attributes = {
154 name: 'video with bad word',
155 privacy: VideoPrivacy.PUBLIC,
156 channelId: servers[0].store.channel.id
157 }
158 153
159 await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) 154 it('Should run filter:api.video.upload.accept.result', async function () {
160 }) 155 await servers[0].videos.upload({ attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
161 156 })
162 it('Should run filter:api.video.pre-import-url.accept.result', async function () {
163 const attributes = {
164 name: 'normal title',
165 privacy: VideoPrivacy.PUBLIC,
166 channelId: servers[0].store.channel.id,
167 targetUrl: FIXTURE_URLS.goodVideo + 'bad'
168 }
169 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
170 })
171 157
172 it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { 158 it('Should run filter:api.live-video.create.accept.result', async function () {
173 const attributes = { 159 const attributes = {
174 name: 'bad torrent', 160 name: 'video with bad word',
175 privacy: VideoPrivacy.PUBLIC, 161 privacy: VideoPrivacy.PUBLIC,
176 channelId: servers[0].store.channel.id, 162 channelId: servers[0].store.channel.id
177 torrentfile: 'video-720p.torrent' as any 163 }
178 }
179 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
180 })
181 164
182 it('Should run filter:api.video.post-import-url.accept.result', async function () { 165 await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
183 this.timeout(60000) 166 })
184 167
185 let videoImportId: number 168 it('Should run filter:api.video.pre-import-url.accept.result', async function () {
169 const attributes = {
170 name: 'normal title',
171 privacy: VideoPrivacy.PUBLIC,
172 channelId: servers[0].store.channel.id,
173 targetUrl: FIXTURE_URLS.goodVideo + 'bad'
174 }
175 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
176 })
186 177
187 { 178 it('Should run filter:api.video.pre-import-torrent.accept.result', async function () {
188 const attributes = { 179 const attributes = {
189 name: 'title with bad word', 180 name: 'bad torrent',
190 privacy: VideoPrivacy.PUBLIC, 181 privacy: VideoPrivacy.PUBLIC,
191 channelId: servers[0].store.channel.id, 182 channelId: servers[0].store.channel.id,
192 targetUrl: FIXTURE_URLS.goodVideo 183 torrentfile: 'video-720p.torrent' as any
193 } 184 }
194 const body = await servers[0].imports.importVideo({ attributes }) 185 await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
195 videoImportId = body.id 186 })
196 }
197 187
198 await waitJobs(servers) 188 it('Should run filter:api.video.post-import-url.accept.result', async function () {
189 this.timeout(60000)
199 190
200 { 191 let videoImportId: number
201 const body = await servers[0].imports.getMyVideoImports()
202 const videoImports = body.data
203 192
204 const videoImport = videoImports.find(i => i.id === videoImportId) 193 {
194 const attributes = {
195 name: 'title with bad word',
196 privacy: VideoPrivacy.PUBLIC,
197 channelId: servers[0].store.channel.id,
198 targetUrl: FIXTURE_URLS.goodVideo
199 }
200 const body = await servers[0].imports.importVideo({ attributes })
201 videoImportId = body.id
202 }
205 203
206 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) 204 await waitJobs(servers)
207 expect(videoImport.state.label).to.equal('Rejected')
208 }
209 })
210 205
211 it('Should run filter:api.video.post-import-torrent.accept.result', async function () { 206 {
212 this.timeout(60000) 207 const body = await servers[0].imports.getMyVideoImports()
208 const videoImports = body.data
213 209
214 let videoImportId: number 210 const videoImport = videoImports.find(i => i.id === videoImportId)
215 211
216 { 212 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
217 const attributes = { 213 expect(videoImport.state.label).to.equal('Rejected')
218 name: 'title with bad word',
219 privacy: VideoPrivacy.PUBLIC,
220 channelId: servers[0].store.channel.id,
221 torrentfile: 'video-720p.torrent' as any
222 } 214 }
223 const body = await servers[0].imports.importVideo({ attributes }) 215 })
224 videoImportId = body.id 216
225 } 217 it('Should run filter:api.video.post-import-torrent.accept.result', async function () {
218 this.timeout(60000)
226 219
227 await waitJobs(servers) 220 let videoImportId: number
228 221
229 { 222 {
230 const { data: videoImports } = await servers[0].imports.getMyVideoImports() 223 const attributes = {
224 name: 'title with bad word',
225 privacy: VideoPrivacy.PUBLIC,
226 channelId: servers[0].store.channel.id,
227 torrentfile: 'video-720p.torrent' as any
228 }
229 const body = await servers[0].imports.importVideo({ attributes })
230 videoImportId = body.id
231 }
231 232
232 const videoImport = videoImports.find(i => i.id === videoImportId) 233 await waitJobs(servers)
233 234
234 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) 235 {
235 expect(videoImport.state.label).to.equal('Rejected') 236 const { data: videoImports } = await servers[0].imports.getMyVideoImports()
236 } 237
237 }) 238 const videoImport = videoImports.find(i => i.id === videoImportId)
238 239
239 it('Should run filter:api.video-thread.create.accept.result', async function () { 240 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
240 await servers[0].comments.createThread({ 241 expect(videoImport.state.label).to.equal('Rejected')
241 videoId: videoUUID, 242 }
242 text: 'comment with bad word',
243 expectedStatus: HttpStatusCode.FORBIDDEN_403
244 }) 243 })
245 }) 244 })
246 245
247 it('Should run filter:api.video-comment-reply.create.accept.result', async function () { 246 describe('Video comments accept', function () {
248 const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' })
249 threadId = created.id
250 247
251 await servers[0].comments.addReply({ 248 it('Should run filter:api.video-thread.create.accept.result', async function () {
252 videoId: videoUUID, 249 await servers[0].comments.createThread({
253 toCommentId: threadId, 250 videoId: videoUUID,
254 text: 'comment with bad word', 251 text: 'comment with bad word',
255 expectedStatus: HttpStatusCode.FORBIDDEN_403 252 expectedStatus: HttpStatusCode.FORBIDDEN_403
253 })
256 }) 254 })
257 await servers[0].comments.addReply({ 255
258 videoId: videoUUID, 256 it('Should run filter:api.video-comment-reply.create.accept.result', async function () {
259 toCommentId: threadId, 257 const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' })
260 text: 'comment with good word', 258 threadId = created.id
261 expectedStatus: HttpStatusCode.OK_200 259
260 await servers[0].comments.addReply({
261 videoId: videoUUID,
262 toCommentId: threadId,
263 text: 'comment with bad word',
264 expectedStatus: HttpStatusCode.FORBIDDEN_403
265 })
266 await servers[0].comments.addReply({
267 videoId: videoUUID,
268 toCommentId: threadId,
269 text: 'comment with good word',
270 expectedStatus: HttpStatusCode.OK_200
271 })
262 }) 272 })
263 })
264 273
265 it('Should run filter:api.video-threads.list.params', async function () { 274 it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a thread creation', async function () {
266 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) 275 this.timeout(30000)
267 276
268 // our plugin do +1 to the count parameter 277 await servers[1].comments.createThread({ videoId: videoUUID, text: 'comment with bad word' })
269 expect(data).to.have.lengthOf(1)
270 })
271 278
272 it('Should run filter:api.video-threads.list.result', async function () { 279 await waitJobs(servers)
273 const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 })
274 280
275 // Plugin do +1 to the total result 281 {
276 expect(total).to.equal(2) 282 const thread = await servers[0].comments.listThreads({ videoId: videoUUID })
277 }) 283 expect(thread.data).to.have.lengthOf(1)
284 expect(thread.data[0].text).to.not.include(' bad ')
285 }
278 286
279 it('Should run filter:api.video-thread-comments.list.params') 287 {
288 const thread = await servers[1].comments.listThreads({ videoId: videoUUID })
289 expect(thread.data).to.have.lengthOf(2)
290 }
291 })
292
293 it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a reply creation', async function () {
294 this.timeout(30000)
280 295
281 it('Should run filter:api.video-thread-comments.list.result', async function () { 296 const { data } = await servers[1].comments.listThreads({ videoId: videoUUID })
282 const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) 297 const threadIdServer2 = data.find(t => t.text === 'thread').id
283 298
284 expect(thread.comment.text.endsWith(' <3')).to.be.true 299 await servers[1].comments.addReply({
300 videoId: videoUUID,
301 toCommentId: threadIdServer2,
302 text: 'comment with bad word'
303 })
304
305 await waitJobs(servers)
306
307 {
308 const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId })
309 expect(tree.children).to.have.lengthOf(1)
310 expect(tree.children[0].comment.text).to.not.include(' bad ')
311 }
312
313 {
314 const tree = await servers[1].comments.getThread({ videoId: videoUUID, threadId: threadIdServer2 })
315 expect(tree.children).to.have.lengthOf(2)
316 }
317 })
285 }) 318 })
286 319
287 it('Should run filter:api.overviews.videos.list.{params,result}', async function () { 320 describe('Video comments', function () {
288 await servers[0].overviews.getVideos({ page: 1 })
289 321
290 // 3 because we get 3 samples per page 322 it('Should run filter:api.video-threads.list.params', async function () {
291 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3) 323 const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 })
292 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3) 324
325 // our plugin do +1 to the count parameter
326 expect(data).to.have.lengthOf(1)
327 })
328
329 it('Should run filter:api.video-threads.list.result', async function () {
330 const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 })
331
332 // Plugin do +1 to the total result
333 expect(total).to.equal(2)
334 })
335
336 it('Should run filter:api.video-thread-comments.list.params')
337
338 it('Should run filter:api.video-thread-comments.list.result', async function () {
339 const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId })
340
341 expect(thread.comment.text.endsWith(' <3')).to.be.true
342 })
343
344 it('Should run filter:api.overviews.videos.list.{params,result}', async function () {
345 await servers[0].overviews.getVideos({ page: 1 })
346
347 // 3 because we get 3 samples per page
348 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3)
349 await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3)
350 })
293 }) 351 })
294 352
295 describe('filter:video.auto-blacklist.result', function () { 353 describe('filter:video.auto-blacklist.result', function () {
@@ -404,30 +462,41 @@ describe('Test plugin filter hooks', function () {
404 }) 462 })
405 463
406 it('Should run filter:api.download.torrent.allowed.result', async function () { 464 it('Should run filter:api.download.torrent.allowed.result', async function () {
407 const res = await makeRawRequest(downloadVideos[0].files[0].torrentDownloadUrl, 403) 465 const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
408 expect(res.body.error).to.equal('Liu Bei') 466 expect(res.body.error).to.equal('Liu Bei')
409 467
410 await makeRawRequest(downloadVideos[1].files[0].torrentDownloadUrl, 200) 468 await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
411 await makeRawRequest(downloadVideos[2].files[0].torrentDownloadUrl, 200) 469 await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
412 }) 470 })
413 471
414 it('Should run filter:api.download.video.allowed.result', async function () { 472 it('Should run filter:api.download.video.allowed.result', async function () {
415 { 473 {
416 const res = await makeRawRequest(downloadVideos[1].files[0].fileDownloadUrl, 403) 474 const res = await makeRawRequest({ url: downloadVideos[1].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
417 expect(res.body.error).to.equal('Cao Cao') 475 expect(res.body.error).to.equal('Cao Cao')
418 476
419 await makeRawRequest(downloadVideos[0].files[0].fileDownloadUrl, 200) 477 await makeRawRequest({ url: downloadVideos[0].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
420 await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200) 478 await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
421 } 479 }
422 480
423 { 481 {
424 const res = await makeRawRequest(downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl, 403) 482 const res = await makeRawRequest({
483 url: downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl,
484 expectedStatus: HttpStatusCode.FORBIDDEN_403
485 })
486
425 expect(res.body.error).to.equal('Sun Jian') 487 expect(res.body.error).to.equal('Sun Jian')
426 488
427 await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200) 489 await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
490
491 await makeRawRequest({
492 url: downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl,
493 expectedStatus: HttpStatusCode.OK_200
494 })
428 495
429 await makeRawRequest(downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, 200) 496 await makeRawRequest({
430 await makeRawRequest(downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl, 200) 497 url: downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl,
498 expectedStatus: HttpStatusCode.OK_200
499 })
431 } 500 }
432 }) 501 })
433 }) 502 })
@@ -458,12 +527,12 @@ describe('Test plugin filter hooks', function () {
458 }) 527 })
459 528
460 it('Should run filter:html.embed.video.allowed.result', async function () { 529 it('Should run filter:html.embed.video.allowed.result', async function () {
461 const res = await makeRawRequest(servers[0].url + embedVideos[0].embedPath, 200) 530 const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
462 expect(res.text).to.equal('Lu Bu') 531 expect(res.text).to.equal('Lu Bu')
463 }) 532 })
464 533
465 it('Should run filter:html.embed.video-playlist.allowed.result', async function () { 534 it('Should run filter:html.embed.video-playlist.allowed.result', async function () {
466 const res = await makeRawRequest(servers[0].url + embedPlaylists[0].embedPath, 200) 535 const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
467 expect(res.text).to.equal('Diao Chan') 536 expect(res.text).to.equal('Diao Chan')
468 }) 537 })
469 }) 538 })
diff --git a/server/tests/plugins/id-and-pass-auth.ts b/server/tests/plugins/id-and-pass-auth.ts
index 85faac5a8..fc24a5656 100644
--- a/server/tests/plugins/id-and-pass-auth.ts
+++ b/server/tests/plugins/id-and-pass-auth.ts
@@ -48,7 +48,7 @@ describe('Test id and pass auth plugins', function () {
48 48
49 expect(body.username).to.equal('spyro') 49 expect(body.username).to.equal('spyro')
50 expect(body.account.displayName).to.equal('Spyro the Dragon') 50 expect(body.account.displayName).to.equal('Spyro the Dragon')
51 expect(body.role).to.equal(UserRole.USER) 51 expect(body.role.id).to.equal(UserRole.USER)
52 }) 52 })
53 53
54 it('Should login Crash, create the user and use the token', async function () { 54 it('Should login Crash, create the user and use the token', async function () {
@@ -63,7 +63,7 @@ describe('Test id and pass auth plugins', function () {
63 63
64 expect(body.username).to.equal('crash') 64 expect(body.username).to.equal('crash')
65 expect(body.account.displayName).to.equal('Crash Bandicoot') 65 expect(body.account.displayName).to.equal('Crash Bandicoot')
66 expect(body.role).to.equal(UserRole.MODERATOR) 66 expect(body.role.id).to.equal(UserRole.MODERATOR)
67 } 67 }
68 }) 68 })
69 69
@@ -79,7 +79,7 @@ describe('Test id and pass auth plugins', function () {
79 79
80 expect(body.username).to.equal('laguna') 80 expect(body.username).to.equal('laguna')
81 expect(body.account.displayName).to.equal('laguna') 81 expect(body.account.displayName).to.equal('laguna')
82 expect(body.role).to.equal(UserRole.USER) 82 expect(body.role.id).to.equal(UserRole.USER)
83 } 83 }
84 }) 84 })
85 85
@@ -129,7 +129,7 @@ describe('Test id and pass auth plugins', function () {
129 expect(body.username).to.equal('crash') 129 expect(body.username).to.equal('crash')
130 expect(body.account.displayName).to.equal('Beautiful Crash') 130 expect(body.account.displayName).to.equal('Beautiful Crash')
131 expect(body.account.description).to.equal('Mutant eastern barred bandicoot') 131 expect(body.account.description).to.equal('Mutant eastern barred bandicoot')
132 expect(body.role).to.equal(UserRole.MODERATOR) 132 expect(body.role.id).to.equal(UserRole.MODERATOR)
133 }) 133 })
134 134
135 it('Should reject token of laguna by the plugin hook', async function () { 135 it('Should reject token of laguna by the plugin hook', async function () {
diff --git a/server/tests/plugins/index.ts b/server/tests/plugins/index.ts
index 4534120fd..210af7236 100644
--- a/server/tests/plugins/index.ts
+++ b/server/tests/plugins/index.ts
@@ -8,5 +8,6 @@ import './plugin-router'
8import './plugin-storage' 8import './plugin-storage'
9import './plugin-transcoding' 9import './plugin-transcoding'
10import './plugin-unloading' 10import './plugin-unloading'
11import './plugin-websocket'
11import './translations' 12import './translations'
12import './video-constants' 13import './video-constants'
diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts
index 955d7ddfd..f2bada4ee 100644
--- a/server/tests/plugins/plugin-helpers.ts
+++ b/server/tests/plugins/plugin-helpers.ts
@@ -83,6 +83,33 @@ describe('Test plugin helpers', function () {
83 }) 83 })
84 }) 84 })
85 85
86 describe('Socket', function () {
87
88 it('Should sendNotification without any exceptions', async () => {
89 const user = await servers[0].users.create({ username: 'notis_redding', password: 'secret1234?' })
90 await makePostBodyRequest({
91 url: servers[0].url,
92 path: '/plugins/test-four/router/send-notification',
93 fields: {
94 userId: user.id
95 },
96 expectedStatus: HttpStatusCode.CREATED_201
97 })
98 })
99
100 it('Should sendVideoLiveNewState without any exceptions', async () => {
101 const res = await servers[0].videos.quickUpload({ name: 'video server 1' })
102
103 await makePostBodyRequest({
104 url: servers[0].url,
105 path: '/plugins/test-four/router/send-video-live-new-state/' + res.uuid,
106 expectedStatus: HttpStatusCode.CREATED_201
107 })
108
109 await servers[0].videos.remove({ id: res.uuid })
110 })
111 })
112
86 describe('Plugin', function () { 113 describe('Plugin', function () {
87 114
88 it('Should get the base static route', async function () { 115 it('Should get the base static route', async function () {
@@ -280,7 +307,7 @@ describe('Test plugin helpers', function () {
280 expect(file.fps).to.equal(25) 307 expect(file.fps).to.equal(25)
281 308
282 expect(await pathExists(file.path)).to.be.true 309 expect(await pathExists(file.path)).to.be.true
283 await makeRawRequest(file.url, HttpStatusCode.OK_200) 310 await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 })
284 } 311 }
285 } 312 }
286 313
@@ -294,12 +321,12 @@ describe('Test plugin helpers', function () {
294 const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE) 321 const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE)
295 expect(miniature).to.exist 322 expect(miniature).to.exist
296 expect(await pathExists(miniature.path)).to.be.true 323 expect(await pathExists(miniature.path)).to.be.true
297 await makeRawRequest(miniature.url, HttpStatusCode.OK_200) 324 await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 })
298 325
299 const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW) 326 const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW)
300 expect(preview).to.exist 327 expect(preview).to.exist
301 expect(await pathExists(preview.path)).to.be.true 328 expect(await pathExists(preview.path)).to.be.true
302 await makeRawRequest(preview.url, HttpStatusCode.OK_200) 329 await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 })
303 } 330 }
304 }) 331 })
305 332
diff --git a/server/tests/plugins/plugin-websocket.ts b/server/tests/plugins/plugin-websocket.ts
new file mode 100644
index 000000000..adaa28b1d
--- /dev/null
+++ b/server/tests/plugins/plugin-websocket.ts
@@ -0,0 +1,70 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import WebSocket from 'ws'
4import { cleanupTests, createSingleServer, PeerTubeServer, PluginsCommand, setAccessTokensToServers } from '@shared/server-commands'
5
6function buildWebSocket (server: PeerTubeServer, path: string) {
7 return new WebSocket('ws://' + server.host + path)
8}
9
10function expectErrorOrTimeout (server: PeerTubeServer, path: string, expectedTimeout: number) {
11 return new Promise<void>((res, rej) => {
12 const ws = buildWebSocket(server, path)
13 ws.on('error', () => res())
14
15 const timeout = setTimeout(() => res(), expectedTimeout)
16
17 ws.on('open', () => {
18 clearTimeout(timeout)
19
20 return rej(new Error('Connect did not timeout'))
21 })
22 })
23}
24
25describe('Test plugin websocket', function () {
26 let server: PeerTubeServer
27 const basePaths = [
28 '/plugins/test-websocket/ws/',
29 '/plugins/test-websocket/0.0.1/ws/'
30 ]
31
32 before(async function () {
33 this.timeout(30000)
34
35 server = await createSingleServer(1)
36 await setAccessTokensToServers([ server ])
37
38 await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-websocket') })
39 })
40
41 it('Should not connect to the websocket without the appropriate path', async function () {
42 const paths = [
43 '/plugins/unknown/ws/',
44 '/plugins/unknown/0.0.1/ws/'
45 ]
46
47 for (const path of paths) {
48 await expectErrorOrTimeout(server, path, 1000)
49 }
50 })
51
52 it('Should not connect to the websocket without the appropriate sub path', async function () {
53 for (const path of basePaths) {
54 await expectErrorOrTimeout(server, path + '/unknown', 1000)
55 }
56 })
57
58 it('Should connect to the websocket and receive pong', function (done) {
59 const ws = buildWebSocket(server, basePaths[0])
60
61 ws.on('open', () => ws.send('ping'))
62 ws.on('message', data => {
63 if (data.toString() === 'pong') return done()
64 })
65 })
66
67 after(async function () {
68 await cleanupTests([ server ])
69 })
70})
diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts
index f8f4a5137..41fd72e89 100644
--- a/server/tests/shared/actors.ts
+++ b/server/tests/shared/actors.ts
@@ -2,8 +2,6 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir } from 'fs-extra' 4import { pathExists, readdir } from 'fs-extra'
5import { join } from 'path'
6import { root } from '@shared/core-utils'
7import { Account, VideoChannel } from '@shared/models' 5import { Account, VideoChannel } from '@shared/models'
8import { PeerTubeServer } from '@shared/server-commands' 6import { PeerTubeServer } from '@shared/server-commands'
9 7
@@ -31,11 +29,9 @@ async function expectAccountFollows (options: {
31 return expectActorFollow({ ...options, data }) 29 return expectActorFollow({ ...options, data })
32} 30}
33 31
34async function checkActorFilesWereRemoved (filename: string, serverNumber: number) { 32async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) {
35 const testDirectory = 'test' + serverNumber
36
37 for (const directory of [ 'avatars' ]) { 33 for (const directory of [ 'avatars' ]) {
38 const directoryPath = join(root(), testDirectory, directory) 34 const directoryPath = server.getDirectoryPath(directory)
39 35
40 const directoryExists = await pathExists(directoryPath) 36 const directoryExists = await pathExists(directoryPath)
41 expect(directoryExists).to.be.true 37 expect(directoryExists).to.be.true
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index c7065a767..90d534a06 100644
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -2,22 +2,18 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir } from 'fs-extra' 4import { pathExists, readdir } from 'fs-extra'
5import { join } from 'path'
6import { root } from '@shared/core-utils'
7import { PeerTubeServer } from '@shared/server-commands' 5import { PeerTubeServer } from '@shared/server-commands'
8 6
9async function checkTmpIsEmpty (server: PeerTubeServer) { 7async function checkTmpIsEmpty (server: PeerTubeServer) {
10 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) 8 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
11 9
12 if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { 10 if (await pathExists(server.getDirectoryPath('tmp/hls'))) {
13 await checkDirectoryIsEmpty(server, 'tmp/hls') 11 await checkDirectoryIsEmpty(server, 'tmp/hls')
14 } 12 }
15} 13}
16 14
17async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { 15async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
18 const testDirectory = 'test' + server.internalServerNumber 16 const directoryPath = server.getDirectoryPath(directory)
19
20 const directoryPath = join(root(), testDirectory, directory)
21 17
22 const directoryExists = await pathExists(directoryPath) 18 const directoryExists = await pathExists(directoryPath)
23 expect(directoryExists).to.be.true 19 expect(directoryExists).to.be.true
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts
index 4bd4786fc..47e0dc481 100644
--- a/server/tests/shared/live.ts
+++ b/server/tests/shared/live.ts
@@ -3,39 +3,118 @@
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir } from 'fs-extra' 4import { pathExists, readdir } from 'fs-extra'
5import { join } from 'path' 5import { join } from 'path'
6import { LiveVideo } from '@shared/models' 6import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models'
7import { PeerTubeServer } from '@shared/server-commands' 7import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands'
8 8import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists'
9async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) { 9
10 let live: LiveVideo 10async function checkLiveCleanup (options: {
11 11 server: PeerTubeServer
12 try { 12 videoUUID: string
13 live = await server.live.get({ videoId: videoUUID }) 13 permanent: boolean
14 } catch {} 14 savedResolutions?: number[]
15}) {
16 const { server, videoUUID, permanent, savedResolutions = [] } = options
15 17
16 const basePath = server.servers.buildDirectory('streaming-playlists') 18 const basePath = server.servers.buildDirectory('streaming-playlists')
17 const hlsPath = join(basePath, 'hls', videoUUID) 19 const hlsPath = join(basePath, 'hls', videoUUID)
18 20
19 if (savedResolutions.length === 0) { 21 if (permanent) {
22 if (!await pathExists(hlsPath)) return
20 23
21 if (live?.permanentLive) { 24 const files = await readdir(hlsPath)
22 expect(await pathExists(hlsPath)).to.be.true 25 expect(files).to.have.lengthOf(0)
26 return
27 }
23 28
24 const hlsFiles = await readdir(hlsPath) 29 if (savedResolutions.length === 0) {
25 expect(hlsFiles).to.have.lengthOf(1) // Only replays directory 30 return checkUnsavedLiveCleanup(server, videoUUID, hlsPath)
31 }
26 32
27 const replayDir = join(hlsPath, 'replay') 33 return checkSavedLiveCleanup(hlsPath, savedResolutions)
28 expect(await pathExists(replayDir)).to.be.true 34}
29 35
30 const replayFiles = await readdir(join(hlsPath, 'replay')) 36// ---------------------------------------------------------------------------
31 expect(replayFiles).to.have.lengthOf(0) 37
32 } else { 38async function testVideoResolutions (options: {
33 expect(await pathExists(hlsPath)).to.be.false 39 originServer: PeerTubeServer
40 servers: PeerTubeServer[]
41 liveVideoId: string
42 resolutions: number[]
43 transcoded: boolean
44 objectStorage: boolean
45}) {
46 const { originServer, servers, liveVideoId, resolutions, transcoded, objectStorage } = options
47
48 for (const server of servers) {
49 const { data } = await server.videos.list()
50 expect(data.find(v => v.uuid === liveVideoId)).to.exist
51
52 const video = await server.videos.get({ id: liveVideoId })
53 expect(video.streamingPlaylists).to.have.lengthOf(1)
54
55 const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
56 expect(hlsPlaylist).to.exist
57 expect(hlsPlaylist.files).to.have.lengthOf(0) // Only fragmented mp4 files are displayed
58
59 await checkResolutionsInMasterPlaylist({
60 server,
61 playlistUrl: hlsPlaylist.playlistUrl,
62 resolutions,
63 transcoded,
64 withRetry: objectStorage
65 })
66
67 if (objectStorage) {
68 expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl())
34 } 69 }
35 70
36 return 71 for (let i = 0; i < resolutions.length; i++) {
72 const segmentNum = 3
73 const segmentName = `${i}-00000${segmentNum}.ts`
74 await originServer.live.waitUntilSegmentGeneration({
75 server: originServer,
76 videoUUID: video.uuid,
77 playlistNumber: i,
78 segment: segmentNum,
79 objectStorage
80 })
81
82 const baseUrl = objectStorage
83 ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls'
84 : originServer.url + '/static/streaming-playlists/hls'
85
86 if (objectStorage) {
87 expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl())
88 }
89
90 const subPlaylist = await originServer.streamingPlaylists.get({
91 url: `${baseUrl}/${video.uuid}/${i}.m3u8`,
92 withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
93 })
94
95 expect(subPlaylist).to.contain(segmentName)
96
97 await checkLiveSegmentHash({
98 server,
99 baseUrlSegment: baseUrl,
100 videoUUID: video.uuid,
101 segmentName,
102 hlsPlaylist
103 })
104 }
37 } 105 }
106}
107
108// ---------------------------------------------------------------------------
109
110export {
111 checkLiveCleanup,
112 testVideoResolutions
113}
114
115// ---------------------------------------------------------------------------
38 116
117async function checkSavedLiveCleanup (hlsPath: string, savedResolutions: number[] = []) {
39 const files = await readdir(hlsPath) 118 const files = await readdir(hlsPath)
40 119
41 // fragmented file and playlist per resolution + master playlist + segments sha256 json file 120 // fragmented file and playlist per resolution + master playlist + segments sha256 json file
@@ -56,6 +135,27 @@ async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, save
56 expect(shaFile).to.exist 135 expect(shaFile).to.exist
57} 136}
58 137
59export { 138async function checkUnsavedLiveCleanup (server: PeerTubeServer, videoUUID: string, hlsPath: string) {
60 checkLiveCleanup 139 let live: LiveVideo
140
141 try {
142 live = await server.live.get({ videoId: videoUUID })
143 } catch {}
144
145 if (live?.permanentLive) {
146 expect(await pathExists(hlsPath)).to.be.true
147
148 const hlsFiles = await readdir(hlsPath)
149 expect(hlsFiles).to.have.lengthOf(1) // Only replays directory
150
151 const replayDir = join(hlsPath, 'replay')
152 expect(await pathExists(replayDir)).to.be.true
153
154 const replayFiles = await readdir(join(hlsPath, 'replay'))
155 expect(replayFiles).to.have.lengthOf(0)
156
157 return
158 }
159
160 expect(await pathExists(hlsPath)).to.be.false
61} 161}
diff --git a/server/tests/shared/mock-servers/mock-object-storage.ts b/server/tests/shared/mock-servers/mock-object-storage.ts
index 99d68e014..8c325bf11 100644
--- a/server/tests/shared/mock-servers/mock-object-storage.ts
+++ b/server/tests/shared/mock-servers/mock-object-storage.ts
@@ -12,7 +12,7 @@ export class MockObjectStorage {
12 const app = express() 12 const app = express()
13 13
14 app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { 14 app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {
15 const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` 15 const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}`
16 16
17 if (process.env.DEBUG) { 17 if (process.env.DEBUG) {
18 console.log('Receiving request on mocked server %s.', req.url) 18 console.log('Receiving request on mocked server %s.', req.url)
diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts
index fdd541d20..8db303fd8 100644
--- a/server/tests/shared/playlists.ts
+++ b/server/tests/shared/playlists.ts
@@ -1,17 +1,14 @@
1import { expect } from 'chai' 1import { expect } from 'chai'
2import { readdir } from 'fs-extra' 2import { readdir } from 'fs-extra'
3import { join } from 'path' 3import { PeerTubeServer } from '@shared/server-commands'
4import { root } from '@shared/core-utils'
5 4
6async function checkPlaylistFilesWereRemoved ( 5async function checkPlaylistFilesWereRemoved (
7 playlistUUID: string, 6 playlistUUID: string,
8 internalServerNumber: number, 7 server: PeerTubeServer,
9 directories = [ 'thumbnails' ] 8 directories = [ 'thumbnails' ]
10) { 9) {
11 const testDirectory = 'test' + internalServerNumber
12
13 for (const directory of directories) { 10 for (const directory of directories) {
14 const directoryPath = join(root(), testDirectory, directory) 11 const directoryPath = server.getDirectoryPath(directory)
15 12
16 const files = await readdir(directoryPath) 13 const files = await readdir(directoryPath)
17 for (const file of files) { 14 for (const file of files) {
diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts
index 4d82b3654..824c3dcef 100644
--- a/server/tests/shared/streaming-playlists.ts
+++ b/server/tests/shared/streaming-playlists.ts
@@ -1,9 +1,13 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
1import { expect } from 'chai' 3import { expect } from 'chai'
2import { basename } from 'path' 4import { basename } from 'path'
3import { removeFragmentedMP4Ext } from '@shared/core-utils' 5import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
4import { sha256 } from '@shared/extra-utils' 6import { sha256 } from '@shared/extra-utils'
5import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models' 7import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
6import { PeerTubeServer } from '@shared/server-commands' 8import { makeRawRequest, PeerTubeServer, webtorrentAdd } from '@shared/server-commands'
9import { expectStartWith } from './checks'
10import { hlsInfohashExist } from './tracker'
7 11
8async function checkSegmentHash (options: { 12async function checkSegmentHash (options: {
9 server: PeerTubeServer 13 server: PeerTubeServer
@@ -26,7 +30,7 @@ async function checkSegmentHash (options: {
26 const offset = parseInt(matches[2], 10) 30 const offset = parseInt(matches[2], 10)
27 const range = `${offset}-${offset + length - 1}` 31 const range = `${offset}-${offset + length - 1}`
28 32
29 const segmentBody = await command.getSegment({ 33 const segmentBody = await command.getFragmentedSegment({
30 url: `${baseUrlSegment}/${videoName}`, 34 url: `${baseUrlSegment}/${videoName}`,
31 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, 35 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206,
32 range: `bytes=${range}` 36 range: `bytes=${range}`
@@ -46,7 +50,7 @@ async function checkLiveSegmentHash (options: {
46 const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options 50 const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options
47 const command = server.streamingPlaylists 51 const command = server.streamingPlaylists
48 52
49 const segmentBody = await command.getSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) 53 const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` })
50 const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) 54 const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
51 55
52 expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) 56 expect(sha256(segmentBody)).to.equal(shaBody[segmentName])
@@ -56,15 +60,17 @@ async function checkResolutionsInMasterPlaylist (options: {
56 server: PeerTubeServer 60 server: PeerTubeServer
57 playlistUrl: string 61 playlistUrl: string
58 resolutions: number[] 62 resolutions: number[]
63 transcoded?: boolean // default true
64 withRetry?: boolean // default false
59}) { 65}) {
60 const { server, playlistUrl, resolutions } = options 66 const { server, playlistUrl, resolutions, withRetry = false, transcoded = true } = options
61 67
62 const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl }) 68 const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl, withRetry })
63 69
64 for (const resolution of resolutions) { 70 for (const resolution of resolutions) {
65 const reg = new RegExp( 71 const reg = transcoded
66 '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"' 72 ? new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"')
67 ) 73 : new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + '')
68 74
69 expect(masterPlaylist).to.match(reg) 75 expect(masterPlaylist).to.match(reg)
70 } 76 }
@@ -73,8 +79,118 @@ async function checkResolutionsInMasterPlaylist (options: {
73 expect(playlistsLength).to.have.lengthOf(resolutions.length) 79 expect(playlistsLength).to.have.lengthOf(resolutions.length)
74} 80}
75 81
82async function completeCheckHlsPlaylist (options: {
83 servers: PeerTubeServer[]
84 videoUUID: string
85 hlsOnly: boolean
86
87 resolutions?: number[]
88 objectStorageBaseUrl: string
89}) {
90 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
91
92 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
93
94 for (const server of options.servers) {
95 const videoDetails = await server.videos.get({ id: videoUUID })
96 const baseUrl = `http://${videoDetails.account.host}`
97
98 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
99
100 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
101 expect(hlsPlaylist).to.not.be.undefined
102
103 const hlsFiles = hlsPlaylist.files
104 expect(hlsFiles).to.have.lengthOf(resolutions.length)
105
106 if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
107 else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
108
109 // Check JSON files
110 for (const resolution of resolutions) {
111 const file = hlsFiles.find(f => f.resolution.id === resolution)
112 expect(file).to.not.be.undefined
113
114 expect(file.magnetUri).to.have.lengthOf.above(2)
115 expect(file.torrentUrl).to.match(
116 new RegExp(`${server.url}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
117 )
118
119 if (objectStorageBaseUrl) {
120 expectStartWith(file.fileUrl, objectStorageBaseUrl)
121 } else {
122 expect(file.fileUrl).to.match(
123 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
124 )
125 }
126
127 expect(file.resolution.label).to.equal(resolution + 'p')
128
129 await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
130 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
131
132 const torrent = await webtorrentAdd(file.magnetUri, true)
133 expect(torrent.files).to.be.an('array')
134 expect(torrent.files.length).to.equal(1)
135 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
136 }
137
138 // Check master playlist
139 {
140 await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
141
142 const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
143
144 let i = 0
145 for (const resolution of resolutions) {
146 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
147 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
148
149 const url = 'http://' + videoDetails.account.host
150 await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
151
152 i++
153 }
154 }
155
156 // Check resolution playlists
157 {
158 for (const resolution of resolutions) {
159 const file = hlsFiles.find(f => f.resolution.id === resolution)
160 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
161
162 const url = objectStorageBaseUrl
163 ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
164 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
165
166 const subPlaylist = await server.streamingPlaylists.get({ url })
167
168 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
169 expect(subPlaylist).to.contain(basename(file.fileUrl))
170 }
171 }
172
173 {
174 const baseUrlAndPath = objectStorageBaseUrl
175 ? objectStorageBaseUrl + 'hls/' + videoUUID
176 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
177
178 for (const resolution of resolutions) {
179 await checkSegmentHash({
180 server,
181 baseUrlPlaylist: baseUrlAndPath,
182 baseUrlSegment: baseUrlAndPath,
183 resolution,
184 hlsPlaylist
185 })
186 }
187 }
188 }
189}
190
76export { 191export {
77 checkSegmentHash, 192 checkSegmentHash,
78 checkLiveSegmentHash, 193 checkLiveSegmentHash,
79 checkResolutionsInMasterPlaylist 194 checkResolutionsInMasterPlaylist,
195 completeCheckHlsPlaylist
80} 196}
diff --git a/server/tests/shared/videos.ts b/server/tests/shared/videos.ts
index e18329e07..c8339584b 100644
--- a/server/tests/shared/videos.ts
+++ b/server/tests/shared/videos.ts
@@ -125,9 +125,9 @@ async function completeVideoCheck (
125 expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) 125 expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`))
126 126
127 await Promise.all([ 127 await Promise.all([
128 makeRawRequest(file.torrentUrl, 200), 128 makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }),
129 makeRawRequest(file.torrentDownloadUrl, 200), 129 makeRawRequest({ url: file.torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }),
130 makeRawRequest(file.metadataUrl, 200) 130 makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.OK_200 })
131 ]) 131 ])
132 132
133 expect(file.resolution.id).to.equal(attributeFile.resolution) 133 expect(file.resolution.id).to.equal(attributeFile.resolution)
diff --git a/server/tools/cli.ts b/server/tools/cli.ts
index a15d73fb4..4607d052a 100644
--- a/server/tools/cli.ts
+++ b/server/tools/cli.ts
@@ -20,7 +20,7 @@ async function getAdminTokenOrDie (server: PeerTubeServer, username: string, pas
20 const token = await server.login.getAccessToken(username, password) 20 const token = await server.login.getAccessToken(username, password)
21 const me = await server.users.getMyInfo({ token }) 21 const me = await server.users.getMyInfo({ token })
22 22
23 if (me.role !== UserRole.ADMINISTRATOR) { 23 if (me.role.id !== UserRole.ADMINISTRATOR) {
24 console.error('You must be an administrator.') 24 console.error('You must be an administrator.')
25 process.exit(-1) 25 process.exit(-1)
26 } 26 }
diff --git a/server/tools/peertube-plugins.ts b/server/tools/peertube-plugins.ts
index 47090b3a5..c51d9ebd1 100644
--- a/server/tools/peertube-plugins.ts
+++ b/server/tools/peertube-plugins.ts
@@ -69,7 +69,7 @@ async function pluginsListCLI (command: Command, options: OptionValues) {
69 69
70 const table = new CliTable3({ 70 const table = new CliTable3({
71 head: [ 'name', 'version', 'homepage' ], 71 head: [ 'name', 'version', 'homepage' ],
72 colWidths: [ 50, 10, 50 ] 72 colWidths: [ 50, 20, 50 ]
73 }) as any 73 }) as any
74 74
75 for (const plugin of data) { 75 for (const plugin of data) {
diff --git a/server/tools/peertube-redundancy.ts b/server/tools/peertube-redundancy.ts
index 4bb9fbc5a..5c82fa420 100644
--- a/server/tools/peertube-redundancy.ts
+++ b/server/tools/peertube-redundancy.ts
@@ -2,7 +2,7 @@ import CliTable3 from 'cli-table3'
2import { Command, program } from 'commander' 2import { Command, program } from 'commander'
3import { URL } from 'url' 3import { URL } from 'url'
4import validator from 'validator' 4import validator from 'validator'
5import { uniqify } from '@shared/core-utils' 5import { forceNumber, uniqify } from '@shared/core-utils'
6import { HttpStatusCode, VideoRedundanciesTarget } from '@shared/models' 6import { HttpStatusCode, VideoRedundanciesTarget } from '@shared/models'
7import { assignToken, buildServer, getServerCredentials } from './cli' 7import { assignToken, buildServer, getServerCredentials } from './cli'
8 8
@@ -138,7 +138,7 @@ async function removeRedundancyCLI (options: { video: number }, command: Command
138 process.exit(-1) 138 process.exit(-1)
139 } 139 }
140 140
141 const videoId = parseInt(options.video + '', 10) 141 const videoId = forceNumber(options.video)
142 142
143 const myVideoRedundancies = await server.redundancy.listVideos({ target: 'my-videos' }) 143 const myVideoRedundancies = await server.redundancy.listVideos({ target: 'my-videos' })
144 let videoRedundancy = myVideoRedundancies.data.find(r => videoId === r.id) 144 let videoRedundancy = myVideoRedundancies.data.find(r => videoId === r.id)
diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index 27d60da72..3738ffc47 100644
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -97,7 +97,7 @@ declare module 'express' {
97 97
98 title?: string 98 title?: string
99 status?: number 99 status?: number
100 type?: ServerErrorCode 100 type?: ServerErrorCode | string
101 instance?: string 101 instance?: string
102 102
103 data?: PeerTubeProblemDocumentData 103 data?: PeerTubeProblemDocumentData
diff --git a/server/types/models/video/video-playlist.ts b/server/types/models/video/video-playlist.ts
index 33fe5416a..40f0dfc14 100644
--- a/server/types/models/video/video-playlist.ts
+++ b/server/types/models/video/video-playlist.ts
@@ -14,6 +14,10 @@ export type MVideoPlaylist = Omit<VideoPlaylistModel, 'OwnerAccount' | 'VideoCha
14// ############################################################################ 14// ############################################################################
15 15
16export type MVideoPlaylistId = Pick<MVideoPlaylist, 'id'> 16export type MVideoPlaylistId = Pick<MVideoPlaylist, 'id'>
17export type MVideoPlaylistSummary =
18 Pick<MVideoPlaylist, 'id'> &
19 Pick<MVideoPlaylist, 'name'> &
20 Pick<MVideoPlaylist, 'uuid'>
17export type MVideoPlaylistPrivacy = Pick<MVideoPlaylist, 'privacy'> 21export type MVideoPlaylistPrivacy = Pick<MVideoPlaylist, 'privacy'>
18export type MVideoPlaylistUUID = Pick<MVideoPlaylist, 'uuid'> 22export type MVideoPlaylistUUID = Pick<MVideoPlaylist, 'uuid'>
19export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: number } 23export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: number }
@@ -22,12 +26,8 @@ export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: numbe
22 26
23// With elements 27// With elements
24 28
25export type MVideoPlaylistWithElements = 29export type MVideoPlaylistSummaryWithElements =
26 MVideoPlaylist & 30 MVideoPlaylistSummary &
27 Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]>
28
29export type MVideoPlaylistIdWithElements =
30 MVideoPlaylistId &
31 Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]> 31 Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]>
32 32
33// ############################################################################ 33// ############################################################################
diff --git a/server/types/plugins/index.ts b/server/types/plugins/index.ts
index de30ff2ab..bf9c35d49 100644
--- a/server/types/plugins/index.ts
+++ b/server/types/plugins/index.ts
@@ -1,3 +1,4 @@
1export * from './plugin-library.model' 1export * from './plugin-library.model'
2export * from './register-server-auth.model' 2export * from './register-server-auth.model'
3export * from './register-server-option.model' 3export * from './register-server-option.model'
4export * from './register-server-websocket-route.model'
diff --git a/server/types/plugins/register-server-option.model.ts b/server/types/plugins/register-server-option.model.ts
index fb4f12a4c..1e2bd830e 100644
--- a/server/types/plugins/register-server-option.model.ts
+++ b/server/types/plugins/register-server-option.model.ts
@@ -1,4 +1,5 @@
1import { Response, Router } from 'express' 1import { Response, Router } from 'express'
2import { Server } from 'http'
2import { Logger } from 'winston' 3import { Logger } from 'winston'
3import { ActorModel } from '@server/models/actor/actor' 4import { ActorModel } from '@server/models/actor/actor'
4import { 5import {
@@ -16,12 +17,13 @@ import {
16 ThumbnailType, 17 ThumbnailType,
17 VideoBlacklistCreate 18 VideoBlacklistCreate
18} from '@shared/models' 19} from '@shared/models'
19import { MUserDefault, MVideoThumbnail } from '../models' 20import { MUserDefault, MVideo, MVideoThumbnail, UserNotificationModelForApi } from '../models'
20import { 21import {
21 RegisterServerAuthExternalOptions, 22 RegisterServerAuthExternalOptions,
22 RegisterServerAuthExternalResult, 23 RegisterServerAuthExternalResult,
23 RegisterServerAuthPassOptions 24 RegisterServerAuthPassOptions
24} from './register-server-auth.model' 25} from './register-server-auth.model'
26import { RegisterServerWebSocketRouteOptions } from './register-server-websocket-route.model'
25 27
26export type PeerTubeHelpers = { 28export type PeerTubeHelpers = {
27 logger: Logger 29 logger: Logger
@@ -83,15 +85,25 @@ export type PeerTubeHelpers = {
83 } 85 }
84 86
85 server: { 87 server: {
88 // PeerTube >= 5.0
89 getHTTPServer: () => Server
90
86 getServerActor: () => Promise<ActorModel> 91 getServerActor: () => Promise<ActorModel>
87 } 92 }
88 93
94 socket: {
95 sendNotification: (userId: number, notification: UserNotificationModelForApi) => void
96 sendVideoLiveNewState: (video: MVideo) => void
97 }
98
89 plugin: { 99 plugin: {
90 // PeerTube >= 3.2 100 // PeerTube >= 3.2
91 getBaseStaticRoute: () => string 101 getBaseStaticRoute: () => string
92 102
93 // PeerTube >= 3.2 103 // PeerTube >= 3.2
94 getBaseRouterRoute: () => string 104 getBaseRouterRoute: () => string
105 // PeerTube >= 5.0
106 getBaseWebSocketRoute: () => string
95 107
96 // PeerTube >= 3.2 108 // PeerTube >= 3.2
97 getDataDirectoryPath: () => string 109 getDataDirectoryPath: () => string
@@ -135,5 +147,12 @@ export type RegisterServerOptions = {
135 // * /plugins/:pluginName/router/... 147 // * /plugins/:pluginName/router/...
136 getRouter(): Router 148 getRouter(): Router
137 149
150 // PeerTube >= 5.0
151 // Register WebSocket route
152 // Base routes of the WebSocket router are
153 // * /plugins/:pluginName/:pluginVersion/ws/...
154 // * /plugins/:pluginName/ws/...
155 registerWebSocketRoute: (options: RegisterServerWebSocketRouteOptions) => void
156
138 peertubeHelpers: PeerTubeHelpers 157 peertubeHelpers: PeerTubeHelpers
139} 158}
diff --git a/server/types/plugins/register-server-websocket-route.model.ts b/server/types/plugins/register-server-websocket-route.model.ts
new file mode 100644
index 000000000..edf64f66b
--- /dev/null
+++ b/server/types/plugins/register-server-websocket-route.model.ts
@@ -0,0 +1,8 @@
1import { IncomingMessage } from 'http'
2import { Duplex } from 'stream'
3
4export type RegisterServerWebSocketRouteOptions = {
5 route: string
6
7 handler: (request: IncomingMessage, socket: Duplex, head: Buffer) => any
8}