diff options
Diffstat (limited to 'server')
162 files changed, 6209 insertions, 1752 deletions
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index 4e5333782..f3792bfc8 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts | |||
@@ -8,6 +8,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' | |||
8 | import { UserRight } from '../../../../shared/models/users' | 8 | import { UserRight } from '../../../../shared/models/users' |
9 | import { authenticate, ensureUserHasRight } from '../../../middlewares' | 9 | import { authenticate, ensureUserHasRight } from '../../../middlewares' |
10 | import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' | 10 | import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' |
11 | import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler' | ||
11 | 12 | ||
12 | const debugRouter = express.Router() | 13 | const debugRouter = express.Router() |
13 | 14 | ||
@@ -45,6 +46,7 @@ async function runCommand (req: express.Request, res: express.Response) { | |||
45 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), | 46 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), |
46 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), | 47 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), |
47 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), | 48 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), |
49 | 'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(), | ||
48 | 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() | 50 | 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() |
49 | } | 51 | } |
50 | 52 | ||
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 0b27d5277..a8677a1d3 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts | |||
@@ -51,6 +51,7 @@ import { myVideosHistoryRouter } from './my-history' | |||
51 | import { myNotificationsRouter } from './my-notifications' | 51 | import { myNotificationsRouter } from './my-notifications' |
52 | import { mySubscriptionsRouter } from './my-subscriptions' | 52 | import { mySubscriptionsRouter } from './my-subscriptions' |
53 | import { myVideoPlaylistsRouter } from './my-video-playlists' | 53 | import { myVideoPlaylistsRouter } from './my-video-playlists' |
54 | import { twoFactorRouter } from './two-factor' | ||
54 | 55 | ||
55 | const auditLogger = auditLoggerFactory('users') | 56 | const auditLogger = auditLoggerFactory('users') |
56 | 57 | ||
@@ -66,6 +67,7 @@ const askSendEmailLimiter = buildRateLimiter({ | |||
66 | }) | 67 | }) |
67 | 68 | ||
68 | const usersRouter = express.Router() | 69 | const usersRouter = express.Router() |
70 | usersRouter.use('/', twoFactorRouter) | ||
69 | usersRouter.use('/', tokensRouter) | 71 | usersRouter.use('/', tokensRouter) |
70 | usersRouter.use('/', myNotificationsRouter) | 72 | usersRouter.use('/', myNotificationsRouter) |
71 | usersRouter.use('/', mySubscriptionsRouter) | 73 | usersRouter.use('/', mySubscriptionsRouter) |
@@ -343,7 +345,7 @@ async function askResetUserPassword (req: express.Request, res: express.Response | |||
343 | 345 | ||
344 | const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) | 346 | const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) |
345 | const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString | 347 | const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString |
346 | await Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url) | 348 | Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url) |
347 | 349 | ||
348 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | 350 | return res.status(HttpStatusCode.NO_CONTENT_204).end() |
349 | } | 351 | } |
diff --git a/server/controllers/api/users/my-video-playlists.ts b/server/controllers/api/users/my-video-playlists.ts index f55ea2ec4..715717610 100644 --- a/server/controllers/api/users/my-video-playlists.ts +++ b/server/controllers/api/users/my-video-playlists.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import { uuidToShort } from '@shared/extra-utils' | ||
1 | import express from 'express' | 2 | import express from 'express' |
2 | import { VideosExistInPlaylists } from '../../../../shared/models/videos/playlist/video-exist-in-playlist.model' | 3 | import { VideosExistInPlaylists } from '../../../../shared/models/videos/playlist/video-exist-in-playlist.model' |
3 | import { asyncMiddleware, authenticate } from '../../../middlewares' | 4 | import { asyncMiddleware, authenticate } from '../../../middlewares' |
@@ -24,7 +25,7 @@ async function doVideosInPlaylistExist (req: express.Request, res: express.Respo | |||
24 | const videoIds = req.query.videoIds.map(i => parseInt(i + '', 10)) | 25 | const videoIds = req.query.videoIds.map(i => parseInt(i + '', 10)) |
25 | const user = res.locals.oauth.token.User | 26 | const user = res.locals.oauth.token.User |
26 | 27 | ||
27 | const results = await VideoPlaylistModel.listPlaylistIdsOf(user.Account.id, videoIds) | 28 | const results = await VideoPlaylistModel.listPlaylistSummariesOf(user.Account.id, videoIds) |
28 | 29 | ||
29 | const existObject: VideosExistInPlaylists = {} | 30 | const existObject: VideosExistInPlaylists = {} |
30 | 31 | ||
@@ -37,6 +38,8 @@ async function doVideosInPlaylistExist (req: express.Request, res: express.Respo | |||
37 | existObject[element.videoId].push({ | 38 | existObject[element.videoId].push({ |
38 | playlistElementId: element.id, | 39 | playlistElementId: element.id, |
39 | playlistId: result.id, | 40 | playlistId: result.id, |
41 | playlistDisplayName: result.name, | ||
42 | playlistShortUUID: uuidToShort(result.uuid), | ||
40 | startTimestamp: element.startTimestamp, | 43 | startTimestamp: element.startTimestamp, |
41 | stopTimestamp: element.stopTimestamp | 44 | stopTimestamp: element.stopTimestamp |
42 | }) | 45 | }) |
diff --git a/server/controllers/api/users/token.ts b/server/controllers/api/users/token.ts index 012a49791..c6afea67c 100644 --- a/server/controllers/api/users/token.ts +++ b/server/controllers/api/users/token.ts | |||
@@ -1,8 +1,9 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
3 | import { CONFIG } from '@server/initializers/config' | 3 | import { CONFIG } from '@server/initializers/config' |
4 | import { OTP } from '@server/initializers/constants' | ||
4 | import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' | 5 | import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' |
5 | import { handleOAuthToken } from '@server/lib/auth/oauth' | 6 | import { handleOAuthToken, MissingTwoFactorError } from '@server/lib/auth/oauth' |
6 | import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' | 7 | import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' |
7 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
8 | import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares' | 9 | import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares' |
@@ -79,6 +80,10 @@ async function handleToken (req: express.Request, res: express.Response, next: e | |||
79 | } catch (err) { | 80 | } catch (err) { |
80 | logger.warn('Login error', { err }) | 81 | logger.warn('Login error', { err }) |
81 | 82 | ||
83 | if (err instanceof MissingTwoFactorError) { | ||
84 | res.set(OTP.HEADER_NAME, OTP.HEADER_REQUIRED_VALUE) | ||
85 | } | ||
86 | |||
82 | return res.fail({ | 87 | return res.fail({ |
83 | status: err.code, | 88 | status: err.code, |
84 | message: err.message, | 89 | message: err.message, |
diff --git a/server/controllers/api/users/two-factor.ts b/server/controllers/api/users/two-factor.ts new file mode 100644 index 000000000..e6ae9e4dd --- /dev/null +++ b/server/controllers/api/users/two-factor.ts | |||
@@ -0,0 +1,95 @@ | |||
1 | import express from 'express' | ||
2 | import { generateOTPSecret, isOTPValid } from '@server/helpers/otp' | ||
3 | import { encrypt } from '@server/helpers/peertube-crypto' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { Redis } from '@server/lib/redis' | ||
6 | import { asyncMiddleware, authenticate, usersCheckCurrentPasswordFactory } from '@server/middlewares' | ||
7 | import { | ||
8 | confirmTwoFactorValidator, | ||
9 | disableTwoFactorValidator, | ||
10 | requestOrConfirmTwoFactorValidator | ||
11 | } from '@server/middlewares/validators/two-factor' | ||
12 | import { HttpStatusCode, TwoFactorEnableResult } from '@shared/models' | ||
13 | |||
14 | const twoFactorRouter = express.Router() | ||
15 | |||
16 | twoFactorRouter.post('/:id/two-factor/request', | ||
17 | authenticate, | ||
18 | asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)), | ||
19 | asyncMiddleware(requestOrConfirmTwoFactorValidator), | ||
20 | asyncMiddleware(requestTwoFactor) | ||
21 | ) | ||
22 | |||
23 | twoFactorRouter.post('/:id/two-factor/confirm-request', | ||
24 | authenticate, | ||
25 | asyncMiddleware(requestOrConfirmTwoFactorValidator), | ||
26 | confirmTwoFactorValidator, | ||
27 | asyncMiddleware(confirmRequestTwoFactor) | ||
28 | ) | ||
29 | |||
30 | twoFactorRouter.post('/:id/two-factor/disable', | ||
31 | authenticate, | ||
32 | asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)), | ||
33 | asyncMiddleware(disableTwoFactorValidator), | ||
34 | asyncMiddleware(disableTwoFactor) | ||
35 | ) | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | export { | ||
40 | twoFactorRouter | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | async function requestTwoFactor (req: express.Request, res: express.Response) { | ||
46 | const user = res.locals.user | ||
47 | |||
48 | const { secret, uri } = generateOTPSecret(user.email) | ||
49 | |||
50 | const encryptedSecret = await encrypt(secret, CONFIG.SECRETS.PEERTUBE) | ||
51 | const requestToken = await Redis.Instance.setTwoFactorRequest(user.id, encryptedSecret) | ||
52 | |||
53 | return res.json({ | ||
54 | otpRequest: { | ||
55 | requestToken, | ||
56 | secret, | ||
57 | uri | ||
58 | } | ||
59 | } as TwoFactorEnableResult) | ||
60 | } | ||
61 | |||
62 | async function confirmRequestTwoFactor (req: express.Request, res: express.Response) { | ||
63 | const requestToken = req.body.requestToken | ||
64 | const otpToken = req.body.otpToken | ||
65 | const user = res.locals.user | ||
66 | |||
67 | const encryptedSecret = await Redis.Instance.getTwoFactorRequestToken(user.id, requestToken) | ||
68 | if (!encryptedSecret) { | ||
69 | return res.fail({ | ||
70 | message: 'Invalid request token', | ||
71 | status: HttpStatusCode.FORBIDDEN_403 | ||
72 | }) | ||
73 | } | ||
74 | |||
75 | if (await isOTPValid({ encryptedSecret, token: otpToken }) !== true) { | ||
76 | return res.fail({ | ||
77 | message: 'Invalid OTP token', | ||
78 | status: HttpStatusCode.FORBIDDEN_403 | ||
79 | }) | ||
80 | } | ||
81 | |||
82 | user.otpSecret = encryptedSecret | ||
83 | await user.save() | ||
84 | |||
85 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
86 | } | ||
87 | |||
88 | async function disableTwoFactor (req: express.Request, res: express.Response) { | ||
89 | const user = res.locals.user | ||
90 | |||
91 | user.otpSecret = null | ||
92 | await user.save() | ||
93 | |||
94 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
95 | } | ||
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 9d7b0260b..6a50aaf4e 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -3,7 +3,7 @@ import { move, readFile } from 'fs-extra' | |||
3 | import { decode } from 'magnet-uri' | 3 | import { decode } from 'magnet-uri' |
4 | import parseTorrent, { Instance } from 'parse-torrent' | 4 | import parseTorrent, { Instance } from 'parse-torrent' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import' | 6 | import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import' |
7 | import { MThumbnail, MVideoThumbnail } from '@server/types/models' | 7 | import { MThumbnail, MVideoThumbnail } from '@server/types/models' |
8 | import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models' | 8 | import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models' |
9 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' | 9 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' |
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index b301515df..ea081e5ab 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -41,6 +41,7 @@ import { ownershipVideoRouter } from './ownership' | |||
41 | import { rateVideoRouter } from './rate' | 41 | import { rateVideoRouter } from './rate' |
42 | import { statsRouter } from './stats' | 42 | import { statsRouter } from './stats' |
43 | import { studioRouter } from './studio' | 43 | import { studioRouter } from './studio' |
44 | import { tokenRouter } from './token' | ||
44 | import { transcodingRouter } from './transcoding' | 45 | import { transcodingRouter } from './transcoding' |
45 | import { updateRouter } from './update' | 46 | import { updateRouter } from './update' |
46 | import { uploadRouter } from './upload' | 47 | import { uploadRouter } from './upload' |
@@ -63,6 +64,7 @@ videosRouter.use('/', uploadRouter) | |||
63 | videosRouter.use('/', updateRouter) | 64 | videosRouter.use('/', updateRouter) |
64 | videosRouter.use('/', filesRouter) | 65 | videosRouter.use('/', filesRouter) |
65 | videosRouter.use('/', transcodingRouter) | 66 | videosRouter.use('/', transcodingRouter) |
67 | videosRouter.use('/', tokenRouter) | ||
66 | 68 | ||
67 | videosRouter.get('/categories', | 69 | videosRouter.get('/categories', |
68 | openapiOperationDoc({ operationId: 'getCategories' }), | 70 | openapiOperationDoc({ operationId: 'getCategories' }), |
diff --git a/server/controllers/api/videos/token.ts b/server/controllers/api/videos/token.ts new file mode 100644 index 000000000..009b6dfb6 --- /dev/null +++ b/server/controllers/api/videos/token.ts | |||
@@ -0,0 +1,33 @@ | |||
1 | import express from 'express' | ||
2 | import { VideoTokensManager } from '@server/lib/video-tokens-manager' | ||
3 | import { VideoToken } from '@shared/models' | ||
4 | import { asyncMiddleware, authenticate, videosCustomGetValidator } from '../../../middlewares' | ||
5 | |||
6 | const tokenRouter = express.Router() | ||
7 | |||
8 | tokenRouter.post('/:id/token', | ||
9 | authenticate, | ||
10 | asyncMiddleware(videosCustomGetValidator('only-video')), | ||
11 | generateToken | ||
12 | ) | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export { | ||
17 | tokenRouter | ||
18 | } | ||
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | function generateToken (req: express.Request, res: express.Response) { | ||
23 | const video = res.locals.onlyVideo | ||
24 | |||
25 | const { token, expires } = VideoTokensManager.Instance.create(video.uuid) | ||
26 | |||
27 | return res.json({ | ||
28 | files: { | ||
29 | token, | ||
30 | expires | ||
31 | } | ||
32 | } as VideoToken) | ||
33 | } | ||
diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts index 9aca761c1..8c9a5322b 100644 --- a/server/controllers/api/videos/transcoding.ts +++ b/server/controllers/api/videos/transcoding.ts | |||
@@ -32,9 +32,10 @@ async function createTranscoding (req: express.Request, res: express.Response) { | |||
32 | 32 | ||
33 | const body: VideoTranscodingCreate = req.body | 33 | const body: VideoTranscodingCreate = req.body |
34 | 34 | ||
35 | const { resolution: maxResolution, audioStream } = await video.probeMaxQualityFile() | 35 | const { resolution: maxResolution, hasAudio } = await video.probeMaxQualityFile() |
36 | |||
36 | const resolutions = await Hooks.wrapObject( | 37 | const resolutions = await Hooks.wrapObject( |
37 | computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false }), | 38 | computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false, hasAudio }), |
38 | 'filter:transcoding.manual.resolutions-to-transcode.result', | 39 | 'filter:transcoding.manual.resolutions-to-transcode.result', |
39 | body | 40 | body |
40 | ) | 41 | ) |
@@ -46,9 +47,10 @@ async function createTranscoding (req: express.Request, res: express.Response) { | |||
46 | video.state = VideoState.TO_TRANSCODE | 47 | video.state = VideoState.TO_TRANSCODE |
47 | await video.save() | 48 | await video.save() |
48 | 49 | ||
49 | const hasAudio = !!audioStream | ||
50 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | 50 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) |
51 | 51 | ||
52 | logger.info('Manually creating transcoding jobs for %s.', body.transcodingType, { childrenResolutions, maxResolution }) | ||
53 | |||
52 | const children = await Bluebird.mapSeries(childrenResolutions, resolution => { | 54 | const children = await Bluebird.mapSeries(childrenResolutions, resolution => { |
53 | if (body.transcodingType === 'hls') { | 55 | if (body.transcodingType === 'hls') { |
54 | return buildHLSJobOption({ | 56 | return buildHLSJobOption({ |
diff --git a/server/controllers/api/videos/update.ts b/server/controllers/api/videos/update.ts index ab1a23d9a..0a910379a 100644 --- a/server/controllers/api/videos/update.ts +++ b/server/controllers/api/videos/update.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { Transaction } from 'sequelize/types' | 2 | import { Transaction } from 'sequelize/types' |
3 | import { changeVideoChannelShare } from '@server/lib/activitypub/share' | 3 | import { changeVideoChannelShare } from '@server/lib/activitypub/share' |
4 | import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' | 4 | import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' |
5 | import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' | 5 | import { setVideoPrivacy } from '@server/lib/video-privacy' |
6 | import { openapiOperationDoc } from '@server/middlewares/doc' | 6 | import { openapiOperationDoc } from '@server/middlewares/doc' |
7 | import { FilteredModelAttributes } from '@server/types' | 7 | import { FilteredModelAttributes } from '@server/types' |
8 | import { MVideoFullLight } from '@server/types/models' | 8 | import { MVideoFullLight } from '@server/types/models' |
9 | import { HttpStatusCode, ManageVideoTorrentPayload, VideoUpdate } from '@shared/models' | 9 | import { HttpStatusCode, VideoUpdate } from '@shared/models' |
10 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' | 10 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' |
11 | import { resetSequelizeInstance } from '../../../helpers/database-utils' | 11 | import { resetSequelizeInstance } from '../../../helpers/database-utils' |
12 | import { createReqFiles } from '../../../helpers/express-utils' | 12 | import { createReqFiles } from '../../../helpers/express-utils' |
@@ -18,6 +18,7 @@ import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' | |||
18 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares' | 18 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares' |
19 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' | 19 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' |
20 | import { VideoModel } from '../../../models/video/video' | 20 | import { VideoModel } from '../../../models/video/video' |
21 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
21 | 22 | ||
22 | const lTags = loggerTagsFactory('api', 'video') | 23 | const lTags = loggerTagsFactory('api', 'video') |
23 | const auditLogger = auditLoggerFactory('videos') | 24 | const auditLogger = auditLoggerFactory('videos') |
@@ -47,8 +48,8 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
47 | const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON()) | 48 | const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON()) |
48 | const videoInfoToUpdate: VideoUpdate = req.body | 49 | const videoInfoToUpdate: VideoUpdate = req.body |
49 | 50 | ||
50 | const wasConfidentialVideo = videoFromReq.isConfidential() | ||
51 | const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation() | 51 | const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation() |
52 | const oldPrivacy = videoFromReq.privacy | ||
52 | 53 | ||
53 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ | 54 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ |
54 | video: videoFromReq, | 55 | video: videoFromReq, |
@@ -57,12 +58,13 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
57 | automaticallyGenerated: false | 58 | automaticallyGenerated: false |
58 | }) | 59 | }) |
59 | 60 | ||
61 | const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid) | ||
62 | |||
60 | try { | 63 | try { |
61 | const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => { | 64 | const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => { |
62 | // Refresh video since thumbnails to prevent concurrent updates | 65 | // Refresh video since thumbnails to prevent concurrent updates |
63 | const video = await VideoModel.loadFull(videoFromReq.id, t) | 66 | const video = await VideoModel.loadFull(videoFromReq.id, t) |
64 | 67 | ||
65 | const sequelizeOptions = { transaction: t } | ||
66 | const oldVideoChannel = video.VideoChannel | 68 | const oldVideoChannel = video.VideoChannel |
67 | 69 | ||
68 | const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [ | 70 | const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [ |
@@ -97,7 +99,7 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
97 | await video.setAsRefreshed(t) | 99 | await video.setAsRefreshed(t) |
98 | } | 100 | } |
99 | 101 | ||
100 | const videoInstanceUpdated = await video.save(sequelizeOptions) as MVideoFullLight | 102 | const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight |
101 | 103 | ||
102 | // Thumbnail & preview updates? | 104 | // Thumbnail & preview updates? |
103 | if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) | 105 | if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) |
@@ -113,7 +115,9 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
113 | await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t }) | 115 | await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t }) |
114 | videoInstanceUpdated.VideoChannel = res.locals.videoChannel | 116 | videoInstanceUpdated.VideoChannel = res.locals.videoChannel |
115 | 117 | ||
116 | if (hadPrivacyForFederation === true) await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t) | 118 | if (hadPrivacyForFederation === true) { |
119 | await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t) | ||
120 | } | ||
117 | } | 121 | } |
118 | 122 | ||
119 | // Schedule an update in the future? | 123 | // Schedule an update in the future? |
@@ -139,7 +143,12 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
139 | 143 | ||
140 | Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res }) | 144 | Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res }) |
141 | 145 | ||
142 | await addVideoJobsAfterUpdate({ video: videoInstanceUpdated, videoInfoToUpdate, wasConfidentialVideo, isNewVideo }) | 146 | await addVideoJobsAfterUpdate({ |
147 | video: videoInstanceUpdated, | ||
148 | nameChanged: !!videoInfoToUpdate.name, | ||
149 | oldPrivacy, | ||
150 | isNewVideo | ||
151 | }) | ||
143 | } catch (err) { | 152 | } catch (err) { |
144 | // Force fields we want to update | 153 | // Force fields we want to update |
145 | // If the transaction is retried, sequelize will think the object has not changed | 154 | // If the transaction is retried, sequelize will think the object has not changed |
@@ -147,6 +156,8 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
147 | resetSequelizeInstance(videoFromReq, videoFieldsSave) | 156 | resetSequelizeInstance(videoFromReq, videoFieldsSave) |
148 | 157 | ||
149 | throw err | 158 | throw err |
159 | } finally { | ||
160 | videoFileLockReleaser() | ||
150 | } | 161 | } |
151 | 162 | ||
152 | return res.type('json') | 163 | return res.type('json') |
@@ -164,7 +175,7 @@ async function updateVideoPrivacy (options: { | |||
164 | const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy) | 175 | const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy) |
165 | 176 | ||
166 | const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10) | 177 | const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10) |
167 | videoInstance.setPrivacy(newPrivacy) | 178 | setVideoPrivacy(videoInstance, newPrivacy) |
168 | 179 | ||
169 | // Unfederate the video if the new privacy is not compatible with federation | 180 | // Unfederate the video if the new privacy is not compatible with federation |
170 | if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) { | 181 | if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) { |
@@ -185,50 +196,3 @@ function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: Vide | |||
185 | return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction) | 196 | return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction) |
186 | } | 197 | } |
187 | } | 198 | } |
188 | |||
189 | async function addVideoJobsAfterUpdate (options: { | ||
190 | video: MVideoFullLight | ||
191 | videoInfoToUpdate: VideoUpdate | ||
192 | wasConfidentialVideo: boolean | ||
193 | isNewVideo: boolean | ||
194 | }) { | ||
195 | const { video, videoInfoToUpdate, wasConfidentialVideo, isNewVideo } = options | ||
196 | const jobs: CreateJobArgument[] = [] | ||
197 | |||
198 | if (!video.isLive && videoInfoToUpdate.name) { | ||
199 | |||
200 | for (const file of (video.VideoFiles || [])) { | ||
201 | const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } | ||
202 | |||
203 | jobs.push({ type: 'manage-video-torrent', payload }) | ||
204 | } | ||
205 | |||
206 | const hls = video.getHLSPlaylist() | ||
207 | |||
208 | for (const file of (hls?.VideoFiles || [])) { | ||
209 | const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } | ||
210 | |||
211 | jobs.push({ type: 'manage-video-torrent', payload }) | ||
212 | } | ||
213 | } | ||
214 | |||
215 | jobs.push({ | ||
216 | type: 'federate-video', | ||
217 | payload: { | ||
218 | videoUUID: video.uuid, | ||
219 | isNewVideo | ||
220 | } | ||
221 | }) | ||
222 | |||
223 | if (wasConfidentialVideo) { | ||
224 | jobs.push({ | ||
225 | type: 'notify', | ||
226 | payload: { | ||
227 | action: 'new-video', | ||
228 | videoUUID: video.uuid | ||
229 | } | ||
230 | }) | ||
231 | } | ||
232 | |||
233 | return JobQueue.Instance.createSequentialJobFlow(...jobs) | ||
234 | } | ||
diff --git a/server/controllers/download.ts b/server/controllers/download.ts index a270180c0..d9f34109f 100644 --- a/server/controllers/download.ts +++ b/server/controllers/download.ts | |||
@@ -5,9 +5,10 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache | |||
5 | import { Hooks } from '@server/lib/plugins/hooks' | 5 | import { Hooks } from '@server/lib/plugins/hooks' |
6 | import { VideoPathManager } from '@server/lib/video-path-manager' | 6 | import { VideoPathManager } from '@server/lib/video-path-manager' |
7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
8 | import { addQueryParams } from '@shared/core-utils' | ||
8 | import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' | 9 | import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' |
9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' | 10 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' |
10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' | 11 | import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares' |
11 | 12 | ||
12 | const downloadRouter = express.Router() | 13 | const downloadRouter = express.Router() |
13 | 14 | ||
@@ -20,12 +21,14 @@ downloadRouter.use( | |||
20 | 21 | ||
21 | downloadRouter.use( | 22 | downloadRouter.use( |
22 | STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', | 23 | STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', |
24 | optionalAuthenticate, | ||
23 | asyncMiddleware(videosDownloadValidator), | 25 | asyncMiddleware(videosDownloadValidator), |
24 | asyncMiddleware(downloadVideoFile) | 26 | asyncMiddleware(downloadVideoFile) |
25 | ) | 27 | ) |
26 | 28 | ||
27 | downloadRouter.use( | 29 | downloadRouter.use( |
28 | STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', | 30 | STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', |
31 | optionalAuthenticate, | ||
29 | asyncMiddleware(videosDownloadValidator), | 32 | asyncMiddleware(videosDownloadValidator), |
30 | asyncMiddleware(downloadHLSVideoFile) | 33 | asyncMiddleware(downloadHLSVideoFile) |
31 | ) | 34 | ) |
@@ -82,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { | |||
82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 85 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
83 | 86 | ||
84 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { | 87 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
85 | return res.redirect(videoFile.getObjectStorageUrl()) | 88 | return redirectToObjectStorage({ req, res, video, file: videoFile }) |
86 | } | 89 | } |
87 | 90 | ||
88 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { | 91 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { |
@@ -118,7 +121,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response | |||
118 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 121 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
119 | 122 | ||
120 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { | 123 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
121 | return res.redirect(videoFile.getObjectStorageUrl()) | 124 | return redirectToObjectStorage({ req, res, video, file: videoFile }) |
122 | } | 125 | } |
123 | 126 | ||
124 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { | 127 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { |
@@ -172,3 +175,20 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?: | |||
172 | 175 | ||
173 | return true | 176 | return true |
174 | } | 177 | } |
178 | |||
179 | function redirectToObjectStorage (options: { | ||
180 | req: express.Request | ||
181 | res: express.Response | ||
182 | video: MVideo | ||
183 | file: MVideoFile | ||
184 | }) { | ||
185 | const { req, res, video, file } = options | ||
186 | |||
187 | const baseUrl = file.getObjectStorageUrl(video) | ||
188 | |||
189 | const url = video.hasPrivateStaticPath() && req.query.videoFileToken | ||
190 | ? addQueryParams(baseUrl, { videoFileToken: req.query.videoFileToken }) | ||
191 | : baseUrl | ||
192 | |||
193 | return res.redirect(url) | ||
194 | } | ||
diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts index 241715fb9..772fe734d 100644 --- a/server/controllers/feeds.ts +++ b/server/controllers/feeds.ts | |||
@@ -4,7 +4,8 @@ import { Feed } from '@peertube/feed' | |||
4 | import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' | 4 | import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' |
5 | import { getServerActor } from '@server/models/application/application' | 5 | import { getServerActor } from '@server/models/application/application' |
6 | import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils' | 6 | import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils' |
7 | import { VideoInclude } from '@shared/models' | 7 | import { MAccountDefault, MChannelBannerAccountDefault, MVideoFullLight } from '@server/types/models' |
8 | import { ActorImageType, VideoInclude } from '@shared/models' | ||
8 | import { buildNSFWFilter } from '../helpers/express-utils' | 9 | import { buildNSFWFilter } from '../helpers/express-utils' |
9 | import { CONFIG } from '../initializers/config' | 10 | import { CONFIG } from '../initializers/config' |
10 | import { MIMETYPES, PREVIEWS_SIZE, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' | 11 | import { MIMETYPES, PREVIEWS_SIZE, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' |
@@ -82,22 +83,12 @@ async function generateVideoCommentsFeed (req: express.Request, res: express.Res | |||
82 | videoChannelId: videoChannel ? videoChannel.id : undefined | 83 | videoChannelId: videoChannel ? videoChannel.id : undefined |
83 | }) | 84 | }) |
84 | 85 | ||
85 | let name: string | 86 | const { name, description, imageUrl } = buildFeedMetadata({ video, account, videoChannel }) |
86 | let description: string | ||
87 | 87 | ||
88 | if (videoChannel) { | ||
89 | name = videoChannel.getDisplayName() | ||
90 | description = videoChannel.description | ||
91 | } else if (account) { | ||
92 | name = account.getDisplayName() | ||
93 | description = account.description | ||
94 | } else { | ||
95 | name = video ? video.name : CONFIG.INSTANCE.NAME | ||
96 | description = video ? video.description : CONFIG.INSTANCE.DESCRIPTION | ||
97 | } | ||
98 | const feed = initFeed({ | 88 | const feed = initFeed({ |
99 | name, | 89 | name, |
100 | description, | 90 | description, |
91 | imageUrl, | ||
101 | resourceType: 'video-comments', | 92 | resourceType: 'video-comments', |
102 | queryString: new URL(WEBSERVER.URL + req.originalUrl).search | 93 | queryString: new URL(WEBSERVER.URL + req.originalUrl).search |
103 | }) | 94 | }) |
@@ -137,23 +128,12 @@ async function generateVideoFeed (req: express.Request, res: express.Response) { | |||
137 | const videoChannel = res.locals.videoChannel | 128 | const videoChannel = res.locals.videoChannel |
138 | const nsfw = buildNSFWFilter(res, req.query.nsfw) | 129 | const nsfw = buildNSFWFilter(res, req.query.nsfw) |
139 | 130 | ||
140 | let name: string | 131 | const { name, description, imageUrl } = buildFeedMetadata({ videoChannel, account }) |
141 | let description: string | ||
142 | |||
143 | if (videoChannel) { | ||
144 | name = videoChannel.getDisplayName() | ||
145 | description = videoChannel.description | ||
146 | } else if (account) { | ||
147 | name = account.getDisplayName() | ||
148 | description = account.description | ||
149 | } else { | ||
150 | name = CONFIG.INSTANCE.NAME | ||
151 | description = CONFIG.INSTANCE.DESCRIPTION | ||
152 | } | ||
153 | 132 | ||
154 | const feed = initFeed({ | 133 | const feed = initFeed({ |
155 | name, | 134 | name, |
156 | description, | 135 | description, |
136 | imageUrl, | ||
157 | resourceType: 'videos', | 137 | resourceType: 'videos', |
158 | queryString: new URL(WEBSERVER.URL + req.url).search | 138 | queryString: new URL(WEBSERVER.URL + req.url).search |
159 | }) | 139 | }) |
@@ -190,12 +170,13 @@ async function generateVideoFeedForSubscriptions (req: express.Request, res: exp | |||
190 | const start = 0 | 170 | const start = 0 |
191 | const account = res.locals.account | 171 | const account = res.locals.account |
192 | const nsfw = buildNSFWFilter(res, req.query.nsfw) | 172 | const nsfw = buildNSFWFilter(res, req.query.nsfw) |
193 | const name = account.getDisplayName() | 173 | |
194 | const description = account.description | 174 | const { name, description, imageUrl } = buildFeedMetadata({ account }) |
195 | 175 | ||
196 | const feed = initFeed({ | 176 | const feed = initFeed({ |
197 | name, | 177 | name, |
198 | description, | 178 | description, |
179 | imageUrl, | ||
199 | resourceType: 'videos', | 180 | resourceType: 'videos', |
200 | queryString: new URL(WEBSERVER.URL + req.url).search | 181 | queryString: new URL(WEBSERVER.URL + req.url).search |
201 | }) | 182 | }) |
@@ -229,11 +210,12 @@ async function generateVideoFeedForSubscriptions (req: express.Request, res: exp | |||
229 | function initFeed (parameters: { | 210 | function initFeed (parameters: { |
230 | name: string | 211 | name: string |
231 | description: string | 212 | description: string |
213 | imageUrl: string | ||
232 | resourceType?: 'videos' | 'video-comments' | 214 | resourceType?: 'videos' | 'video-comments' |
233 | queryString?: string | 215 | queryString?: string |
234 | }) { | 216 | }) { |
235 | const webserverUrl = WEBSERVER.URL | 217 | const webserverUrl = WEBSERVER.URL |
236 | const { name, description, resourceType, queryString } = parameters | 218 | const { name, description, resourceType, queryString, imageUrl } = parameters |
237 | 219 | ||
238 | return new Feed({ | 220 | return new Feed({ |
239 | title: name, | 221 | title: name, |
@@ -241,7 +223,7 @@ function initFeed (parameters: { | |||
241 | // updated: TODO: somehowGetLatestUpdate, // optional, default = today | 223 | // updated: TODO: somehowGetLatestUpdate, // optional, default = today |
242 | id: webserverUrl, | 224 | id: webserverUrl, |
243 | link: webserverUrl, | 225 | link: webserverUrl, |
244 | image: webserverUrl + '/client/assets/images/icons/icon-96x96.png', | 226 | image: imageUrl, |
245 | favicon: webserverUrl + '/client/assets/images/favicon.png', | 227 | favicon: webserverUrl + '/client/assets/images/favicon.png', |
246 | copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + | 228 | copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + |
247 | ` and potential licenses granted by each content's rightholder.`, | 229 | ` and potential licenses granted by each content's rightholder.`, |
@@ -369,3 +351,39 @@ function sendFeed (feed: Feed, req: express.Request, res: express.Response) { | |||
369 | 351 | ||
370 | return res.send(feed.rss2()).end() | 352 | return res.send(feed.rss2()).end() |
371 | } | 353 | } |
354 | |||
355 | function buildFeedMetadata (options: { | ||
356 | videoChannel?: MChannelBannerAccountDefault | ||
357 | account?: MAccountDefault | ||
358 | video?: MVideoFullLight | ||
359 | }) { | ||
360 | const { video, videoChannel, account } = options | ||
361 | |||
362 | let imageUrl = WEBSERVER.URL + '/client/assets/images/icons/icon-96x96.png' | ||
363 | let name: string | ||
364 | let description: string | ||
365 | |||
366 | if (videoChannel) { | ||
367 | name = videoChannel.getDisplayName() | ||
368 | description = videoChannel.description | ||
369 | |||
370 | if (videoChannel.Actor.hasImage(ActorImageType.AVATAR)) { | ||
371 | imageUrl = WEBSERVER.URL + videoChannel.Actor.Avatars[0].getStaticPath() | ||
372 | } | ||
373 | } else if (account) { | ||
374 | name = account.getDisplayName() | ||
375 | description = account.description | ||
376 | |||
377 | if (account.Actor.hasImage(ActorImageType.AVATAR)) { | ||
378 | imageUrl = WEBSERVER.URL + account.Actor.Avatars[0].getStaticPath() | ||
379 | } | ||
380 | } else if (video) { | ||
381 | name = video.name | ||
382 | description = video.description | ||
383 | } else { | ||
384 | name = CONFIG.INSTANCE.NAME | ||
385 | description = CONFIG.INSTANCE.DESCRIPTION | ||
386 | } | ||
387 | |||
388 | return { name, description, imageUrl } | ||
389 | } | ||
diff --git a/server/controllers/index.ts b/server/controllers/index.ts index e8833d58c..eaa2dd7c8 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts | |||
@@ -1,15 +1,15 @@ | |||
1 | export * from './activitypub' | 1 | export * from './activitypub' |
2 | export * from './api' | 2 | export * from './api' |
3 | export * from './bots' | ||
3 | export * from './client' | 4 | export * from './client' |
4 | export * from './download' | 5 | export * from './download' |
5 | export * from './feeds' | 6 | export * from './feeds' |
6 | export * from './services' | ||
7 | export * from './static' | ||
8 | export * from './lazy-static' | 7 | export * from './lazy-static' |
9 | export * from './live' | ||
10 | export * from './misc' | 8 | export * from './misc' |
11 | export * from './webfinger' | 9 | export * from './object-storage-proxy' |
12 | export * from './tracker' | ||
13 | export * from './bots' | ||
14 | export * from './plugins' | 10 | export * from './plugins' |
11 | export * from './services' | ||
12 | export * from './static' | ||
13 | export * from './tracker' | ||
14 | export * from './webfinger' | ||
15 | export * from './well-known' | 15 | export * from './well-known' |
diff --git a/server/controllers/live.ts b/server/controllers/live.ts deleted file mode 100644 index 81008f120..000000000 --- a/server/controllers/live.ts +++ /dev/null | |||
@@ -1,32 +0,0 @@ | |||
1 | import cors from 'cors' | ||
2 | import express from 'express' | ||
3 | import { mapToJSON } from '@server/helpers/core-utils' | ||
4 | import { LiveSegmentShaStore } from '@server/lib/live' | ||
5 | import { HttpStatusCode } from '@shared/models' | ||
6 | |||
7 | const liveRouter = express.Router() | ||
8 | |||
9 | liveRouter.use('/segments-sha256/:videoUUID', | ||
10 | cors(), | ||
11 | getSegmentsSha256 | ||
12 | ) | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export { | ||
17 | liveRouter | ||
18 | } | ||
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | function getSegmentsSha256 (req: express.Request, res: express.Response) { | ||
23 | const videoUUID = req.params.videoUUID | ||
24 | |||
25 | const result = LiveSegmentShaStore.Instance.getSegmentsSha256(videoUUID) | ||
26 | |||
27 | if (!result) { | ||
28 | return res.status(HttpStatusCode.NOT_FOUND_404).end() | ||
29 | } | ||
30 | |||
31 | return res.json(mapToJSON(result)) | ||
32 | } | ||
diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts new file mode 100644 index 000000000..3ce279671 --- /dev/null +++ b/server/controllers/object-storage-proxy.ts | |||
@@ -0,0 +1,86 @@ | |||
1 | import cors from 'cors' | ||
2 | import express from 'express' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' | ||
5 | import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' | ||
6 | import { | ||
7 | asyncMiddleware, | ||
8 | ensureCanAccessPrivateVideoHLSFiles, | ||
9 | ensureCanAccessVideoPrivateWebTorrentFiles, | ||
10 | ensurePrivateObjectStorageProxyIsEnabled, | ||
11 | optionalAuthenticate | ||
12 | } from '@server/middlewares' | ||
13 | import { HttpStatusCode } from '@shared/models' | ||
14 | |||
15 | const objectStorageProxyRouter = express.Router() | ||
16 | |||
17 | objectStorageProxyRouter.use(cors()) | ||
18 | |||
19 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename', | ||
20 | ensurePrivateObjectStorageProxyIsEnabled, | ||
21 | optionalAuthenticate, | ||
22 | asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), | ||
23 | asyncMiddleware(proxifyWebTorrent) | ||
24 | ) | ||
25 | |||
26 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', | ||
27 | ensurePrivateObjectStorageProxyIsEnabled, | ||
28 | optionalAuthenticate, | ||
29 | asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), | ||
30 | asyncMiddleware(proxifyHLS) | ||
31 | ) | ||
32 | |||
33 | // --------------------------------------------------------------------------- | ||
34 | |||
35 | export { | ||
36 | objectStorageProxyRouter | ||
37 | } | ||
38 | |||
39 | async function proxifyWebTorrent (req: express.Request, res: express.Response) { | ||
40 | const filename = req.params.filename | ||
41 | |||
42 | logger.debug('Proxifying WebTorrent file %s from object storage.', filename) | ||
43 | |||
44 | try { | ||
45 | const stream = await getWebTorrentFileReadStream({ | ||
46 | filename, | ||
47 | rangeHeader: req.header('range') | ||
48 | }) | ||
49 | |||
50 | return stream.pipe(res) | ||
51 | } catch (err) { | ||
52 | return handleObjectStorageFailure(res, err) | ||
53 | } | ||
54 | } | ||
55 | |||
56 | async function proxifyHLS (req: express.Request, res: express.Response) { | ||
57 | const playlist = res.locals.videoStreamingPlaylist | ||
58 | const video = res.locals.onlyVideo | ||
59 | const filename = req.params.filename | ||
60 | |||
61 | logger.debug('Proxifying HLS file %s from object storage.', filename) | ||
62 | |||
63 | try { | ||
64 | const stream = await getHLSFileReadStream({ | ||
65 | playlist: playlist.withVideo(video), | ||
66 | filename, | ||
67 | rangeHeader: req.header('range') | ||
68 | }) | ||
69 | |||
70 | return stream.pipe(res) | ||
71 | } catch (err) { | ||
72 | return handleObjectStorageFailure(res, err) | ||
73 | } | ||
74 | } | ||
75 | |||
76 | function handleObjectStorageFailure (res: express.Response, err: Error) { | ||
77 | if (err.name === 'NoSuchKey') { | ||
78 | return res.sendStatus(HttpStatusCode.NOT_FOUND_404) | ||
79 | } | ||
80 | |||
81 | return res.fail({ | ||
82 | status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, | ||
83 | message: err.message, | ||
84 | type: err.name | ||
85 | }) | ||
86 | } | ||
diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 33c429eb1..6ef9154b9 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts | |||
@@ -1,30 +1,63 @@ | |||
1 | import cors from 'cors' | 1 | import cors from 'cors' |
2 | import express from 'express' | 2 | import express from 'express' |
3 | import { handleStaticError } from '@server/middlewares' | 3 | import { |
4 | asyncMiddleware, | ||
5 | ensureCanAccessPrivateVideoHLSFiles, | ||
6 | ensureCanAccessVideoPrivateWebTorrentFiles, | ||
7 | handleStaticError, | ||
8 | optionalAuthenticate | ||
9 | } from '@server/middlewares' | ||
4 | import { CONFIG } from '../initializers/config' | 10 | import { CONFIG } from '../initializers/config' |
5 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' | 11 | import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' |
6 | 12 | ||
7 | const staticRouter = express.Router() | 13 | const staticRouter = express.Router() |
8 | 14 | ||
9 | // Cors is very important to let other servers access torrent and video files | 15 | // Cors is very important to let other servers access torrent and video files |
10 | staticRouter.use(cors()) | 16 | staticRouter.use(cors()) |
11 | 17 | ||
12 | // Videos path for webseed | 18 | // --------------------------------------------------------------------------- |
19 | // WebTorrent/Classic videos | ||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | const privateWebTorrentStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true | ||
23 | ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles) ] | ||
24 | : [] | ||
25 | |||
26 | staticRouter.use( | ||
27 | STATIC_PATHS.PRIVATE_WEBSEED, | ||
28 | ...privateWebTorrentStaticMiddlewares, | ||
29 | express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), | ||
30 | handleStaticError | ||
31 | ) | ||
13 | staticRouter.use( | 32 | staticRouter.use( |
14 | STATIC_PATHS.WEBSEED, | 33 | STATIC_PATHS.WEBSEED, |
15 | express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }), | 34 | express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }), |
16 | handleStaticError | 35 | handleStaticError |
17 | ) | 36 | ) |
37 | |||
18 | staticRouter.use( | 38 | staticRouter.use( |
19 | STATIC_PATHS.REDUNDANCY, | 39 | STATIC_PATHS.REDUNDANCY, |
20 | express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }), | 40 | express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }), |
21 | handleStaticError | 41 | handleStaticError |
22 | ) | 42 | ) |
23 | 43 | ||
44 | // --------------------------------------------------------------------------- | ||
24 | // HLS | 45 | // HLS |
46 | // --------------------------------------------------------------------------- | ||
47 | |||
48 | const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true | ||
49 | ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] | ||
50 | : [] | ||
51 | |||
52 | staticRouter.use( | ||
53 | STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, | ||
54 | ...privateHLSStaticMiddlewares, | ||
55 | express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), | ||
56 | handleStaticError | ||
57 | ) | ||
25 | staticRouter.use( | 58 | staticRouter.use( |
26 | STATIC_PATHS.STREAMING_PLAYLISTS.HLS, | 59 | STATIC_PATHS.STREAMING_PLAYLISTS.HLS, |
27 | express.static(HLS_STREAMING_PLAYLIST_DIRECTORY, { fallthrough: false }), | 60 | express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }), |
28 | handleStaticError | 61 | handleStaticError |
29 | ) | 62 | ) |
30 | 63 | ||
diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts index f467bd629..ce5883571 100644 --- a/server/controllers/well-known.ts +++ b/server/controllers/well-known.ts | |||
@@ -5,6 +5,7 @@ import { root } from '@shared/core-utils' | |||
5 | import { CONFIG } from '../initializers/config' | 5 | import { CONFIG } from '../initializers/config' |
6 | import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' | 6 | import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' |
7 | import { cacheRoute } from '../middlewares/cache/cache' | 7 | import { cacheRoute } from '../middlewares/cache/cache' |
8 | import { handleStaticError } from '@server/middlewares' | ||
8 | 9 | ||
9 | const wellKnownRouter = express.Router() | 10 | const wellKnownRouter = express.Router() |
10 | 11 | ||
@@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta', | |||
69 | } | 70 | } |
70 | ) | 71 | ) |
71 | 72 | ||
73 | wellKnownRouter.use('/.well-known/', | ||
74 | cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN), | ||
75 | express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }), | ||
76 | handleStaticError | ||
77 | ) | ||
78 | |||
72 | // --------------------------------------------------------------------------- | 79 | // --------------------------------------------------------------------------- |
73 | 80 | ||
74 | export { | 81 | export { |
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index c762f6a29..73bd994c1 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -6,7 +6,7 @@ | |||
6 | */ | 6 | */ |
7 | 7 | ||
8 | import { exec, ExecOptions } from 'child_process' | 8 | import { exec, ExecOptions } from 'child_process' |
9 | import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions } from 'crypto' | 9 | import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto' |
10 | import { truncate } from 'lodash' | 10 | import { truncate } from 'lodash' |
11 | import { pipeline } from 'stream' | 11 | import { pipeline } from 'stream' |
12 | import { URL } from 'url' | 12 | import { URL } from 'url' |
@@ -311,7 +311,17 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) | |||
311 | } | 311 | } |
312 | } | 312 | } |
313 | 313 | ||
314 | // eslint-disable-next-line max-len | ||
315 | function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> { | ||
316 | return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> { | ||
317 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
318 | func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
319 | }) | ||
320 | } | ||
321 | } | ||
322 | |||
314 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) | 323 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) |
324 | const scryptPromise = promisify3<string, string, number, Buffer>(scrypt) | ||
315 | const execPromise2 = promisify2<string, any, string>(exec) | 325 | const execPromise2 = promisify2<string, any, string>(exec) |
316 | const execPromise = promisify1<string, string>(exec) | 326 | const execPromise = promisify1<string, string>(exec) |
317 | const pipelinePromise = promisify(pipeline) | 327 | const pipelinePromise = promisify(pipeline) |
@@ -339,6 +349,8 @@ export { | |||
339 | promisify1, | 349 | promisify1, |
340 | promisify2, | 350 | promisify2, |
341 | 351 | ||
352 | scryptPromise, | ||
353 | |||
342 | randomBytesPromise, | 354 | randomBytesPromise, |
343 | 355 | ||
344 | generateRSAKeyPairPromise, | 356 | generateRSAKeyPairPromise, |
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 2a2f008b9..97b3577af 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -7,11 +7,11 @@ import { peertubeTruncate } from '../../core-utils' | |||
7 | import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' | 7 | import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' |
8 | import { isLiveLatencyModeValid } from '../video-lives' | 8 | import { isLiveLatencyModeValid } from '../video-lives' |
9 | import { | 9 | import { |
10 | isVideoDescriptionValid, | ||
10 | isVideoDurationValid, | 11 | isVideoDurationValid, |
11 | isVideoNameValid, | 12 | isVideoNameValid, |
12 | isVideoStateValid, | 13 | isVideoStateValid, |
13 | isVideoTagValid, | 14 | isVideoTagValid, |
14 | isVideoTruncatedDescriptionValid, | ||
15 | isVideoViewsValid | 15 | isVideoViewsValid |
16 | } from '../videos' | 16 | } from '../videos' |
17 | import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' | 17 | import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' |
@@ -32,7 +32,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { | |||
32 | logger.debug('Video has invalid urls', { video }) | 32 | logger.debug('Video has invalid urls', { video }) |
33 | return false | 33 | return false |
34 | } | 34 | } |
35 | if (!setRemoteVideoTruncatedContent(video)) { | 35 | if (!setRemoteVideoContent(video)) { |
36 | logger.debug('Video has invalid content', { video }) | 36 | logger.debug('Video has invalid content', { video }) |
37 | return false | 37 | return false |
38 | } | 38 | } |
@@ -168,7 +168,7 @@ function isRemoteStringIdentifierValid (data: any) { | |||
168 | } | 168 | } |
169 | 169 | ||
170 | function isRemoteVideoContentValid (mediaType: string, content: string) { | 170 | function isRemoteVideoContentValid (mediaType: string, content: string) { |
171 | return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) | 171 | return mediaType === 'text/markdown' && isVideoDescriptionValid(content) |
172 | } | 172 | } |
173 | 173 | ||
174 | function setValidRemoteIcon (video: any) { | 174 | function setValidRemoteIcon (video: any) { |
@@ -194,9 +194,9 @@ function setValidRemoteVideoUrls (video: any) { | |||
194 | return true | 194 | return true |
195 | } | 195 | } |
196 | 196 | ||
197 | function setRemoteVideoTruncatedContent (video: any) { | 197 | function setRemoteVideoContent (video: any) { |
198 | if (video.content) { | 198 | if (video.content) { |
199 | video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max }) | 199 | video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max }) |
200 | } | 200 | } |
201 | 201 | ||
202 | return true | 202 | return true |
diff --git a/server/helpers/custom-validators/servers.ts b/server/helpers/custom-validators/servers.ts index b9f45c282..94fda05aa 100644 --- a/server/helpers/custom-validators/servers.ts +++ b/server/helpers/custom-validators/servers.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import validator from 'validator' | 1 | import validator from 'validator' |
2 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | 3 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' |
3 | import { isTestOrDevInstance } from '../core-utils' | ||
4 | import { exists, isArray } from './misc' | 4 | import { exists, isArray } from './misc' |
5 | 5 | ||
6 | function isHostValid (host: string) { | 6 | function isHostValid (host: string) { |
@@ -10,7 +10,7 @@ function isHostValid (host: string) { | |||
10 | } | 10 | } |
11 | 11 | ||
12 | // We validate 'localhost', so we don't have the top level domain | 12 | // We validate 'localhost', so we don't have the top level domain |
13 | if (isTestOrDevInstance()) { | 13 | if (CONFIG.WEBSERVER.HOSTNAME === 'localhost') { |
14 | isURLOptions.require_tld = false | 14 | isURLOptions.require_tld = false |
15 | } | 15 | } |
16 | 16 | ||
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 3ebfe2937..9e8177f77 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts | |||
@@ -45,10 +45,6 @@ function isVideoDurationValid (value: string) { | |||
45 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) | 45 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) |
46 | } | 46 | } |
47 | 47 | ||
48 | function isVideoTruncatedDescriptionValid (value: string) { | ||
49 | return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.TRUNCATED_DESCRIPTION) | ||
50 | } | ||
51 | |||
52 | function isVideoDescriptionValid (value: string) { | 48 | function isVideoDescriptionValid (value: string) { |
53 | return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)) | 49 | return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)) |
54 | } | 50 | } |
@@ -151,7 +147,6 @@ export { | |||
151 | isVideoCategoryValid, | 147 | isVideoCategoryValid, |
152 | isVideoLicenceValid, | 148 | isVideoLicenceValid, |
153 | isVideoLanguageValid, | 149 | isVideoLanguageValid, |
154 | isVideoTruncatedDescriptionValid, | ||
155 | isVideoDescriptionValid, | 150 | isVideoDescriptionValid, |
156 | isVideoFileInfoHashValid, | 151 | isVideoFileInfoHashValid, |
157 | isVideoNameValid, | 152 | isVideoNameValid, |
diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts index 7a81a1313..d84703eb9 100644 --- a/server/helpers/ffmpeg/ffmpeg-vod.ts +++ b/server/helpers/ffmpeg/ffmpeg-vod.ts | |||
@@ -1,14 +1,15 @@ | |||
1 | import { MutexInterface } from 'async-mutex' | ||
1 | import { Job } from 'bullmq' | 2 | import { Job } from 'bullmq' |
2 | import { FfmpegCommand } from 'fluent-ffmpeg' | 3 | import { FfmpegCommand } from 'fluent-ffmpeg' |
3 | import { readFile, writeFile } from 'fs-extra' | 4 | import { readFile, writeFile } from 'fs-extra' |
4 | import { dirname } from 'path' | 5 | import { dirname } from 'path' |
6 | import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
5 | import { pick } from '@shared/core-utils' | 7 | import { pick } from '@shared/core-utils' |
6 | import { AvailableEncoders, VideoResolution } from '@shared/models' | 8 | import { AvailableEncoders, VideoResolution } from '@shared/models' |
7 | import { logger, loggerTagsFactory } from '../logger' | 9 | import { logger, loggerTagsFactory } from '../logger' |
8 | import { getFFmpeg, runCommand } from './ffmpeg-commons' | 10 | import { getFFmpeg, runCommand } from './ffmpeg-commons' |
9 | import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' | 11 | import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' |
10 | import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils' | 12 | import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils' |
11 | import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
12 | 13 | ||
13 | const lTags = loggerTagsFactory('ffmpeg') | 14 | const lTags = loggerTagsFactory('ffmpeg') |
14 | 15 | ||
@@ -22,6 +23,10 @@ interface BaseTranscodeVODOptions { | |||
22 | inputPath: string | 23 | inputPath: string |
23 | outputPath: string | 24 | outputPath: string |
24 | 25 | ||
26 | // Will be released after the ffmpeg started | ||
27 | // To prevent a bug where the input file does not exist anymore when running ffmpeg | ||
28 | inputFileMutexReleaser: MutexInterface.Releaser | ||
29 | |||
25 | availableEncoders: AvailableEncoders | 30 | availableEncoders: AvailableEncoders |
26 | profile: string | 31 | profile: string |
27 | 32 | ||
@@ -94,6 +99,12 @@ async function transcodeVOD (options: TranscodeVODOptions) { | |||
94 | 99 | ||
95 | command = await builders[options.type](command, options) | 100 | command = await builders[options.type](command, options) |
96 | 101 | ||
102 | command.on('start', () => { | ||
103 | setTimeout(() => { | ||
104 | options.inputFileMutexReleaser() | ||
105 | }, 1000) | ||
106 | }) | ||
107 | |||
97 | await runCommand({ command, job: options.job }) | 108 | await runCommand({ command, job: options.job }) |
98 | 109 | ||
99 | await fixHLSPlaylistIfNeeded(options) | 110 | await fixHLSPlaylistIfNeeded(options) |
diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts index 2c6253d44..fb270b3cb 100644 --- a/server/helpers/ffmpeg/ffprobe-utils.ts +++ b/server/helpers/ffmpeg/ffprobe-utils.ts | |||
@@ -15,6 +15,7 @@ import { | |||
15 | import { VideoResolution, VideoTranscodingFPS } from '@shared/models' | 15 | import { VideoResolution, VideoTranscodingFPS } from '@shared/models' |
16 | import { CONFIG } from '../../initializers/config' | 16 | import { CONFIG } from '../../initializers/config' |
17 | import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' | 17 | import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' |
18 | import { toEven } from '../core-utils' | ||
18 | import { logger } from '../logger' | 19 | import { logger } from '../logger' |
19 | 20 | ||
20 | /** | 21 | /** |
@@ -96,8 +97,9 @@ function computeResolutionsToTranscode (options: { | |||
96 | type: 'vod' | 'live' | 97 | type: 'vod' | 'live' |
97 | includeInput: boolean | 98 | includeInput: boolean |
98 | strictLower: boolean | 99 | strictLower: boolean |
100 | hasAudio: boolean | ||
99 | }) { | 101 | }) { |
100 | const { input, type, includeInput, strictLower } = options | 102 | const { input, type, includeInput, strictLower, hasAudio } = options |
101 | 103 | ||
102 | const configResolutions = type === 'vod' | 104 | const configResolutions = type === 'vod' |
103 | ? CONFIG.TRANSCODING.RESOLUTIONS | 105 | ? CONFIG.TRANSCODING.RESOLUTIONS |
@@ -125,12 +127,15 @@ function computeResolutionsToTranscode (options: { | |||
125 | if (input < resolution) continue | 127 | if (input < resolution) continue |
126 | // We only want lower resolutions than input file | 128 | // We only want lower resolutions than input file |
127 | if (strictLower && input === resolution) continue | 129 | if (strictLower && input === resolution) continue |
130 | // Audio resolutio but no audio in the video | ||
131 | if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue | ||
128 | 132 | ||
129 | resolutionsEnabled.add(resolution) | 133 | resolutionsEnabled.add(resolution) |
130 | } | 134 | } |
131 | 135 | ||
132 | if (includeInput) { | 136 | if (includeInput) { |
133 | resolutionsEnabled.add(input) | 137 | // Always use an even resolution to avoid issues with ffmpeg |
138 | resolutionsEnabled.add(toEven(input)) | ||
134 | } | 139 | } |
135 | 140 | ||
136 | return Array.from(resolutionsEnabled) | 141 | return Array.from(resolutionsEnabled) |
diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts new file mode 100644 index 000000000..a32cc9621 --- /dev/null +++ b/server/helpers/otp.ts | |||
@@ -0,0 +1,58 @@ | |||
1 | import { Secret, TOTP } from 'otpauth' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { WEBSERVER } from '@server/initializers/constants' | ||
4 | import { decrypt } from './peertube-crypto' | ||
5 | |||
6 | async function isOTPValid (options: { | ||
7 | encryptedSecret: string | ||
8 | token: string | ||
9 | }) { | ||
10 | const { token, encryptedSecret } = options | ||
11 | |||
12 | const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE) | ||
13 | |||
14 | const totp = new TOTP({ | ||
15 | ...baseOTPOptions(), | ||
16 | |||
17 | secret | ||
18 | }) | ||
19 | |||
20 | const delta = totp.validate({ | ||
21 | token, | ||
22 | window: 1 | ||
23 | }) | ||
24 | |||
25 | if (delta === null) return false | ||
26 | |||
27 | return true | ||
28 | } | ||
29 | |||
30 | function generateOTPSecret (email: string) { | ||
31 | const totp = new TOTP({ | ||
32 | ...baseOTPOptions(), | ||
33 | |||
34 | label: email, | ||
35 | secret: new Secret() | ||
36 | }) | ||
37 | |||
38 | return { | ||
39 | secret: totp.secret.base32, | ||
40 | uri: totp.toString() | ||
41 | } | ||
42 | } | ||
43 | |||
44 | export { | ||
45 | isOTPValid, | ||
46 | generateOTPSecret | ||
47 | } | ||
48 | |||
49 | // --------------------------------------------------------------------------- | ||
50 | |||
51 | function baseOTPOptions () { | ||
52 | return { | ||
53 | issuer: WEBSERVER.HOST, | ||
54 | algorithm: 'SHA1', | ||
55 | digits: 6, | ||
56 | period: 30 | ||
57 | } | ||
58 | } | ||
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 8aca50900..ae7d11800 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts | |||
@@ -1,11 +1,11 @@ | |||
1 | import { compare, genSalt, hash } from 'bcrypt' | 1 | import { compare, genSalt, hash } from 'bcrypt' |
2 | import { createSign, createVerify } from 'crypto' | 2 | import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' |
3 | import { Request } from 'express' | 3 | import { Request } from 'express' |
4 | import { cloneDeep } from 'lodash' | 4 | import { cloneDeep } from 'lodash' |
5 | import { sha256 } from '@shared/extra-utils' | 5 | import { sha256 } from '@shared/extra-utils' |
6 | import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' | 6 | import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' |
7 | import { MActor } from '../types/models' | 7 | import { MActor } from '../types/models' |
8 | import { generateRSAKeyPairPromise, promisify1, promisify2 } from './core-utils' | 8 | import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils' |
9 | import { jsonld } from './custom-jsonld-signature' | 9 | import { jsonld } from './custom-jsonld-signature' |
10 | import { logger } from './logger' | 10 | import { logger } from './logger' |
11 | 11 | ||
@@ -21,9 +21,13 @@ function createPrivateAndPublicKeys () { | |||
21 | return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) | 21 | return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) |
22 | } | 22 | } |
23 | 23 | ||
24 | // --------------------------------------------------------------------------- | ||
24 | // User password checks | 25 | // User password checks |
26 | // --------------------------------------------------------------------------- | ||
25 | 27 | ||
26 | function comparePassword (plainPassword: string, hashPassword: string) { | 28 | function comparePassword (plainPassword: string, hashPassword: string) { |
29 | if (!plainPassword) return Promise.resolve(false) | ||
30 | |||
27 | return bcryptComparePromise(plainPassword, hashPassword) | 31 | return bcryptComparePromise(plainPassword, hashPassword) |
28 | } | 32 | } |
29 | 33 | ||
@@ -33,7 +37,9 @@ async function cryptPassword (password: string) { | |||
33 | return bcryptHashPromise(password, salt) | 37 | return bcryptHashPromise(password, salt) |
34 | } | 38 | } |
35 | 39 | ||
40 | // --------------------------------------------------------------------------- | ||
36 | // HTTP Signature | 41 | // HTTP Signature |
42 | // --------------------------------------------------------------------------- | ||
37 | 43 | ||
38 | function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { | 44 | function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { |
39 | if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { | 45 | if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { |
@@ -62,7 +68,9 @@ function parseHTTPSignature (req: Request, clockSkew?: number) { | |||
62 | return parsed | 68 | return parsed |
63 | } | 69 | } |
64 | 70 | ||
71 | // --------------------------------------------------------------------------- | ||
65 | // JSONLD | 72 | // JSONLD |
73 | // --------------------------------------------------------------------------- | ||
66 | 74 | ||
67 | function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> { | 75 | function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> { |
68 | if (signedDocument.signature.type === 'RsaSignature2017') { | 76 | if (signedDocument.signature.type === 'RsaSignature2017') { |
@@ -112,6 +120,8 @@ async function signJsonLDObject <T> (byActor: MActor, data: T) { | |||
112 | return Object.assign(data, { signature }) | 120 | return Object.assign(data, { signature }) |
113 | } | 121 | } |
114 | 122 | ||
123 | // --------------------------------------------------------------------------- | ||
124 | |||
115 | function buildDigest (body: any) { | 125 | function buildDigest (body: any) { |
116 | const rawBody = typeof body === 'string' ? body : JSON.stringify(body) | 126 | const rawBody = typeof body === 'string' ? body : JSON.stringify(body) |
117 | 127 | ||
@@ -119,6 +129,34 @@ function buildDigest (body: any) { | |||
119 | } | 129 | } |
120 | 130 | ||
121 | // --------------------------------------------------------------------------- | 131 | // --------------------------------------------------------------------------- |
132 | // Encryption | ||
133 | // --------------------------------------------------------------------------- | ||
134 | |||
135 | async function encrypt (str: string, secret: string) { | ||
136 | const iv = await randomBytesPromise(ENCRYPTION.IV) | ||
137 | |||
138 | const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) | ||
139 | const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv) | ||
140 | |||
141 | let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':' | ||
142 | encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING) | ||
143 | encrypted += cipher.final(ENCRYPTION.ENCODING) | ||
144 | |||
145 | return encrypted | ||
146 | } | ||
147 | |||
148 | async function decrypt (encryptedArg: string, secret: string) { | ||
149 | const [ ivStr, encryptedStr ] = encryptedArg.split(':') | ||
150 | |||
151 | const iv = Buffer.from(ivStr, 'hex') | ||
152 | const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) | ||
153 | |||
154 | const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv) | ||
155 | |||
156 | return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8') | ||
157 | } | ||
158 | |||
159 | // --------------------------------------------------------------------------- | ||
122 | 160 | ||
123 | export { | 161 | export { |
124 | isHTTPSignatureDigestValid, | 162 | isHTTPSignatureDigestValid, |
@@ -129,7 +167,10 @@ export { | |||
129 | comparePassword, | 167 | comparePassword, |
130 | createPrivateAndPublicKeys, | 168 | createPrivateAndPublicKeys, |
131 | cryptPassword, | 169 | cryptPassword, |
132 | signJsonLDObject | 170 | signJsonLDObject, |
171 | |||
172 | encrypt, | ||
173 | decrypt | ||
133 | } | 174 | } |
134 | 175 | ||
135 | // --------------------------------------------------------------------------- | 176 | // --------------------------------------------------------------------------- |
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts index 3cb17edd0..f5f476913 100644 --- a/server/helpers/upload.ts +++ b/server/helpers/upload.ts | |||
@@ -1,10 +1,10 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' | 2 | import { DIRECTORIES } from '@server/initializers/constants' |
3 | 3 | ||
4 | function getResumableUploadPath (filename?: string) { | 4 | function getResumableUploadPath (filename?: string) { |
5 | if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename) | 5 | if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename) |
6 | 6 | ||
7 | return RESUMABLE_UPLOAD_DIRECTORY | 7 | return DIRECTORIES.RESUMABLE_UPLOAD |
8 | } | 8 | } |
9 | 9 | ||
10 | // --------------------------------------------------------------------------- | 10 | // --------------------------------------------------------------------------- |
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 88bdb16b6..a3c93e6fe 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { decode, encode } from 'bencode' | 1 | import { decode, encode } from 'bencode' |
2 | import createTorrent from 'create-torrent' | 2 | import createTorrent from 'create-torrent' |
3 | import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra' | 3 | import { createWriteStream, ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra' |
4 | import magnetUtil from 'magnet-uri' | 4 | import magnetUtil from 'magnet-uri' |
5 | import parseTorrent from 'parse-torrent' | 5 | import parseTorrent from 'parse-torrent' |
6 | import { dirname, join } from 'path' | 6 | import { dirname, join } from 'path' |
@@ -134,6 +134,11 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli | |||
134 | 134 | ||
135 | const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) | 135 | const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) |
136 | 136 | ||
137 | if (!await pathExists(oldTorrentPath)) { | ||
138 | logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath) | ||
139 | return | ||
140 | } | ||
141 | |||
137 | const torrentContent = await readFile(oldTorrentPath) | 142 | const torrentContent = await readFile(oldTorrentPath) |
138 | const decoded = decode(torrentContent) | 143 | const decoded = decode(torrentContent) |
139 | 144 | ||
@@ -151,7 +156,7 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli | |||
151 | logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath) | 156 | logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath) |
152 | 157 | ||
153 | await writeFile(newTorrentPath, encode(decoded)) | 158 | await writeFile(newTorrentPath, encode(decoded)) |
154 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) | 159 | await remove(oldTorrentPath) |
155 | 160 | ||
156 | videoFile.torrentFilename = newTorrentFilename | 161 | videoFile.torrentFilename = newTorrentFilename |
157 | videoFile.infoHash = sha1(encode(decoded.info)) | 162 | videoFile.infoHash = sha1(encode(decoded.info)) |
@@ -164,7 +169,10 @@ function generateMagnetUri ( | |||
164 | ) { | 169 | ) { |
165 | const xs = videoFile.getTorrentUrl() | 170 | const xs = videoFile.getTorrentUrl() |
166 | const announce = trackerUrls | 171 | const announce = trackerUrls |
167 | let urlList = [ videoFile.getFileUrl(video) ] | 172 | |
173 | let urlList = video.hasPrivateStaticPath() | ||
174 | ? [] | ||
175 | : [ videoFile.getFileUrl(video) ] | ||
168 | 176 | ||
169 | const redundancies = videoFile.RedundancyVideos | 177 | const redundancies = videoFile.RedundancyVideos |
170 | if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) | 178 | if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) |
@@ -240,6 +248,8 @@ function buildAnnounceList () { | |||
240 | } | 248 | } |
241 | 249 | ||
242 | function buildUrlList (video: MVideo, videoFile: MVideoFile) { | 250 | function buildUrlList (video: MVideo, videoFile: MVideoFile) { |
251 | if (video.hasPrivateStaticPath()) return [] | ||
252 | |||
243 | return [ videoFile.getFileUrl(video) ] | 253 | return [ videoFile.getFileUrl(video) ] |
244 | } | 254 | } |
245 | 255 | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts index fc4c40787..a2f630953 100644 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/helpers/youtube-dl/youtube-dl-cli.ts | |||
@@ -128,14 +128,14 @@ export class YoutubeDLCLI { | |||
128 | const data = await this.run({ url, args: completeArgs, processOptions }) | 128 | const data = await this.run({ url, args: completeArgs, processOptions }) |
129 | if (!data) return undefined | 129 | if (!data) return undefined |
130 | 130 | ||
131 | const info = data.map(this.parseInfo) | 131 | const info = data.map(d => JSON.parse(d)) |
132 | 132 | ||
133 | return info.length === 1 | 133 | return info.length === 1 |
134 | ? info[0] | 134 | ? info[0] |
135 | : info | 135 | : info |
136 | } | 136 | } |
137 | 137 | ||
138 | getListInfo (options: { | 138 | async getListInfo (options: { |
139 | url: string | 139 | url: string |
140 | latestVideosCount?: number | 140 | latestVideosCount?: number |
141 | processOptions: execa.NodeOptions | 141 | processOptions: execa.NodeOptions |
@@ -151,12 +151,17 @@ export class YoutubeDLCLI { | |||
151 | additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) | 151 | additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) |
152 | } | 152 | } |
153 | 153 | ||
154 | return this.getInfo({ | 154 | const result = await this.getInfo({ |
155 | url: options.url, | 155 | url: options.url, |
156 | format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), | 156 | format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), |
157 | processOptions: options.processOptions, | 157 | processOptions: options.processOptions, |
158 | additionalYoutubeDLArgs | 158 | additionalYoutubeDLArgs |
159 | }) | 159 | }) |
160 | |||
161 | if (!result) return result | ||
162 | if (!Array.isArray(result)) return [ result ] | ||
163 | |||
164 | return result | ||
160 | } | 165 | } |
161 | 166 | ||
162 | async getSubs (options: { | 167 | async getSubs (options: { |
@@ -241,8 +246,4 @@ export class YoutubeDLCLI { | |||
241 | 246 | ||
242 | return args | 247 | return args |
243 | } | 248 | } |
244 | |||
245 | private parseInfo (data: string) { | ||
246 | return JSON.parse(data) | ||
247 | } | ||
248 | } | 249 | } |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index 42839d1c9..09e878eee 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -42,6 +42,7 @@ function checkConfig () { | |||
42 | logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.') | 42 | logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.') |
43 | } | 43 | } |
44 | 44 | ||
45 | checkSecretsConfig() | ||
45 | checkEmailConfig() | 46 | checkEmailConfig() |
46 | checkNSFWPolicyConfig() | 47 | checkNSFWPolicyConfig() |
47 | checkLocalRedundancyConfig() | 48 | checkLocalRedundancyConfig() |
@@ -103,6 +104,12 @@ export { | |||
103 | 104 | ||
104 | // --------------------------------------------------------------------------- | 105 | // --------------------------------------------------------------------------- |
105 | 106 | ||
107 | function checkSecretsConfig () { | ||
108 | if (!CONFIG.SECRETS.PEERTUBE) { | ||
109 | throw new Error('secrets.peertube is missing in config. Generate one using `openssl rand -hex 32`') | ||
110 | } | ||
111 | } | ||
112 | |||
106 | function checkEmailConfig () { | 113 | function checkEmailConfig () { |
107 | if (!isEmailEnabled()) { | 114 | if (!isEmailEnabled()) { |
108 | if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { | 115 | if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { |
@@ -271,6 +278,14 @@ function checkObjectStorageConfig () { | |||
271 | 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' | 278 | 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' |
272 | ) | 279 | ) |
273 | } | 280 | } |
281 | |||
282 | if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC) { | ||
283 | throw new Error('object_storage.upload_acl.public must be set') | ||
284 | } | ||
285 | |||
286 | if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE) { | ||
287 | throw new Error('object_storage.upload_acl.private must be set') | ||
288 | } | ||
274 | } | 289 | } |
275 | } | 290 | } |
276 | 291 | ||
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 3188903be..42be7ee6e 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -11,12 +11,13 @@ const config: IConfig = require('config') | |||
11 | function checkMissedConfig () { | 11 | function checkMissedConfig () { |
12 | const required = [ 'listen.port', 'listen.hostname', | 12 | const required = [ 'listen.port', 'listen.hostname', |
13 | 'webserver.https', 'webserver.hostname', 'webserver.port', | 13 | 'webserver.https', 'webserver.hostname', 'webserver.port', |
14 | 'secrets.peertube', | ||
14 | 'trust_proxy', | 15 | 'trust_proxy', |
15 | 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max', | 16 | 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max', |
16 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', | 17 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', |
17 | 'email.body.signature', 'email.subject.prefix', | 18 | 'email.body.signature', 'email.subject.prefix', |
18 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', | 19 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', |
19 | 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', | 20 | 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known', |
20 | 'log.level', | 21 | 'log.level', |
21 | 'user.video_quota', 'user.video_quota_daily', | 22 | 'user.video_quota', 'user.video_quota_daily', |
22 | 'video_channels.max_per_user', | 23 | 'video_channels.max_per_user', |
@@ -34,6 +35,7 @@ function checkMissedConfig () { | |||
34 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', | 35 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', |
35 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', | 36 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', |
36 | 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', | 37 | 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', |
38 | 'import.video_channel_synchronization.full_sync_videos_limit', | ||
37 | 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', | 39 | 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', |
38 | 'client.videos.miniature.display_author_avatar', | 40 | 'client.videos.miniature.display_author_avatar', |
39 | 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', | 41 | 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', |
@@ -45,6 +47,12 @@ function checkMissedConfig () { | |||
45 | 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', | 47 | 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', |
46 | 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', | 48 | 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', |
47 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', | 49 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', |
50 | 'static_files.private_files_require_auth', | ||
51 | 'object_storage.enabled', 'object_storage.endpoint', 'object_storage.region', 'object_storage.upload_acl.public', | ||
52 | 'object_storage.upload_acl.private', 'object_storage.proxy.proxify_private_files', 'object_storage.credentials.access_key_id', | ||
53 | 'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name', | ||
54 | 'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.videos.bucket_name', | ||
55 | 'object_storage.videos.prefix', 'object_storage.videos.base_url', | ||
48 | 'theme.default', | 56 | 'theme.default', |
49 | 'feeds.videos.count', 'feeds.comments.count', | 57 | 'feeds.videos.count', 'feeds.comments.count', |
50 | 'geo_ip.enabled', 'geo_ip.country.database_url', | 58 | 'geo_ip.enabled', 'geo_ip.country.database_url', |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 2c92bea22..3dd1f6971 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -20,6 +20,9 @@ const CONFIG = { | |||
20 | PORT: config.get<number>('listen.port'), | 20 | PORT: config.get<number>('listen.port'), |
21 | HOSTNAME: config.get<string>('listen.hostname') | 21 | HOSTNAME: config.get<string>('listen.hostname') |
22 | }, | 22 | }, |
23 | SECRETS: { | ||
24 | PEERTUBE: config.get<string>('secrets.peertube') | ||
25 | }, | ||
23 | DATABASE: { | 26 | DATABASE: { |
24 | DBNAME: config.has('database.name') ? config.get<string>('database.name') : 'peertube' + config.get<string>('database.suffix'), | 27 | DBNAME: config.has('database.name') ? config.get<string>('database.name') : 'peertube' + config.get<string>('database.suffix'), |
25 | HOSTNAME: config.get<string>('database.hostname'), | 28 | HOSTNAME: config.get<string>('database.hostname'), |
@@ -107,18 +110,28 @@ const CONFIG = { | |||
107 | TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')), | 110 | TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')), |
108 | CACHE_DIR: buildPath(config.get<string>('storage.cache')), | 111 | CACHE_DIR: buildPath(config.get<string>('storage.cache')), |
109 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), | 112 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), |
110 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) | 113 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')), |
114 | WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known')) | ||
115 | }, | ||
116 | STATIC_FILES: { | ||
117 | PRIVATE_FILES_REQUIRE_AUTH: config.get<boolean>('static_files.private_files_require_auth') | ||
111 | }, | 118 | }, |
112 | OBJECT_STORAGE: { | 119 | OBJECT_STORAGE: { |
113 | ENABLED: config.get<boolean>('object_storage.enabled'), | 120 | ENABLED: config.get<boolean>('object_storage.enabled'), |
114 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), | 121 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), |
115 | ENDPOINT: config.get<string>('object_storage.endpoint'), | 122 | ENDPOINT: config.get<string>('object_storage.endpoint'), |
116 | REGION: config.get<string>('object_storage.region'), | 123 | REGION: config.get<string>('object_storage.region'), |
117 | UPLOAD_ACL: config.get<string>('object_storage.upload_acl'), | 124 | UPLOAD_ACL: { |
125 | PUBLIC: config.get<string>('object_storage.upload_acl.public'), | ||
126 | PRIVATE: config.get<string>('object_storage.upload_acl.private') | ||
127 | }, | ||
118 | CREDENTIALS: { | 128 | CREDENTIALS: { |
119 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), | 129 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), |
120 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') | 130 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') |
121 | }, | 131 | }, |
132 | PROXY: { | ||
133 | PROXIFY_PRIVATE_FILES: config.get<boolean>('object_storage.proxy.proxify_private_files') | ||
134 | }, | ||
122 | VIDEOS: { | 135 | VIDEOS: { |
123 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), | 136 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), |
124 | PREFIX: config.get<string>('object_storage.videos.prefix'), | 137 | PREFIX: config.get<string>('object_storage.videos.prefix'), |
@@ -405,6 +418,9 @@ const CONFIG = { | |||
405 | get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) }, | 418 | get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) }, |
406 | get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { | 419 | get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { |
407 | return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization') | 420 | return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization') |
421 | }, | ||
422 | get FULL_SYNC_VIDEOS_LIMIT () { | ||
423 | return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit') | ||
408 | } | 424 | } |
409 | } | 425 | } |
410 | }, | 426 | }, |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 7039ab457..66eb31230 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { RepeatOptions } from 'bullmq' | 1 | import { RepeatOptions } from 'bullmq' |
2 | import { randomBytes } from 'crypto' | 2 | import { Encoding, randomBytes } from 'crypto' |
3 | import { invert } from 'lodash' | 3 | import { invert } from 'lodash' |
4 | import { join } from 'path' | 4 | import { join } from 'path' |
5 | import { randomInt, root } from '@shared/core-utils' | 5 | import { randomInt, root } from '@shared/core-utils' |
@@ -25,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
25 | 25 | ||
26 | // --------------------------------------------------------------------------- | 26 | // --------------------------------------------------------------------------- |
27 | 27 | ||
28 | const LAST_MIGRATION_VERSION = 740 | 28 | const LAST_MIGRATION_VERSION = 745 |
29 | 29 | ||
30 | // --------------------------------------------------------------------------- | 30 | // --------------------------------------------------------------------------- |
31 | 31 | ||
@@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = { | |||
116 | ACTIVITY_PUB: { | 116 | ACTIVITY_PUB: { |
117 | VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example | 117 | VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example |
118 | }, | 118 | }, |
119 | STATS: '4 hours' | 119 | STATS: '4 hours', |
120 | WELL_KNOWN: '1 day' | ||
120 | } | 121 | } |
121 | 122 | ||
122 | // --------------------------------------------------------------------------- | 123 | // --------------------------------------------------------------------------- |
@@ -636,9 +637,18 @@ let PRIVATE_RSA_KEY_SIZE = 2048 | |||
636 | // Password encryption | 637 | // Password encryption |
637 | const BCRYPT_SALT_SIZE = 10 | 638 | const BCRYPT_SALT_SIZE = 10 |
638 | 639 | ||
640 | const ENCRYPTION = { | ||
641 | ALGORITHM: 'aes-256-cbc', | ||
642 | IV: 16, | ||
643 | SALT: 'peertube', | ||
644 | ENCODING: 'hex' as Encoding | ||
645 | } | ||
646 | |||
639 | const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes | 647 | const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes |
640 | const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days | 648 | const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days |
641 | 649 | ||
650 | const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes | ||
651 | |||
642 | const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes | 652 | const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes |
643 | 653 | ||
644 | const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { | 654 | const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { |
@@ -652,10 +662,15 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { | |||
652 | // Express static paths (router) | 662 | // Express static paths (router) |
653 | const STATIC_PATHS = { | 663 | const STATIC_PATHS = { |
654 | THUMBNAILS: '/static/thumbnails/', | 664 | THUMBNAILS: '/static/thumbnails/', |
665 | |||
655 | WEBSEED: '/static/webseed/', | 666 | WEBSEED: '/static/webseed/', |
667 | PRIVATE_WEBSEED: '/static/webseed/private/', | ||
668 | |||
656 | REDUNDANCY: '/static/redundancy/', | 669 | REDUNDANCY: '/static/redundancy/', |
670 | |||
657 | STREAMING_PLAYLISTS: { | 671 | STREAMING_PLAYLISTS: { |
658 | HLS: '/static/streaming-playlists/hls' | 672 | HLS: '/static/streaming-playlists/hls', |
673 | PRIVATE_HLS: '/static/streaming-playlists/hls/private/' | ||
659 | } | 674 | } |
660 | } | 675 | } |
661 | const STATIC_DOWNLOAD_PATHS = { | 676 | const STATIC_DOWNLOAD_PATHS = { |
@@ -670,6 +685,13 @@ const LAZY_STATIC_PATHS = { | |||
670 | VIDEO_CAPTIONS: '/lazy-static/video-captions/', | 685 | VIDEO_CAPTIONS: '/lazy-static/video-captions/', |
671 | TORRENTS: '/lazy-static/torrents/' | 686 | TORRENTS: '/lazy-static/torrents/' |
672 | } | 687 | } |
688 | const OBJECT_STORAGE_PROXY_PATHS = { | ||
689 | PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/', | ||
690 | |||
691 | STREAMING_PLAYLISTS: { | ||
692 | PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/' | ||
693 | } | ||
694 | } | ||
673 | 695 | ||
674 | // Cache control | 696 | // Cache control |
675 | const STATIC_MAX_AGE = { | 697 | const STATIC_MAX_AGE = { |
@@ -735,12 +757,32 @@ const LRU_CACHE = { | |||
735 | }, | 757 | }, |
736 | ACTOR_IMAGE_STATIC: { | 758 | ACTOR_IMAGE_STATIC: { |
737 | MAX_SIZE: 500 | 759 | MAX_SIZE: 500 |
760 | }, | ||
761 | STATIC_VIDEO_FILES_RIGHTS_CHECK: { | ||
762 | MAX_SIZE: 5000, | ||
763 | TTL: parseDurationToMs('10 seconds') | ||
764 | }, | ||
765 | VIDEO_TOKENS: { | ||
766 | MAX_SIZE: 100_000, | ||
767 | TTL: parseDurationToMs('8 hours') | ||
738 | } | 768 | } |
739 | } | 769 | } |
740 | 770 | ||
741 | const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads') | 771 | const DIRECTORIES = { |
742 | const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') | 772 | RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'), |
743 | const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') | 773 | |
774 | HLS_STREAMING_PLAYLIST: { | ||
775 | PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'), | ||
776 | PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private') | ||
777 | }, | ||
778 | |||
779 | VIDEOS: { | ||
780 | PUBLIC: CONFIG.STORAGE.VIDEOS_DIR, | ||
781 | PRIVATE: join(CONFIG.STORAGE.VIDEOS_DIR, 'private') | ||
782 | }, | ||
783 | |||
784 | HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') | ||
785 | } | ||
744 | 786 | ||
745 | const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS | 787 | const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS |
746 | 788 | ||
@@ -804,6 +846,10 @@ const REDUNDANCY = { | |||
804 | } | 846 | } |
805 | 847 | ||
806 | const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) | 848 | const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) |
849 | const OTP = { | ||
850 | HEADER_NAME: 'x-peertube-otp', | ||
851 | HEADER_REQUIRED_VALUE: 'required; app' | ||
852 | } | ||
807 | 853 | ||
808 | const ASSETS_PATH = { | 854 | const ASSETS_PATH = { |
809 | DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'), | 855 | DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'), |
@@ -952,13 +998,14 @@ const VIDEO_FILTERS = { | |||
952 | export { | 998 | export { |
953 | WEBSERVER, | 999 | WEBSERVER, |
954 | API_VERSION, | 1000 | API_VERSION, |
1001 | ENCRYPTION, | ||
955 | VIDEO_LIVE, | 1002 | VIDEO_LIVE, |
956 | PEERTUBE_VERSION, | 1003 | PEERTUBE_VERSION, |
957 | LAZY_STATIC_PATHS, | 1004 | LAZY_STATIC_PATHS, |
1005 | OBJECT_STORAGE_PROXY_PATHS, | ||
958 | SEARCH_INDEX, | 1006 | SEARCH_INDEX, |
959 | RESUMABLE_UPLOAD_DIRECTORY, | 1007 | DIRECTORIES, |
960 | RESUMABLE_UPLOAD_SESSION_LIFETIME, | 1008 | RESUMABLE_UPLOAD_SESSION_LIFETIME, |
961 | HLS_REDUNDANCY_DIRECTORY, | ||
962 | P2P_MEDIA_LOADER_PEER_VERSION, | 1009 | P2P_MEDIA_LOADER_PEER_VERSION, |
963 | ACTOR_IMAGES_SIZE, | 1010 | ACTOR_IMAGES_SIZE, |
964 | ACCEPT_HEADERS, | 1011 | ACCEPT_HEADERS, |
@@ -985,13 +1032,13 @@ export { | |||
985 | FOLLOW_STATES, | 1032 | FOLLOW_STATES, |
986 | DEFAULT_USER_THEME_NAME, | 1033 | DEFAULT_USER_THEME_NAME, |
987 | SERVER_ACTOR_NAME, | 1034 | SERVER_ACTOR_NAME, |
1035 | TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, | ||
988 | PLUGIN_GLOBAL_CSS_FILE_NAME, | 1036 | PLUGIN_GLOBAL_CSS_FILE_NAME, |
989 | PLUGIN_GLOBAL_CSS_PATH, | 1037 | PLUGIN_GLOBAL_CSS_PATH, |
990 | PRIVATE_RSA_KEY_SIZE, | 1038 | PRIVATE_RSA_KEY_SIZE, |
991 | VIDEO_FILTERS, | 1039 | VIDEO_FILTERS, |
992 | ROUTE_CACHE_LIFETIME, | 1040 | ROUTE_CACHE_LIFETIME, |
993 | SORTABLE_COLUMNS, | 1041 | SORTABLE_COLUMNS, |
994 | HLS_STREAMING_PLAYLIST_DIRECTORY, | ||
995 | JOB_TTL, | 1042 | JOB_TTL, |
996 | DEFAULT_THEME_NAME, | 1043 | DEFAULT_THEME_NAME, |
997 | NSFW_POLICY_TYPES, | 1044 | NSFW_POLICY_TYPES, |
@@ -1040,6 +1087,7 @@ export { | |||
1040 | PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME, | 1087 | PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME, |
1041 | ASSETS_PATH, | 1088 | ASSETS_PATH, |
1042 | FILES_CONTENT_HASH, | 1089 | FILES_CONTENT_HASH, |
1090 | OTP, | ||
1043 | loadLanguages, | 1091 | loadLanguages, |
1044 | buildLanguages, | 1092 | buildLanguages, |
1045 | generateContentHash | 1093 | generateContentHash |
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index b02be9567..f5d8eedf1 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts | |||
@@ -10,7 +10,7 @@ import { ApplicationModel } from '../models/application/application' | |||
10 | import { OAuthClientModel } from '../models/oauth/oauth-client' | 10 | import { OAuthClientModel } from '../models/oauth/oauth-client' |
11 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' | 11 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' |
12 | import { CONFIG } from './config' | 12 | import { CONFIG } from './config' |
13 | import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' | 13 | import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants' |
14 | import { sequelizeTypescript } from './database' | 14 | import { sequelizeTypescript } from './database' |
15 | 15 | ||
16 | async function installApplication () { | 16 | async function installApplication () { |
@@ -92,11 +92,13 @@ function createDirectoriesIfNotExist () { | |||
92 | tasks.push(ensureDir(dir)) | 92 | tasks.push(ensureDir(dir)) |
93 | } | 93 | } |
94 | 94 | ||
95 | // Playlist directories | 95 | tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)) |
96 | tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY)) | 96 | tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC)) |
97 | tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC)) | ||
98 | tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE)) | ||
97 | 99 | ||
98 | // Resumable upload directory | 100 | // Resumable upload directory |
99 | tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY)) | 101 | tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD)) |
100 | 102 | ||
101 | return Promise.all(tasks) | 103 | return Promise.all(tasks) |
102 | } | 104 | } |
diff --git a/server/initializers/migrations/0745-user-otp.ts b/server/initializers/migrations/0745-user-otp.ts new file mode 100644 index 000000000..157308ea1 --- /dev/null +++ b/server/initializers/migrations/0745-user-otp.ts | |||
@@ -0,0 +1,29 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction | ||
5 | queryInterface: Sequelize.QueryInterface | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | db: any | ||
8 | }): Promise<void> { | ||
9 | const { transaction } = utils | ||
10 | |||
11 | const data = { | ||
12 | type: Sequelize.STRING, | ||
13 | defaultValue: null, | ||
14 | allowNull: true | ||
15 | } | ||
16 | await utils.queryInterface.addColumn('user', 'otpSecret', data, { transaction }) | ||
17 | |||
18 | } | ||
19 | |||
20 | async function down (utils: { | ||
21 | queryInterface: Sequelize.QueryInterface | ||
22 | transaction: Sequelize.Transaction | ||
23 | }) { | ||
24 | } | ||
25 | |||
26 | export { | ||
27 | up, | ||
28 | down | ||
29 | } | ||
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 76ed37aae..1e6e8956c 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -109,8 +109,10 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc | |||
109 | let video: MVideoAccountLightBlacklistAllFiles | 109 | let video: MVideoAccountLightBlacklistAllFiles |
110 | let created: boolean | 110 | let created: boolean |
111 | let comment: MCommentOwnerVideo | 111 | let comment: MCommentOwnerVideo |
112 | |||
112 | try { | 113 | try { |
113 | const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) | 114 | const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) |
115 | if (!resolveThreadResult) return // Comment not accepted | ||
114 | 116 | ||
115 | video = resolveThreadResult.video | 117 | video = resolveThreadResult.video |
116 | created = resolveThreadResult.commentCreated | 118 | created = resolveThreadResult.commentCreated |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 911c7cd30..b65baf0e9 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -4,7 +4,9 @@ import { logger } from '../../helpers/logger' | |||
4 | import { doJSONRequest } from '../../helpers/requests' | 4 | import { doJSONRequest } from '../../helpers/requests' |
5 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' | 5 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' | 7 | import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' |
8 | import { isRemoteVideoCommentAccepted } from '../moderation' | ||
9 | import { Hooks } from '../plugins/hooks' | ||
8 | import { getOrCreateAPActor } from './actors' | 10 | import { getOrCreateAPActor } from './actors' |
9 | import { checkUrlsSameHost } from './url' | 11 | import { checkUrlsSameHost } from './url' |
10 | import { getOrCreateAPVideo } from './videos' | 12 | import { getOrCreateAPVideo } from './videos' |
@@ -103,6 +105,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { | |||
103 | firstReply.changed('updatedAt', true) | 105 | firstReply.changed('updatedAt', true) |
104 | firstReply.Video = video | 106 | firstReply.Video = video |
105 | 107 | ||
108 | if (await isRemoteCommentAccepted(firstReply) !== true) { | ||
109 | return undefined | ||
110 | } | ||
111 | |||
106 | comments[comments.length - 1] = await firstReply.save() | 112 | comments[comments.length - 1] = await firstReply.save() |
107 | 113 | ||
108 | for (let i = comments.length - 2; i >= 0; i--) { | 114 | for (let i = comments.length - 2; i >= 0; i--) { |
@@ -113,6 +119,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { | |||
113 | comment.changed('updatedAt', true) | 119 | comment.changed('updatedAt', true) |
114 | comment.Video = video | 120 | comment.Video = video |
115 | 121 | ||
122 | if (await isRemoteCommentAccepted(comment) !== true) { | ||
123 | return undefined | ||
124 | } | ||
125 | |||
116 | comments[i] = await comment.save() | 126 | comments[i] = await comment.save() |
117 | } | 127 | } |
118 | 128 | ||
@@ -169,3 +179,26 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { | |||
169 | commentCreated: true | 179 | commentCreated: true |
170 | }) | 180 | }) |
171 | } | 181 | } |
182 | |||
183 | async function isRemoteCommentAccepted (comment: MComment) { | ||
184 | // Already created | ||
185 | if (comment.id) return true | ||
186 | |||
187 | const acceptParameters = { | ||
188 | comment | ||
189 | } | ||
190 | |||
191 | const acceptedResult = await Hooks.wrapFun( | ||
192 | isRemoteVideoCommentAccepted, | ||
193 | acceptParameters, | ||
194 | 'filter:activity-pub.remote-video-comment.create.accept.result' | ||
195 | ) | ||
196 | |||
197 | if (!acceptedResult || acceptedResult.accepted !== true) { | ||
198 | logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters }) | ||
199 | |||
200 | return false | ||
201 | } | ||
202 | |||
203 | return true | ||
204 | } | ||
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts index fa1887315..bc0d4301f 100644 --- a/server/lib/auth/oauth.ts +++ b/server/lib/auth/oauth.ts | |||
@@ -9,11 +9,23 @@ import OAuth2Server, { | |||
9 | UnsupportedGrantTypeError | 9 | UnsupportedGrantTypeError |
10 | } from '@node-oauth/oauth2-server' | 10 | } from '@node-oauth/oauth2-server' |
11 | import { randomBytesPromise } from '@server/helpers/core-utils' | 11 | import { randomBytesPromise } from '@server/helpers/core-utils' |
12 | import { isOTPValid } from '@server/helpers/otp' | ||
12 | import { MOAuthClient } from '@server/types/models' | 13 | import { MOAuthClient } from '@server/types/models' |
13 | import { sha1 } from '@shared/extra-utils' | 14 | import { sha1 } from '@shared/extra-utils' |
14 | import { OAUTH_LIFETIME } from '../../initializers/constants' | 15 | import { HttpStatusCode } from '@shared/models' |
16 | import { OAUTH_LIFETIME, OTP } from '../../initializers/constants' | ||
15 | import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' | 17 | import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' |
16 | 18 | ||
19 | class MissingTwoFactorError extends Error { | ||
20 | code = HttpStatusCode.UNAUTHORIZED_401 | ||
21 | name = 'missing_two_factor' | ||
22 | } | ||
23 | |||
24 | class InvalidTwoFactorError extends Error { | ||
25 | code = HttpStatusCode.BAD_REQUEST_400 | ||
26 | name = 'invalid_two_factor' | ||
27 | } | ||
28 | |||
17 | /** | 29 | /** |
18 | * | 30 | * |
19 | * Reimplement some functions of OAuth2Server to inject external auth methods | 31 | * Reimplement some functions of OAuth2Server to inject external auth methods |
@@ -83,17 +95,15 @@ async function handleOAuthToken (req: express.Request, options: { refreshTokenAu | |||
83 | 95 | ||
84 | function handleOAuthAuthenticate ( | 96 | function handleOAuthAuthenticate ( |
85 | req: express.Request, | 97 | req: express.Request, |
86 | res: express.Response, | 98 | res: express.Response |
87 | authenticateInQuery = false | ||
88 | ) { | 99 | ) { |
89 | const options = authenticateInQuery | 100 | return oAuthServer.authenticate(new Request(req), new Response(res)) |
90 | ? { allowBearerTokensInQueryString: true } | ||
91 | : {} | ||
92 | |||
93 | return oAuthServer.authenticate(new Request(req), new Response(res), options) | ||
94 | } | 101 | } |
95 | 102 | ||
96 | export { | 103 | export { |
104 | MissingTwoFactorError, | ||
105 | InvalidTwoFactorError, | ||
106 | |||
97 | handleOAuthToken, | 107 | handleOAuthToken, |
98 | handleOAuthAuthenticate | 108 | handleOAuthAuthenticate |
99 | } | 109 | } |
@@ -118,6 +128,16 @@ async function handlePasswordGrant (options: { | |||
118 | const user = await getUser(request.body.username, request.body.password, bypassLogin) | 128 | const user = await getUser(request.body.username, request.body.password, bypassLogin) |
119 | if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid') | 129 | if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid') |
120 | 130 | ||
131 | if (user.otpSecret) { | ||
132 | if (!request.headers[OTP.HEADER_NAME]) { | ||
133 | throw new MissingTwoFactorError('Missing two factor header') | ||
134 | } | ||
135 | |||
136 | if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) { | ||
137 | throw new InvalidTwoFactorError('Invalid two factor header') | ||
138 | } | ||
139 | } | ||
140 | |||
121 | const token = await buildToken() | 141 | const token = await buildToken() |
122 | 142 | ||
123 | return saveToken(token, client, user, { bypassLogin }) | 143 | return saveToken(token, client, user, { bypassLogin }) |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index a0a5afc0f..a41f1ae48 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -15,7 +15,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers | |||
15 | import { sequelizeTypescript } from '../initializers/database' | 15 | import { sequelizeTypescript } from '../initializers/database' |
16 | import { VideoFileModel } from '../models/video/video-file' | 16 | import { VideoFileModel } from '../models/video/video-file' |
17 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 17 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
18 | import { storeHLSFile } from './object-storage' | 18 | import { storeHLSFileFromFilename } from './object-storage' |
19 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' | 19 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' |
20 | import { VideoPathManager } from './video-path-manager' | 20 | import { VideoPathManager } from './video-path-manager' |
21 | 21 | ||
@@ -95,7 +95,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist | |||
95 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 95 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') |
96 | 96 | ||
97 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 97 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
98 | playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) | 98 | playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) |
99 | await remove(masterPlaylistPath) | 99 | await remove(masterPlaylistPath) |
100 | } | 100 | } |
101 | 101 | ||
@@ -146,7 +146,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist | |||
146 | await outputJSON(outputPath, json) | 146 | await outputJSON(outputPath, json) |
147 | 147 | ||
148 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 148 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
149 | playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) | 149 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) |
150 | await remove(outputPath) | 150 | await remove(outputPath) |
151 | } | 151 | } |
152 | 152 | ||
diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts index 03aa414c9..cef93afda 100644 --- a/server/lib/job-queue/handlers/manage-video-torrent.ts +++ b/server/lib/job-queue/handlers/manage-video-torrent.ts | |||
@@ -1,5 +1,7 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { extractVideo } from '@server/helpers/video' | ||
2 | import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent' | 3 | import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent' |
4 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
3 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
4 | import { VideoFileModel } from '@server/models/video/video-file' | 6 | import { VideoFileModel } from '@server/models/video/video-file' |
5 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 7 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
@@ -30,17 +32,23 @@ async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'c | |||
30 | 32 | ||
31 | if (!video || !file) return | 33 | if (!video || !file) return |
32 | 34 | ||
33 | await createTorrentAndSetInfoHash(video, file) | 35 | const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
34 | 36 | ||
35 | // Refresh videoFile because the createTorrentAndSetInfoHash could be long | 37 | try { |
36 | const refreshedFile = await VideoFileModel.loadWithVideo(file.id) | 38 | await createTorrentAndSetInfoHash(video, file) |
37 | // File does not exist anymore, remove the generated torrent | ||
38 | if (!refreshedFile) return file.removeTorrent() | ||
39 | 39 | ||
40 | refreshedFile.infoHash = file.infoHash | 40 | // Refresh videoFile because the createTorrentAndSetInfoHash could be long |
41 | refreshedFile.torrentFilename = file.torrentFilename | 41 | const refreshedFile = await VideoFileModel.loadWithVideo(file.id) |
42 | // File does not exist anymore, remove the generated torrent | ||
43 | if (!refreshedFile) return file.removeTorrent() | ||
42 | 44 | ||
43 | return refreshedFile.save() | 45 | refreshedFile.infoHash = file.infoHash |
46 | refreshedFile.torrentFilename = file.torrentFilename | ||
47 | |||
48 | await refreshedFile.save() | ||
49 | } finally { | ||
50 | fileMutexReleaser() | ||
51 | } | ||
44 | } | 52 | } |
45 | 53 | ||
46 | async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { | 54 | async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { |
@@ -52,9 +60,16 @@ async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { ac | |||
52 | 60 | ||
53 | if ((!video && !streamingPlaylist) || !file) return | 61 | if ((!video && !streamingPlaylist) || !file) return |
54 | 62 | ||
55 | await updateTorrentMetadata(video || streamingPlaylist, file) | 63 | const extractedVideo = extractVideo(video || streamingPlaylist) |
64 | const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid) | ||
56 | 65 | ||
57 | await file.save() | 66 | try { |
67 | await updateTorrentMetadata(video || streamingPlaylist, file) | ||
68 | |||
69 | await file.save() | ||
70 | } finally { | ||
71 | fileMutexReleaser() | ||
72 | } | ||
58 | } | 73 | } |
59 | 74 | ||
60 | async function loadVideoOrLog (videoId: number) { | 75 | async function loadVideoOrLog (videoId: number) { |
@@ -82,7 +97,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) { | |||
82 | async function loadFileOrLog (videoFileId: number) { | 97 | async function loadFileOrLog (videoFileId: number) { |
83 | if (!videoFileId) return undefined | 98 | if (!videoFileId) return undefined |
84 | 99 | ||
85 | const file = await VideoFileModel.loadWithVideo(videoFileId) | 100 | const file = await VideoFileModel.load(videoFileId) |
86 | 101 | ||
87 | if (!file) { | 102 | if (!file) { |
88 | logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) | 103 | logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) |
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index 25bdebeea..a1530cc57 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -3,10 +3,10 @@ import { remove } from 'fs-extra' | |||
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' | 5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' |
6 | import { CONFIG } from '@server/initializers/config' | ||
7 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' | 6 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' |
8 | import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' | 7 | import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' |
9 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | 8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' | 10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' |
11 | import { VideoModel } from '@server/models/video/video' | 11 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 12 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
@@ -28,6 +28,8 @@ export async function processMoveToObjectStorage (job: Job) { | |||
28 | 28 | ||
29 | const lTags = lTagsBase(video.uuid, video.url) | 29 | const lTags = lTagsBase(video.uuid, video.url) |
30 | 30 | ||
31 | const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
32 | |||
31 | try { | 33 | try { |
32 | if (video.VideoFiles) { | 34 | if (video.VideoFiles) { |
33 | logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags) | 35 | logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags) |
@@ -49,6 +51,10 @@ export async function processMoveToObjectStorage (job: Job) { | |||
49 | } | 51 | } |
50 | } catch (err) { | 52 | } catch (err) { |
51 | await onMoveToObjectStorageFailure(job, err) | 53 | await onMoveToObjectStorageFailure(job, err) |
54 | |||
55 | throw err | ||
56 | } finally { | ||
57 | fileMutexReleaser() | ||
52 | } | 58 | } |
53 | 59 | ||
54 | return payload.videoUUID | 60 | return payload.videoUUID |
@@ -72,9 +78,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | |||
72 | for (const file of video.VideoFiles) { | 78 | for (const file of video.VideoFiles) { |
73 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | 79 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue |
74 | 80 | ||
75 | const fileUrl = await storeWebTorrentFile(file.filename) | 81 | const fileUrl = await storeWebTorrentFile(video, file) |
76 | 82 | ||
77 | const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) | 83 | const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) |
78 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | 84 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) |
79 | } | 85 | } |
80 | } | 86 | } |
@@ -88,10 +94,10 @@ async function moveHLSFiles (video: MVideoWithAllFiles) { | |||
88 | 94 | ||
89 | // Resolution playlist | 95 | // Resolution playlist |
90 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | 96 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) |
91 | await storeHLSFile(playlistWithVideo, playlistFilename) | 97 | await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) |
92 | 98 | ||
93 | // Resolution fragmented file | 99 | // Resolution fragmented file |
94 | const fileUrl = await storeHLSFile(playlistWithVideo, file.filename) | 100 | const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) |
95 | 101 | ||
96 | const oldPath = join(getHLSDirectory(video), file.filename) | 102 | const oldPath = join(getHLSDirectory(video), file.filename) |
97 | 103 | ||
@@ -113,9 +119,9 @@ async function doAfterLastJob (options: { | |||
113 | const playlistWithVideo = playlist.withVideo(video) | 119 | const playlistWithVideo = playlist.withVideo(video) |
114 | 120 | ||
115 | // Master playlist | 121 | // Master playlist |
116 | playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename) | 122 | playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) |
117 | // Sha256 segments file | 123 | // Sha256 segments file |
118 | playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename) | 124 | playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) |
119 | 125 | ||
120 | playlist.storage = VideoStorage.OBJECT_STORAGE | 126 | playlist.storage = VideoStorage.OBJECT_STORAGE |
121 | 127 | ||
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts index 600292844..035f88e96 100644 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ b/server/lib/job-queue/handlers/video-channel-import.ts | |||
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel' | |||
5 | import { VideoChannelModel } from '@server/models/video/video-channel' | 5 | import { VideoChannelModel } from '@server/models/video/video-channel' |
6 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | 6 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' |
7 | import { MChannelSync } from '@server/types/models' | 7 | import { MChannelSync } from '@server/types/models' |
8 | import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' | 8 | import { VideoChannelImportPayload } from '@shared/models' |
9 | 9 | ||
10 | export async function processVideoChannelImport (job: Job) { | 10 | export async function processVideoChannelImport (job: Job) { |
11 | const payload = job.data as VideoChannelImportPayload | 11 | const payload = job.data as VideoChannelImportPayload |
@@ -32,17 +32,12 @@ export async function processVideoChannelImport (job: Job) { | |||
32 | 32 | ||
33 | const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) | 33 | const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) |
34 | 34 | ||
35 | try { | 35 | logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) |
36 | logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) | 36 | |
37 | 37 | await synchronizeChannel({ | |
38 | await synchronizeChannel({ | 38 | channel: videoChannel, |
39 | channel: videoChannel, | 39 | externalChannelUrl: payload.externalChannelUrl, |
40 | externalChannelUrl: payload.externalChannelUrl, | 40 | channelSync, |
41 | channelSync | 41 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT |
42 | }) | 42 | }) |
43 | } catch (err) { | ||
44 | logger.error(`Failed to import channel ${videoChannel.name}`, { err }) | ||
45 | channelSync.state = VideoChannelSyncState.FAILED | ||
46 | await channelSync.save() | ||
47 | } | ||
48 | } | 43 | } |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 9901b878c..83d582cb4 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -12,7 +12,8 @@ import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@serv | |||
12 | import { VideoPathManager } from '@server/lib/video-path-manager' | 12 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { buildNextVideoState } from '@server/lib/video-state' | 13 | import { buildNextVideoState } from '@server/lib/video-state' |
14 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 14 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
15 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | 15 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' |
16 | import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | ||
16 | import { getLowercaseExtension } from '@shared/core-utils' | 17 | import { getLowercaseExtension } from '@shared/core-utils' |
17 | import { isAudioFile } from '@shared/extra-utils' | 18 | import { isAudioFile } from '@shared/extra-utils' |
18 | import { | 19 | import { |
@@ -36,7 +37,6 @@ import { sequelizeTypescript } from '../../../initializers/database' | |||
36 | import { VideoModel } from '../../../models/video/video' | 37 | import { VideoModel } from '../../../models/video/video' |
37 | import { VideoFileModel } from '../../../models/video/video-file' | 38 | import { VideoFileModel } from '../../../models/video/video-file' |
38 | import { VideoImportModel } from '../../../models/video/video-import' | 39 | import { VideoImportModel } from '../../../models/video/video-import' |
39 | import { MThumbnail } from '../../../types/models/video/thumbnail' | ||
40 | import { federateVideoIfNeeded } from '../../activitypub/videos' | 40 | import { federateVideoIfNeeded } from '../../activitypub/videos' |
41 | import { Notifier } from '../../notifier' | 41 | import { Notifier } from '../../notifier' |
42 | import { generateVideoMiniature } from '../../thumbnail' | 42 | import { generateVideoMiniature } from '../../thumbnail' |
@@ -178,125 +178,159 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
178 | } | 178 | } |
179 | 179 | ||
180 | // Video is accepted, resuming preparation | 180 | // Video is accepted, resuming preparation |
181 | const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) | 181 | const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid) |
182 | // To clean files if the import fails | ||
183 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) | ||
184 | |||
185 | // Move file | ||
186 | const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) | ||
187 | await move(tempVideoPath, videoDestFile) | ||
188 | tempVideoPath = null // This path is not used anymore | ||
189 | |||
190 | // Generate miniature if the import did not created it | ||
191 | let thumbnailModel: MThumbnail | ||
192 | let thumbnailSave: object | ||
193 | if (!videoImportWithFiles.Video.getMiniature()) { | ||
194 | thumbnailModel = await generateVideoMiniature({ | ||
195 | video: videoImportWithFiles.Video, | ||
196 | videoFile, | ||
197 | type: ThumbnailType.MINIATURE | ||
198 | }) | ||
199 | thumbnailSave = thumbnailModel.toJSON() | ||
200 | } | ||
201 | 182 | ||
202 | // Generate preview if the import did not created it | 183 | try { |
203 | let previewModel: MThumbnail | 184 | const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile) |
204 | let previewSave: object | ||
205 | if (!videoImportWithFiles.Video.getPreview()) { | ||
206 | previewModel = await generateVideoMiniature({ | ||
207 | video: videoImportWithFiles.Video, | ||
208 | videoFile, | ||
209 | type: ThumbnailType.PREVIEW | ||
210 | }) | ||
211 | previewSave = previewModel.toJSON() | ||
212 | } | ||
213 | 185 | ||
214 | // Create torrent | 186 | // Move file |
215 | await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) | 187 | const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) |
188 | await move(tempVideoPath, videoDestFile) | ||
216 | 189 | ||
217 | const videoFileSave = videoFile.toJSON() | 190 | tempVideoPath = null // This path is not used anymore |
218 | 191 | ||
219 | const { videoImportUpdated, video } = await retryTransactionWrapper(() => { | 192 | let { |
220 | return sequelizeTypescript.transaction(async t => { | 193 | miniatureModel: thumbnailModel, |
221 | const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo | 194 | miniatureJSONSave: thumbnailSave |
195 | } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE) | ||
222 | 196 | ||
223 | // Refresh video | 197 | let { |
224 | const video = await VideoModel.load(videoImportToUpdate.videoId, t) | 198 | miniatureModel: previewModel, |
225 | if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.') | 199 | miniatureJSONSave: previewSave |
200 | } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW) | ||
226 | 201 | ||
227 | const videoFileCreated = await videoFile.save({ transaction: t }) | 202 | // Create torrent |
203 | await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) | ||
228 | 204 | ||
229 | // Update video DB object | 205 | const videoFileSave = videoFile.toJSON() |
230 | video.duration = duration | ||
231 | video.state = buildNextVideoState(video.state) | ||
232 | await video.save({ transaction: t }) | ||
233 | 206 | ||
234 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) | 207 | const { videoImportUpdated, video } = await retryTransactionWrapper(() => { |
235 | if (previewModel) await video.addAndSaveThumbnail(previewModel, t) | 208 | return sequelizeTypescript.transaction(async t => { |
209 | // Refresh video | ||
210 | const video = await VideoModel.load(videoImportWithFiles.videoId, t) | ||
211 | if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.') | ||
236 | 212 | ||
237 | // Now we can federate the video (reload from database, we need more attributes) | 213 | await videoFile.save({ transaction: t }) |
238 | const videoForFederation = await VideoModel.loadFull(video.uuid, t) | ||
239 | await federateVideoIfNeeded(videoForFederation, true, t) | ||
240 | 214 | ||
241 | // Update video import object | 215 | // Update video DB object |
242 | videoImportToUpdate.state = VideoImportState.SUCCESS | 216 | video.duration = duration |
243 | const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo | 217 | video.state = buildNextVideoState(video.state) |
244 | videoImportUpdated.Video = video | 218 | await video.save({ transaction: t }) |
245 | 219 | ||
246 | videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] }) | 220 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) |
221 | if (previewModel) await video.addAndSaveThumbnail(previewModel, t) | ||
247 | 222 | ||
248 | logger.info('Video %s imported.', video.uuid) | 223 | // Now we can federate the video (reload from database, we need more attributes) |
224 | const videoForFederation = await VideoModel.loadFull(video.uuid, t) | ||
225 | await federateVideoIfNeeded(videoForFederation, true, t) | ||
249 | 226 | ||
250 | return { videoImportUpdated, video: videoForFederation } | 227 | // Update video import object |
251 | }).catch(err => { | 228 | videoImportWithFiles.state = VideoImportState.SUCCESS |
252 | // Reset fields | 229 | const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport |
253 | if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave) | ||
254 | if (previewModel) previewModel = new ThumbnailModel(previewSave) | ||
255 | 230 | ||
256 | videoFile = new VideoFileModel(videoFileSave) | 231 | logger.info('Video %s imported.', video.uuid) |
257 | 232 | ||
258 | throw err | 233 | return { videoImportUpdated, video: videoForFederation } |
259 | }) | 234 | }).catch(err => { |
260 | }) | 235 | // Reset fields |
236 | if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave) | ||
237 | if (previewModel) previewModel = new ThumbnailModel(previewSave) | ||
261 | 238 | ||
262 | Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: videoImportUpdated, success: true }) | 239 | videoFile = new VideoFileModel(videoFileSave) |
263 | 240 | ||
264 | if (video.isBlacklisted()) { | 241 | throw err |
265 | const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) | 242 | }) |
243 | }) | ||
266 | 244 | ||
267 | Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) | 245 | await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User }) |
268 | } else { | 246 | } finally { |
269 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 247 | videoFileLockReleaser() |
270 | } | 248 | } |
249 | } catch (err) { | ||
250 | await onImportError(err, tempVideoPath, videoImport) | ||
271 | 251 | ||
272 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | 252 | throw err |
273 | await JobQueue.Instance.createJob( | 253 | } |
274 | await buildMoveToObjectStorageJob({ video: videoImportUpdated.Video, previousVideoState: VideoState.TO_IMPORT }) | 254 | } |
275 | ) | ||
276 | } | ||
277 | 255 | ||
278 | // Create transcoding jobs? | 256 | async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> { |
279 | if (video.state === VideoState.TO_TRANSCODE) { | 257 | // Refresh video, privacy may have changed |
280 | await JobQueue.Instance.createJob( | 258 | const video = await videoImport.Video.reload() |
281 | await buildOptimizeOrMergeAudioJob({ video: videoImportUpdated.Video, videoFile, user: videoImport.User }) | 259 | const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) |
282 | ) | ||
283 | } | ||
284 | 260 | ||
285 | } catch (err) { | 261 | return Object.assign(videoImport, { Video: videoWithFiles }) |
286 | try { | 262 | } |
287 | if (tempVideoPath) await remove(tempVideoPath) | ||
288 | } catch (errUnlink) { | ||
289 | logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) | ||
290 | } | ||
291 | 263 | ||
292 | videoImport.error = err.message | 264 | async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) { |
293 | if (videoImport.state !== VideoImportState.REJECTED) { | 265 | // Generate miniature if the import did not created it |
294 | videoImport.state = VideoImportState.FAILED | 266 | const needsMiniature = thumbnailType === ThumbnailType.MINIATURE |
267 | ? !videoImportWithFiles.Video.getMiniature() | ||
268 | : !videoImportWithFiles.Video.getPreview() | ||
269 | |||
270 | if (!needsMiniature) { | ||
271 | return { | ||
272 | miniatureModel: null, | ||
273 | miniatureJSONSave: null | ||
295 | } | 274 | } |
296 | await videoImport.save() | 275 | } |
297 | 276 | ||
298 | Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) | 277 | const miniatureModel = await generateVideoMiniature({ |
278 | video: videoImportWithFiles.Video, | ||
279 | videoFile, | ||
280 | type: thumbnailType | ||
281 | }) | ||
282 | const miniatureJSONSave = miniatureModel.toJSON() | ||
299 | 283 | ||
300 | throw err | 284 | return { |
285 | miniatureModel, | ||
286 | miniatureJSONSave | ||
287 | } | ||
288 | } | ||
289 | |||
290 | async function afterImportSuccess (options: { | ||
291 | videoImport: MVideoImport | ||
292 | video: MVideoFullLight | ||
293 | videoFile: MVideoFile | ||
294 | user: MUserId | ||
295 | }) { | ||
296 | const { video, videoFile, videoImport, user } = options | ||
297 | |||
298 | Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true }) | ||
299 | |||
300 | if (video.isBlacklisted()) { | ||
301 | const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) | ||
302 | |||
303 | Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) | ||
304 | } else { | ||
305 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
301 | } | 306 | } |
307 | |||
308 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
309 | await JobQueue.Instance.createJob( | ||
310 | await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) | ||
311 | ) | ||
312 | return | ||
313 | } | ||
314 | |||
315 | if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? | ||
316 | await JobQueue.Instance.createJob( | ||
317 | await buildOptimizeOrMergeAudioJob({ video, videoFile, user }) | ||
318 | ) | ||
319 | } | ||
320 | } | ||
321 | |||
322 | async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) { | ||
323 | try { | ||
324 | if (tempVideoPath) await remove(tempVideoPath) | ||
325 | } catch (errUnlink) { | ||
326 | logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) | ||
327 | } | ||
328 | |||
329 | videoImport.error = err.message | ||
330 | if (videoImport.state !== VideoImportState.REJECTED) { | ||
331 | videoImport.state = VideoImportState.FAILED | ||
332 | } | ||
333 | await videoImport.save() | ||
334 | |||
335 | Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) | ||
302 | } | 336 | } |
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 8a3ee09a2..c6263f55a 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -4,7 +4,7 @@ import { join } from 'path' | |||
4 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' | 4 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' |
5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | 6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' |
7 | import { cleanupPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' | 7 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' |
8 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' | 8 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' |
9 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 9 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
10 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' | 10 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' |
@@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin | |||
18 | import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' | 18 | import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' |
19 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' | 19 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' |
20 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 20 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
21 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
21 | 22 | ||
22 | const lTags = loggerTagsFactory('live', 'job') | 23 | const lTags = loggerTagsFactory('live', 'job') |
23 | 24 | ||
@@ -34,13 +35,13 @@ async function processVideoLiveEnding (job: Job) { | |||
34 | const live = await VideoLiveModel.loadByVideoId(payload.videoId) | 35 | const live = await VideoLiveModel.loadByVideoId(payload.videoId) |
35 | const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) | 36 | const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) |
36 | 37 | ||
37 | const permanentLive = live.permanentLive | ||
38 | |||
39 | if (!video || !live || !liveSession) { | 38 | if (!video || !live || !liveSession) { |
40 | logError() | 39 | logError() |
41 | return | 40 | return |
42 | } | 41 | } |
43 | 42 | ||
43 | const permanentLive = live.permanentLive | ||
44 | |||
44 | liveSession.endingProcessed = true | 45 | liveSession.endingProcessed = true |
45 | await liveSession.save() | 46 | await liveSession.save() |
46 | 47 | ||
@@ -141,23 +142,22 @@ async function replaceLiveByReplay (options: { | |||
141 | }) { | 142 | }) { |
142 | const { video, liveSession, live, permanentLive, replayDirectory } = options | 143 | const { video, liveSession, live, permanentLive, replayDirectory } = options |
143 | 144 | ||
144 | await cleanupTMPLiveFiles(video) | 145 | const videoWithFiles = await VideoModel.loadFull(video.id) |
146 | const hlsPlaylist = videoWithFiles.getHLSPlaylist() | ||
147 | |||
148 | await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) | ||
145 | 149 | ||
146 | await live.destroy() | 150 | await live.destroy() |
147 | 151 | ||
148 | video.isLive = false | 152 | videoWithFiles.isLive = false |
149 | video.waitTranscoding = true | 153 | videoWithFiles.waitTranscoding = true |
150 | video.state = VideoState.TO_TRANSCODE | 154 | videoWithFiles.state = VideoState.TO_TRANSCODE |
151 | 155 | ||
152 | await video.save() | 156 | await videoWithFiles.save() |
153 | 157 | ||
154 | liveSession.replayVideoId = video.id | 158 | liveSession.replayVideoId = videoWithFiles.id |
155 | await liveSession.save() | 159 | await liveSession.save() |
156 | 160 | ||
157 | // Remove old HLS playlist video files | ||
158 | const videoWithFiles = await VideoModel.loadFull(video.id) | ||
159 | |||
160 | const hlsPlaylist = videoWithFiles.getHLSPlaylist() | ||
161 | await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) | 161 | await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) |
162 | 162 | ||
163 | // Reset playlist | 163 | // Reset playlist |
@@ -206,18 +206,27 @@ async function assignReplayFilesToVideo (options: { | |||
206 | const concatenatedTsFiles = await readdir(replayDirectory) | 206 | const concatenatedTsFiles = await readdir(replayDirectory) |
207 | 207 | ||
208 | for (const concatenatedTsFile of concatenatedTsFiles) { | 208 | for (const concatenatedTsFile of concatenatedTsFiles) { |
209 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
210 | |||
209 | const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) | 211 | const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) |
210 | 212 | ||
211 | const probe = await ffprobePromise(concatenatedTsFilePath) | 213 | const probe = await ffprobePromise(concatenatedTsFilePath) |
212 | const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) | 214 | const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) |
213 | const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) | 215 | const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) |
214 | 216 | ||
215 | await generateHlsPlaylistResolutionFromTS({ | 217 | try { |
216 | video, | 218 | await generateHlsPlaylistResolutionFromTS({ |
217 | concatenatedTsFilePath, | 219 | video, |
218 | resolution, | 220 | inputFileMutexReleaser, |
219 | isAAC: audioStream?.codec_name === 'aac' | 221 | concatenatedTsFilePath, |
220 | }) | 222 | resolution, |
223 | isAAC: audioStream?.codec_name === 'aac' | ||
224 | }) | ||
225 | } catch (err) { | ||
226 | logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) | ||
227 | } | ||
228 | |||
229 | inputFileMutexReleaser() | ||
221 | } | 230 | } |
222 | 231 | ||
223 | return video | 232 | return video |
@@ -234,7 +243,7 @@ async function cleanupLiveAndFederate (options: { | |||
234 | 243 | ||
235 | if (streamingPlaylist) { | 244 | if (streamingPlaylist) { |
236 | if (permanentLive) { | 245 | if (permanentLive) { |
237 | await cleanupPermanentLive(video, streamingPlaylist) | 246 | await cleanupAndDestroyPermanentLive(video, streamingPlaylist) |
238 | } else { | 247 | } else { |
239 | await cleanupUnsavedNormalLive(video, streamingPlaylist) | 248 | await cleanupUnsavedNormalLive(video, streamingPlaylist) |
240 | } | 249 | } |
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index b0e92acf7..3e6d23363 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -13,7 +13,6 @@ import { | |||
13 | MergeAudioTranscodingPayload, | 13 | MergeAudioTranscodingPayload, |
14 | NewWebTorrentResolutionTranscodingPayload, | 14 | NewWebTorrentResolutionTranscodingPayload, |
15 | OptimizeTranscodingPayload, | 15 | OptimizeTranscodingPayload, |
16 | VideoResolution, | ||
17 | VideoTranscodingPayload | 16 | VideoTranscodingPayload |
18 | } from '@shared/models' | 17 | } from '@shared/models' |
19 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 18 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
@@ -94,15 +93,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV | |||
94 | 93 | ||
95 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | 94 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() |
96 | 95 | ||
97 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { | 96 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
98 | return generateHlsPlaylistResolution({ | 97 | |
99 | video, | 98 | try { |
100 | videoInputPath, | 99 | await videoFileInput.getVideo().reload() |
101 | resolution: payload.resolution, | 100 | |
102 | copyCodecs: payload.copyCodecs, | 101 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { |
103 | job | 102 | return generateHlsPlaylistResolution({ |
103 | video, | ||
104 | videoInputPath, | ||
105 | inputFileMutexReleaser, | ||
106 | resolution: payload.resolution, | ||
107 | copyCodecs: payload.copyCodecs, | ||
108 | job | ||
109 | }) | ||
104 | }) | 110 | }) |
105 | }) | 111 | } finally { |
112 | inputFileMutexReleaser() | ||
113 | } | ||
106 | 114 | ||
107 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) | 115 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) |
108 | 116 | ||
@@ -177,38 +185,44 @@ async function onVideoFirstWebTorrentTranscoding ( | |||
177 | transcodeType: TranscodeVODOptionsType, | 185 | transcodeType: TranscodeVODOptionsType, |
178 | user: MUserId | 186 | user: MUserId |
179 | ) { | 187 | ) { |
180 | const { resolution, audioStream } = await videoArg.probeMaxQualityFile() | 188 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) |
181 | 189 | ||
182 | // Maybe the video changed in database, refresh it | 190 | try { |
183 | const videoDatabase = await VideoModel.loadFull(videoArg.uuid) | 191 | // Maybe the video changed in database, refresh it |
184 | // Video does not exist anymore | 192 | const videoDatabase = await VideoModel.loadFull(videoArg.uuid) |
185 | if (!videoDatabase) return undefined | 193 | // Video does not exist anymore |
186 | 194 | if (!videoDatabase) return undefined | |
187 | // Generate HLS version of the original file | 195 | |
188 | const originalFileHLSPayload = { | 196 | const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile() |
189 | ...payload, | 197 | |
190 | 198 | // Generate HLS version of the original file | |
191 | hasAudio: !!audioStream, | 199 | const originalFileHLSPayload = { |
192 | resolution: videoDatabase.getMaxQualityFile().resolution, | 200 | ...payload, |
193 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | 201 | |
194 | copyCodecs: transcodeType !== 'quick-transcode', | 202 | hasAudio: !!audioStream, |
195 | isMaxQuality: true | 203 | resolution: videoDatabase.getMaxQualityFile().resolution, |
196 | } | 204 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues |
197 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | 205 | copyCodecs: transcodeType !== 'quick-transcode', |
198 | const hasNewResolutions = await createLowerResolutionsJobs({ | 206 | isMaxQuality: true |
199 | video: videoDatabase, | 207 | } |
200 | user, | 208 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) |
201 | videoFileResolution: resolution, | 209 | const hasNewResolutions = await createLowerResolutionsJobs({ |
202 | hasAudio: !!audioStream, | 210 | video: videoDatabase, |
203 | type: 'webtorrent', | 211 | user, |
204 | isNewVideo: payload.isNewVideo ?? true | 212 | videoFileResolution: resolution, |
205 | }) | 213 | hasAudio: !!audioStream, |
206 | 214 | type: 'webtorrent', | |
207 | await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') | 215 | isNewVideo: payload.isNewVideo ?? true |
208 | 216 | }) | |
209 | // Move to next state if there are no other resolutions to generate | 217 | |
210 | if (!hasHls && !hasNewResolutions) { | 218 | await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') |
211 | await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) | 219 | |
220 | // Move to next state if there are no other resolutions to generate | ||
221 | if (!hasHls && !hasNewResolutions) { | ||
222 | await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) | ||
223 | } | ||
224 | } finally { | ||
225 | mutexReleaser() | ||
212 | } | 226 | } |
213 | } | 227 | } |
214 | 228 | ||
@@ -266,7 +280,7 @@ async function createLowerResolutionsJobs (options: { | |||
266 | 280 | ||
267 | // Create transcoding jobs if there are enabled resolutions | 281 | // Create transcoding jobs if there are enabled resolutions |
268 | const resolutionsEnabled = await Hooks.wrapObject( | 282 | const resolutionsEnabled = await Hooks.wrapObject( |
269 | computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true }), | 283 | computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), |
270 | 'filter:transcoding.auto.resolutions-to-transcode.result', | 284 | 'filter:transcoding.auto.resolutions-to-transcode.result', |
271 | options | 285 | options |
272 | ) | 286 | ) |
@@ -274,8 +288,6 @@ async function createLowerResolutionsJobs (options: { | |||
274 | const resolutionCreated: string[] = [] | 288 | const resolutionCreated: string[] = [] |
275 | 289 | ||
276 | for (const resolution of resolutionsEnabled) { | 290 | for (const resolution of resolutionsEnabled) { |
277 | if (resolution === VideoResolution.H_NOVIDEO && hasAudio === false) continue | ||
278 | |||
279 | let dataInput: VideoTranscodingPayload | 291 | let dataInput: VideoTranscodingPayload |
280 | 292 | ||
281 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') { | 293 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') { |
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 16715862b..5e459f3c3 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts | |||
@@ -21,14 +21,14 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | |||
21 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 21 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
22 | import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' | 22 | import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' |
23 | import { pick, wait } from '@shared/core-utils' | 23 | import { pick, wait } from '@shared/core-utils' |
24 | import { LiveVideoError, VideoState, VideoStreamingPlaylistType } from '@shared/models' | 24 | import { LiveVideoError, VideoState, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' |
25 | import { federateVideoIfNeeded } from '../activitypub/videos' | 25 | import { federateVideoIfNeeded } from '../activitypub/videos' |
26 | import { JobQueue } from '../job-queue' | 26 | import { JobQueue } from '../job-queue' |
27 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' | 27 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' |
28 | import { PeerTubeSocket } from '../peertube-socket' | 28 | import { PeerTubeSocket } from '../peertube-socket' |
29 | import { Hooks } from '../plugins/hooks' | 29 | import { Hooks } from '../plugins/hooks' |
30 | import { LiveQuotaStore } from './live-quota-store' | 30 | import { LiveQuotaStore } from './live-quota-store' |
31 | import { cleanupPermanentLive } from './live-utils' | 31 | import { cleanupAndDestroyPermanentLive } from './live-utils' |
32 | import { MuxingSession } from './shared' | 32 | import { MuxingSession } from './shared' |
33 | 33 | ||
34 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') | 34 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') |
@@ -224,7 +224,7 @@ class LiveManager { | |||
224 | if (oldStreamingPlaylist) { | 224 | if (oldStreamingPlaylist) { |
225 | if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) | 225 | if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) |
226 | 226 | ||
227 | await cleanupPermanentLive(video, oldStreamingPlaylist) | 227 | await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) |
228 | } | 228 | } |
229 | 229 | ||
230 | this.videoSessions.set(video.id, sessionId) | 230 | this.videoSessions.set(video.id, sessionId) |
@@ -245,7 +245,7 @@ class LiveManager { | |||
245 | ) | 245 | ) |
246 | 246 | ||
247 | const allResolutions = await Hooks.wrapObject( | 247 | const allResolutions = await Hooks.wrapObject( |
248 | this.buildAllResolutionsToTranscode(resolution), | 248 | this.buildAllResolutionsToTranscode(resolution, hasAudio), |
249 | 'filter:transcoding.auto.resolutions-to-transcode.result', | 249 | 'filter:transcoding.auto.resolutions-to-transcode.result', |
250 | { video } | 250 | { video } |
251 | ) | 251 | ) |
@@ -301,7 +301,7 @@ class LiveManager { | |||
301 | ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) | 301 | ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) |
302 | }) | 302 | }) |
303 | 303 | ||
304 | muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) | 304 | muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) |
305 | 305 | ||
306 | muxingSession.on('bad-socket-health', ({ videoId }) => { | 306 | muxingSession.on('bad-socket-health', ({ videoId }) => { |
307 | logger.error( | 307 | logger.error( |
@@ -460,11 +460,11 @@ class LiveManager { | |||
460 | return join(directory, files.sort().reverse()[0]) | 460 | return join(directory, files.sort().reverse()[0]) |
461 | } | 461 | } |
462 | 462 | ||
463 | private buildAllResolutionsToTranscode (originResolution: number) { | 463 | private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) { |
464 | const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | 464 | const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION |
465 | 465 | ||
466 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED | 466 | const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED |
467 | ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false }) | 467 | ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio }) |
468 | : [] | 468 | : [] |
469 | 469 | ||
470 | if (resolutionsEnabled.length === 0) { | 470 | if (resolutionsEnabled.length === 0) { |
@@ -485,6 +485,10 @@ class LiveManager { | |||
485 | 485 | ||
486 | playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) | 486 | playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) |
487 | 487 | ||
488 | playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED | ||
489 | ? VideoStorage.OBJECT_STORAGE | ||
490 | : VideoStorage.FILE_SYSTEM | ||
491 | |||
488 | return playlist.save() | 492 | return playlist.save() |
489 | } | 493 | } |
490 | 494 | ||
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts index 4af6f3ebf..4d03754a9 100644 --- a/server/lib/live/live-segment-sha-store.ts +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -1,61 +1,79 @@ | |||
1 | import { writeJson } from 'fs-extra' | ||
1 | import { basename } from 'path' | 2 | import { basename } from 'path' |
3 | import { mapToJSON } from '@server/helpers/core-utils' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { MStreamingPlaylistVideo } from '@server/types/models' | ||
3 | import { buildSha256Segment } from '../hls' | 6 | import { buildSha256Segment } from '../hls' |
7 | import { storeHLSFileFromPath } from '../object-storage' | ||
8 | import PQueue from 'p-queue' | ||
4 | 9 | ||
5 | const lTags = loggerTagsFactory('live') | 10 | const lTags = loggerTagsFactory('live') |
6 | 11 | ||
7 | class LiveSegmentShaStore { | 12 | class LiveSegmentShaStore { |
8 | 13 | ||
9 | private static instance: LiveSegmentShaStore | 14 | private readonly segmentsSha256 = new Map<string, string>() |
10 | 15 | ||
11 | private readonly segmentsSha256 = new Map<string, Map<string, string>>() | 16 | private readonly videoUUID: string |
12 | 17 | private readonly sha256Path: string | |
13 | private constructor () { | 18 | private readonly streamingPlaylist: MStreamingPlaylistVideo |
14 | } | 19 | private readonly sendToObjectStorage: boolean |
15 | 20 | private readonly writeQueue = new PQueue({ concurrency: 1 }) | |
16 | getSegmentsSha256 (videoUUID: string) { | 21 | |
17 | return this.segmentsSha256.get(videoUUID) | 22 | constructor (options: { |
23 | videoUUID: string | ||
24 | sha256Path: string | ||
25 | streamingPlaylist: MStreamingPlaylistVideo | ||
26 | sendToObjectStorage: boolean | ||
27 | }) { | ||
28 | this.videoUUID = options.videoUUID | ||
29 | this.sha256Path = options.sha256Path | ||
30 | this.streamingPlaylist = options.streamingPlaylist | ||
31 | this.sendToObjectStorage = options.sendToObjectStorage | ||
18 | } | 32 | } |
19 | 33 | ||
20 | async addSegmentSha (videoUUID: string, segmentPath: string) { | 34 | async addSegmentSha (segmentPath: string) { |
21 | const segmentName = basename(segmentPath) | 35 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID)) |
22 | logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID)) | ||
23 | 36 | ||
24 | const shaResult = await buildSha256Segment(segmentPath) | 37 | const shaResult = await buildSha256Segment(segmentPath) |
25 | 38 | ||
26 | if (!this.segmentsSha256.has(videoUUID)) { | 39 | const segmentName = basename(segmentPath) |
27 | this.segmentsSha256.set(videoUUID, new Map()) | 40 | this.segmentsSha256.set(segmentName, shaResult) |
28 | } | ||
29 | 41 | ||
30 | const filesMap = this.segmentsSha256.get(videoUUID) | 42 | try { |
31 | filesMap.set(segmentName, shaResult) | 43 | await this.writeToDisk() |
44 | } catch (err) { | ||
45 | logger.error('Cannot write sha segments to disk.', { err }) | ||
46 | } | ||
32 | } | 47 | } |
33 | 48 | ||
34 | removeSegmentSha (videoUUID: string, segmentPath: string) { | 49 | async removeSegmentSha (segmentPath: string) { |
35 | const segmentName = basename(segmentPath) | 50 | const segmentName = basename(segmentPath) |
36 | 51 | ||
37 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) | 52 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) |
38 | 53 | ||
39 | const filesMap = this.segmentsSha256.get(videoUUID) | 54 | if (!this.segmentsSha256.has(segmentName)) { |
40 | if (!filesMap) { | 55 | logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID)) |
41 | logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID)) | ||
42 | return | 56 | return |
43 | } | 57 | } |
44 | 58 | ||
45 | if (!filesMap.has(segmentName)) { | 59 | this.segmentsSha256.delete(segmentName) |
46 | logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID)) | ||
47 | return | ||
48 | } | ||
49 | 60 | ||
50 | filesMap.delete(segmentName) | 61 | await this.writeToDisk() |
51 | } | 62 | } |
52 | 63 | ||
53 | cleanupShaSegments (videoUUID: string) { | 64 | private writeToDisk () { |
54 | this.segmentsSha256.delete(videoUUID) | 65 | return this.writeQueue.add(async () => { |
55 | } | 66 | await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) |
67 | |||
68 | if (this.sendToObjectStorage) { | ||
69 | const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) | ||
56 | 70 | ||
57 | static get Instance () { | 71 | if (this.streamingPlaylist.segmentsSha256Url !== url) { |
58 | return this.instance || (this.instance = new this()) | 72 | this.streamingPlaylist.segmentsSha256Url = url |
73 | await this.streamingPlaylist.save() | ||
74 | } | ||
75 | } | ||
76 | }) | ||
59 | } | 77 | } |
60 | } | 78 | } |
61 | 79 | ||
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index bba876642..c0dec9829 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts | |||
@@ -1,9 +1,10 @@ | |||
1 | import { pathExists, readdir, remove } from 'fs-extra' | 1 | import { pathExists, readdir, remove } from 'fs-extra' |
2 | import { basename, join } from 'path' | 2 | import { basename, join } from 'path' |
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 4 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' |
5 | import { VideoStorage } from '@shared/models' | ||
6 | import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' | ||
5 | import { getLiveDirectory } from '../paths' | 7 | import { getLiveDirectory } from '../paths' |
6 | import { LiveSegmentShaStore } from './live-segment-sha-store' | ||
7 | 8 | ||
8 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | 9 | function buildConcatenatedName (segmentOrPlaylistPath: string) { |
9 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | 10 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) |
@@ -11,8 +12,8 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) { | |||
11 | return 'concat-' + num[1] + '.ts' | 12 | return 'concat-' + num[1] + '.ts' |
12 | } | 13 | } |
13 | 14 | ||
14 | async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 15 | async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
15 | await cleanupTMPLiveFiles(video) | 16 | await cleanupTMPLiveFiles(video, streamingPlaylist) |
16 | 17 | ||
17 | await streamingPlaylist.destroy() | 18 | await streamingPlaylist.destroy() |
18 | } | 19 | } |
@@ -20,32 +21,51 @@ async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamin | |||
20 | async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 21 | async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
21 | const hlsDirectory = getLiveDirectory(video) | 22 | const hlsDirectory = getLiveDirectory(video) |
22 | 23 | ||
24 | // We uploaded files to object storage too, remove them | ||
25 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
26 | await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) | ||
27 | } | ||
28 | |||
23 | await remove(hlsDirectory) | 29 | await remove(hlsDirectory) |
24 | 30 | ||
25 | await streamingPlaylist.destroy() | 31 | await streamingPlaylist.destroy() |
32 | } | ||
26 | 33 | ||
27 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | 34 | async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
35 | await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video)) | ||
36 | |||
37 | await cleanupTMPLiveFilesFromFilesystem(video) | ||
28 | } | 38 | } |
29 | 39 | ||
30 | async function cleanupTMPLiveFiles (video: MVideo) { | 40 | export { |
31 | const hlsDirectory = getLiveDirectory(video) | 41 | cleanupAndDestroyPermanentLive, |
42 | cleanupUnsavedNormalLive, | ||
43 | cleanupTMPLiveFiles, | ||
44 | buildConcatenatedName | ||
45 | } | ||
46 | |||
47 | // --------------------------------------------------------------------------- | ||
32 | 48 | ||
33 | LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) | 49 | function isTMPLiveFile (name: string) { |
50 | return name.endsWith('.ts') || | ||
51 | name.endsWith('.m3u8') || | ||
52 | name.endsWith('.json') || | ||
53 | name.endsWith('.mpd') || | ||
54 | name.endsWith('.m4s') || | ||
55 | name.endsWith('.tmp') | ||
56 | } | ||
57 | |||
58 | async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { | ||
59 | const hlsDirectory = getLiveDirectory(video) | ||
34 | 60 | ||
35 | if (!await pathExists(hlsDirectory)) return | 61 | if (!await pathExists(hlsDirectory)) return |
36 | 62 | ||
37 | logger.info('Cleanup TMP live files of %s.', hlsDirectory) | 63 | logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory) |
38 | 64 | ||
39 | const files = await readdir(hlsDirectory) | 65 | const files = await readdir(hlsDirectory) |
40 | 66 | ||
41 | for (const filename of files) { | 67 | for (const filename of files) { |
42 | if ( | 68 | if (isTMPLiveFile(filename)) { |
43 | filename.endsWith('.ts') || | ||
44 | filename.endsWith('.m3u8') || | ||
45 | filename.endsWith('.mpd') || | ||
46 | filename.endsWith('.m4s') || | ||
47 | filename.endsWith('.tmp') | ||
48 | ) { | ||
49 | const p = join(hlsDirectory, filename) | 69 | const p = join(hlsDirectory, filename) |
50 | 70 | ||
51 | remove(p) | 71 | remove(p) |
@@ -54,9 +74,16 @@ async function cleanupTMPLiveFiles (video: MVideo) { | |||
54 | } | 74 | } |
55 | } | 75 | } |
56 | 76 | ||
57 | export { | 77 | async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { |
58 | cleanupPermanentLive, | 78 | if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return |
59 | cleanupUnsavedNormalLive, | 79 | |
60 | cleanupTMPLiveFiles, | 80 | logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid) |
61 | buildConcatenatedName | 81 | |
82 | const keys = await listHLSFileKeysOf(streamingPlaylist) | ||
83 | |||
84 | for (const key of keys) { | ||
85 | if (isTMPLiveFile(key)) { | ||
86 | await removeHLSFileObjectStorageByFullKey(key) | ||
87 | } | ||
88 | } | ||
62 | } | 89 | } |
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index 505717dce..6ec126955 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -3,14 +3,17 @@ import { mapSeries } from 'bluebird' | |||
3 | import { FSWatcher, watch } from 'chokidar' | 3 | import { FSWatcher, watch } from 'chokidar' |
4 | import { FfmpegCommand } from 'fluent-ffmpeg' | 4 | import { FfmpegCommand } from 'fluent-ffmpeg' |
5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | 5 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' |
6 | import PQueue from 'p-queue' | ||
6 | import { basename, join } from 'path' | 7 | import { basename, join } from 'path' |
7 | import { EventEmitter } from 'stream' | 8 | import { EventEmitter } from 'stream' |
8 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg' | 9 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg' |
9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | 10 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' |
10 | import { CONFIG } from '@server/initializers/config' | 11 | import { CONFIG } from '@server/initializers/config' |
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | 12 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' |
13 | import { removeHLSFileObjectStorageByPath, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage' | ||
12 | import { VideoFileModel } from '@server/models/video/video-file' | 14 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | 15 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' |
16 | import { VideoStorage } from '@shared/models' | ||
14 | import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' | 17 | import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' |
15 | import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' | 18 | import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' |
16 | import { isAbleToUploadVideo } from '../../user' | 19 | import { isAbleToUploadVideo } from '../../user' |
@@ -19,9 +22,8 @@ import { LiveSegmentShaStore } from '../live-segment-sha-store' | |||
19 | import { buildConcatenatedName } from '../live-utils' | 22 | import { buildConcatenatedName } from '../live-utils' |
20 | 23 | ||
21 | import memoizee = require('memoizee') | 24 | import memoizee = require('memoizee') |
22 | |||
23 | interface MuxingSessionEvents { | 25 | interface MuxingSessionEvents { |
24 | 'master-playlist-created': (options: { videoId: number }) => void | 26 | 'live-ready': (options: { videoId: number }) => void |
25 | 27 | ||
26 | 'bad-socket-health': (options: { videoId: number }) => void | 28 | 'bad-socket-health': (options: { videoId: number }) => void |
27 | 'duration-exceeded': (options: { videoId: number }) => void | 29 | 'duration-exceeded': (options: { videoId: number }) => void |
@@ -68,12 +70,18 @@ class MuxingSession extends EventEmitter { | |||
68 | private readonly outDirectory: string | 70 | private readonly outDirectory: string |
69 | private readonly replayDirectory: string | 71 | private readonly replayDirectory: string |
70 | 72 | ||
73 | private readonly liveSegmentShaStore: LiveSegmentShaStore | ||
74 | |||
71 | private readonly lTags: LoggerTagsFn | 75 | private readonly lTags: LoggerTagsFn |
72 | 76 | ||
73 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} | 77 | private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} |
74 | 78 | ||
75 | private tsWatcher: FSWatcher | 79 | private tsWatcher: FSWatcher |
76 | private masterWatcher: FSWatcher | 80 | private masterWatcher: FSWatcher |
81 | private m3u8Watcher: FSWatcher | ||
82 | |||
83 | private masterPlaylistCreated = false | ||
84 | private liveReady = false | ||
77 | 85 | ||
78 | private aborted = false | 86 | private aborted = false |
79 | 87 | ||
@@ -123,6 +131,13 @@ class MuxingSession extends EventEmitter { | |||
123 | this.outDirectory = getLiveDirectory(this.videoLive.Video) | 131 | this.outDirectory = getLiveDirectory(this.videoLive.Video) |
124 | this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) | 132 | this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) |
125 | 133 | ||
134 | this.liveSegmentShaStore = new LiveSegmentShaStore({ | ||
135 | videoUUID: this.videoLive.Video.uuid, | ||
136 | sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), | ||
137 | streamingPlaylist: this.streamingPlaylist, | ||
138 | sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED | ||
139 | }) | ||
140 | |||
126 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) | 141 | this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) |
127 | } | 142 | } |
128 | 143 | ||
@@ -159,8 +174,9 @@ class MuxingSession extends EventEmitter { | |||
159 | 174 | ||
160 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) | 175 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) |
161 | 176 | ||
162 | this.watchTSFiles() | ||
163 | this.watchMasterFile() | 177 | this.watchMasterFile() |
178 | this.watchTSFiles() | ||
179 | this.watchM3U8File() | ||
164 | 180 | ||
165 | let ffmpegShellCommand: string | 181 | let ffmpegShellCommand: string |
166 | this.ffmpegCommand.on('start', cmdline => { | 182 | this.ffmpegCommand.on('start', cmdline => { |
@@ -219,7 +235,7 @@ class MuxingSession extends EventEmitter { | |||
219 | setTimeout(() => { | 235 | setTimeout(() => { |
220 | // Wait latest segments generation, and close watchers | 236 | // Wait latest segments generation, and close watchers |
221 | 237 | ||
222 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) | 238 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) |
223 | .then(() => { | 239 | .then(() => { |
224 | // Process remaining segments hash | 240 | // Process remaining segments hash |
225 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | 241 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { |
@@ -240,14 +256,48 @@ class MuxingSession extends EventEmitter { | |||
240 | private watchMasterFile () { | 256 | private watchMasterFile () { |
241 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) | 257 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) |
242 | 258 | ||
243 | this.masterWatcher.on('add', () => { | 259 | this.masterWatcher.on('add', async () => { |
244 | this.emit('master-playlist-created', { videoId: this.videoId }) | 260 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { |
261 | try { | ||
262 | const url = await storeHLSFileFromFilename(this.streamingPlaylist, this.streamingPlaylist.playlistFilename) | ||
263 | |||
264 | this.streamingPlaylist.playlistUrl = url | ||
265 | await this.streamingPlaylist.save() | ||
266 | } catch (err) { | ||
267 | logger.error('Cannot upload live master file to object storage.', { err, ...this.lTags() }) | ||
268 | } | ||
269 | } | ||
270 | |||
271 | this.masterPlaylistCreated = true | ||
245 | 272 | ||
246 | this.masterWatcher.close() | 273 | this.masterWatcher.close() |
247 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) | 274 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) |
248 | }) | 275 | }) |
249 | } | 276 | } |
250 | 277 | ||
278 | private watchM3U8File () { | ||
279 | this.m3u8Watcher = watch(this.outDirectory + '/*.m3u8') | ||
280 | |||
281 | const sendQueues = new Map<string, PQueue>() | ||
282 | |||
283 | const onChangeOrAdd = async (m3u8Path: string) => { | ||
284 | if (this.streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return | ||
285 | |||
286 | try { | ||
287 | if (!sendQueues.has(m3u8Path)) { | ||
288 | sendQueues.set(m3u8Path, new PQueue({ concurrency: 1 })) | ||
289 | } | ||
290 | |||
291 | const queue = sendQueues.get(m3u8Path) | ||
292 | await queue.add(() => storeHLSFileFromPath(this.streamingPlaylist, m3u8Path)) | ||
293 | } catch (err) { | ||
294 | logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) | ||
295 | } | ||
296 | } | ||
297 | |||
298 | this.m3u8Watcher.on('change', onChangeOrAdd) | ||
299 | } | ||
300 | |||
251 | private watchTSFiles () { | 301 | private watchTSFiles () { |
252 | const startStreamDateTime = new Date().getTime() | 302 | const startStreamDateTime = new Date().getTime() |
253 | 303 | ||
@@ -282,7 +332,21 @@ class MuxingSession extends EventEmitter { | |||
282 | } | 332 | } |
283 | } | 333 | } |
284 | 334 | ||
285 | const deleteHandler = (segmentPath: string) => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) | 335 | const deleteHandler = async (segmentPath: string) => { |
336 | try { | ||
337 | await this.liveSegmentShaStore.removeSegmentSha(segmentPath) | ||
338 | } catch (err) { | ||
339 | logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) | ||
340 | } | ||
341 | |||
342 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
343 | try { | ||
344 | await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) | ||
345 | } catch (err) { | ||
346 | logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) | ||
347 | } | ||
348 | } | ||
349 | } | ||
286 | 350 | ||
287 | this.tsWatcher.on('add', p => addHandler(p)) | 351 | this.tsWatcher.on('add', p => addHandler(p)) |
288 | this.tsWatcher.on('unlink', p => deleteHandler(p)) | 352 | this.tsWatcher.on('unlink', p => deleteHandler(p)) |
@@ -315,6 +379,7 @@ class MuxingSession extends EventEmitter { | |||
315 | extname: '.ts', | 379 | extname: '.ts', |
316 | infoHash: null, | 380 | infoHash: null, |
317 | fps: this.fps, | 381 | fps: this.fps, |
382 | storage: this.streamingPlaylist.storage, | ||
318 | videoStreamingPlaylistId: this.streamingPlaylist.id | 383 | videoStreamingPlaylistId: this.streamingPlaylist.id |
319 | }) | 384 | }) |
320 | 385 | ||
@@ -343,18 +408,36 @@ class MuxingSession extends EventEmitter { | |||
343 | } | 408 | } |
344 | 409 | ||
345 | private processSegments (segmentPaths: string[]) { | 410 | private processSegments (segmentPaths: string[]) { |
346 | mapSeries(segmentPaths, async previousSegment => { | 411 | mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) |
347 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | 412 | .catch(err => { |
348 | await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) | 413 | if (this.aborted) return |
414 | |||
415 | logger.error('Cannot process segments', { err, ...this.lTags() }) | ||
416 | }) | ||
417 | } | ||
418 | |||
419 | private async processSegment (segmentPath: string) { | ||
420 | // Add sha hash of previous segments, because ffmpeg should have finished generating them | ||
421 | await this.liveSegmentShaStore.addSegmentSha(segmentPath) | ||
422 | |||
423 | if (this.saveReplay) { | ||
424 | await this.addSegmentToReplay(segmentPath) | ||
425 | } | ||
349 | 426 | ||
350 | if (this.saveReplay) { | 427 | if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { |
351 | await this.addSegmentToReplay(previousSegment) | 428 | try { |
429 | await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) | ||
430 | } catch (err) { | ||
431 | logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) | ||
352 | } | 432 | } |
353 | }).catch(err => { | 433 | } |
354 | if (this.aborted) return | ||
355 | 434 | ||
356 | logger.error('Cannot process segments', { err, ...this.lTags() }) | 435 | // Master playlist and segment JSON file are created, live is ready |
357 | }) | 436 | if (this.masterPlaylistCreated && !this.liveReady) { |
437 | this.liveReady = true | ||
438 | |||
439 | this.emit('live-ready', { videoId: this.videoId }) | ||
440 | } | ||
358 | } | 441 | } |
359 | 442 | ||
360 | private hasClientSocketInBadHealth (sessionId: string) { | 443 | private hasClientSocketInBadHealth (sessionId: string) { |
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts index c23f5b6a6..3cc92ca30 100644 --- a/server/lib/moderation.ts +++ b/server/lib/moderation.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { VideoUploadFile } from 'express' | 1 | import express, { VideoUploadFile } from 'express' |
2 | import { PathLike } from 'fs-extra' | 2 | import { PathLike } from 'fs-extra' |
3 | import { Transaction } from 'sequelize/types' | 3 | import { Transaction } from 'sequelize/types' |
4 | import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' | 4 | import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' |
@@ -13,18 +13,15 @@ import { | |||
13 | MAbuseFull, | 13 | MAbuseFull, |
14 | MAccountDefault, | 14 | MAccountDefault, |
15 | MAccountLight, | 15 | MAccountLight, |
16 | MComment, | ||
16 | MCommentAbuseAccountVideo, | 17 | MCommentAbuseAccountVideo, |
17 | MCommentOwnerVideo, | 18 | MCommentOwnerVideo, |
18 | MUser, | 19 | MUser, |
19 | MVideoAbuseVideoFull, | 20 | MVideoAbuseVideoFull, |
20 | MVideoAccountLightBlacklistAllFiles | 21 | MVideoAccountLightBlacklistAllFiles |
21 | } from '@server/types/models' | 22 | } from '@server/types/models' |
22 | import { ActivityCreate } from '../../shared/models/activitypub' | ||
23 | import { VideoObject } from '../../shared/models/activitypub/objects' | ||
24 | import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' | ||
25 | import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' | 23 | import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' |
26 | import { VideoCommentCreate } from '../../shared/models/videos/comment' | 24 | import { VideoCommentCreate } from '../../shared/models/videos/comment' |
27 | import { ActorModel } from '../models/actor/actor' | ||
28 | import { UserModel } from '../models/user/user' | 25 | import { UserModel } from '../models/user/user' |
29 | import { VideoModel } from '../models/video/video' | 26 | import { VideoModel } from '../models/video/video' |
30 | import { VideoCommentModel } from '../models/video/video-comment' | 27 | import { VideoCommentModel } from '../models/video/video-comment' |
@@ -36,7 +33,9 @@ export type AcceptResult = { | |||
36 | errorMessage?: string | 33 | errorMessage?: string |
37 | } | 34 | } |
38 | 35 | ||
39 | // Can be filtered by plugins | 36 | // --------------------------------------------------------------------------- |
37 | |||
38 | // Stub function that can be filtered by plugins | ||
40 | function isLocalVideoAccepted (object: { | 39 | function isLocalVideoAccepted (object: { |
41 | videoBody: VideoCreate | 40 | videoBody: VideoCreate |
42 | videoFile: VideoUploadFile | 41 | videoFile: VideoUploadFile |
@@ -45,6 +44,9 @@ function isLocalVideoAccepted (object: { | |||
45 | return { accepted: true } | 44 | return { accepted: true } |
46 | } | 45 | } |
47 | 46 | ||
47 | // --------------------------------------------------------------------------- | ||
48 | |||
49 | // Stub function that can be filtered by plugins | ||
48 | function isLocalLiveVideoAccepted (object: { | 50 | function isLocalLiveVideoAccepted (object: { |
49 | liveVideoBody: LiveVideoCreate | 51 | liveVideoBody: LiveVideoCreate |
50 | user: UserModel | 52 | user: UserModel |
@@ -52,7 +54,11 @@ function isLocalLiveVideoAccepted (object: { | |||
52 | return { accepted: true } | 54 | return { accepted: true } |
53 | } | 55 | } |
54 | 56 | ||
57 | // --------------------------------------------------------------------------- | ||
58 | |||
59 | // Stub function that can be filtered by plugins | ||
55 | function isLocalVideoThreadAccepted (_object: { | 60 | function isLocalVideoThreadAccepted (_object: { |
61 | req: express.Request | ||
56 | commentBody: VideoCommentCreate | 62 | commentBody: VideoCommentCreate |
57 | video: VideoModel | 63 | video: VideoModel |
58 | user: UserModel | 64 | user: UserModel |
@@ -60,7 +66,9 @@ function isLocalVideoThreadAccepted (_object: { | |||
60 | return { accepted: true } | 66 | return { accepted: true } |
61 | } | 67 | } |
62 | 68 | ||
69 | // Stub function that can be filtered by plugins | ||
63 | function isLocalVideoCommentReplyAccepted (_object: { | 70 | function isLocalVideoCommentReplyAccepted (_object: { |
71 | req: express.Request | ||
64 | commentBody: VideoCommentCreate | 72 | commentBody: VideoCommentCreate |
65 | parentComment: VideoCommentModel | 73 | parentComment: VideoCommentModel |
66 | video: VideoModel | 74 | video: VideoModel |
@@ -69,22 +77,18 @@ function isLocalVideoCommentReplyAccepted (_object: { | |||
69 | return { accepted: true } | 77 | return { accepted: true } |
70 | } | 78 | } |
71 | 79 | ||
72 | function isRemoteVideoAccepted (_object: { | 80 | // --------------------------------------------------------------------------- |
73 | activity: ActivityCreate | ||
74 | videoAP: VideoObject | ||
75 | byActor: ActorModel | ||
76 | }): AcceptResult { | ||
77 | return { accepted: true } | ||
78 | } | ||
79 | 81 | ||
82 | // Stub function that can be filtered by plugins | ||
80 | function isRemoteVideoCommentAccepted (_object: { | 83 | function isRemoteVideoCommentAccepted (_object: { |
81 | activity: ActivityCreate | 84 | comment: MComment |
82 | commentAP: VideoCommentObject | ||
83 | byActor: ActorModel | ||
84 | }): AcceptResult { | 85 | }): AcceptResult { |
85 | return { accepted: true } | 86 | return { accepted: true } |
86 | } | 87 | } |
87 | 88 | ||
89 | // --------------------------------------------------------------------------- | ||
90 | |||
91 | // Stub function that can be filtered by plugins | ||
88 | function isPreImportVideoAccepted (object: { | 92 | function isPreImportVideoAccepted (object: { |
89 | videoImportBody: VideoImportCreate | 93 | videoImportBody: VideoImportCreate |
90 | user: MUser | 94 | user: MUser |
@@ -92,6 +96,7 @@ function isPreImportVideoAccepted (object: { | |||
92 | return { accepted: true } | 96 | return { accepted: true } |
93 | } | 97 | } |
94 | 98 | ||
99 | // Stub function that can be filtered by plugins | ||
95 | function isPostImportVideoAccepted (object: { | 100 | function isPostImportVideoAccepted (object: { |
96 | videoFilePath: PathLike | 101 | videoFilePath: PathLike |
97 | videoFile: VideoFileModel | 102 | videoFile: VideoFileModel |
@@ -100,6 +105,8 @@ function isPostImportVideoAccepted (object: { | |||
100 | return { accepted: true } | 105 | return { accepted: true } |
101 | } | 106 | } |
102 | 107 | ||
108 | // --------------------------------------------------------------------------- | ||
109 | |||
103 | async function createVideoAbuse (options: { | 110 | async function createVideoAbuse (options: { |
104 | baseAbuse: FilteredModelAttributes<AbuseModel> | 111 | baseAbuse: FilteredModelAttributes<AbuseModel> |
105 | videoInstance: MVideoAccountLightBlacklistAllFiles | 112 | videoInstance: MVideoAccountLightBlacklistAllFiles |
@@ -189,12 +196,13 @@ function createAccountAbuse (options: { | |||
189 | }) | 196 | }) |
190 | } | 197 | } |
191 | 198 | ||
199 | // --------------------------------------------------------------------------- | ||
200 | |||
192 | export { | 201 | export { |
193 | isLocalLiveVideoAccepted, | 202 | isLocalLiveVideoAccepted, |
194 | 203 | ||
195 | isLocalVideoAccepted, | 204 | isLocalVideoAccepted, |
196 | isLocalVideoThreadAccepted, | 205 | isLocalVideoThreadAccepted, |
197 | isRemoteVideoAccepted, | ||
198 | isRemoteVideoCommentAccepted, | 206 | isRemoteVideoCommentAccepted, |
199 | isLocalVideoCommentReplyAccepted, | 207 | isLocalVideoCommentReplyAccepted, |
200 | isPreImportVideoAccepted, | 208 | isPreImportVideoAccepted, |
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts index 16161362c..3046d76bc 100644 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -1,19 +1,23 @@ | |||
1 | import { map } from 'bluebird' | ||
1 | import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' | 2 | import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' |
2 | import { dirname } from 'path' | 3 | import { dirname } from 'path' |
3 | import { Readable } from 'stream' | 4 | import { Readable } from 'stream' |
4 | import { | 5 | import { |
6 | _Object, | ||
5 | CompleteMultipartUploadCommandOutput, | 7 | CompleteMultipartUploadCommandOutput, |
6 | DeleteObjectCommand, | 8 | DeleteObjectCommand, |
7 | GetObjectCommand, | 9 | GetObjectCommand, |
8 | ListObjectsV2Command, | 10 | ListObjectsV2Command, |
9 | PutObjectCommandInput | 11 | PutObjectAclCommand, |
12 | PutObjectCommandInput, | ||
13 | S3Client | ||
10 | } from '@aws-sdk/client-s3' | 14 | } from '@aws-sdk/client-s3' |
11 | import { Upload } from '@aws-sdk/lib-storage' | 15 | import { Upload } from '@aws-sdk/lib-storage' |
12 | import { pipelinePromise } from '@server/helpers/core-utils' | 16 | import { pipelinePromise } from '@server/helpers/core-utils' |
13 | import { isArray } from '@server/helpers/custom-validators/misc' | 17 | import { isArray } from '@server/helpers/custom-validators/misc' |
14 | import { logger } from '@server/helpers/logger' | 18 | import { logger } from '@server/helpers/logger' |
15 | import { CONFIG } from '@server/initializers/config' | 19 | import { CONFIG } from '@server/initializers/config' |
16 | import { getPrivateUrl } from '../urls' | 20 | import { getInternalUrl } from '../urls' |
17 | import { getClient } from './client' | 21 | import { getClient } from './client' |
18 | import { lTags } from './logger' | 22 | import { lTags } from './logger' |
19 | 23 | ||
@@ -22,73 +26,125 @@ type BucketInfo = { | |||
22 | PREFIX?: string | 26 | PREFIX?: string |
23 | } | 27 | } |
24 | 28 | ||
29 | async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { | ||
30 | const s3Client = getClient() | ||
31 | |||
32 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
33 | const listCommand = new ListObjectsV2Command({ | ||
34 | Bucket: bucketInfo.BUCKET_NAME, | ||
35 | Prefix: commandPrefix | ||
36 | }) | ||
37 | |||
38 | const listedObjects = await s3Client.send(listCommand) | ||
39 | |||
40 | if (isArray(listedObjects.Contents) !== true) return [] | ||
41 | |||
42 | return listedObjects.Contents.map(c => c.Key) | ||
43 | } | ||
44 | |||
45 | // --------------------------------------------------------------------------- | ||
46 | |||
25 | async function storeObject (options: { | 47 | async function storeObject (options: { |
26 | inputPath: string | 48 | inputPath: string |
27 | objectStorageKey: string | 49 | objectStorageKey: string |
28 | bucketInfo: BucketInfo | 50 | bucketInfo: BucketInfo |
51 | isPrivate: boolean | ||
29 | }): Promise<string> { | 52 | }): Promise<string> { |
30 | const { inputPath, objectStorageKey, bucketInfo } = options | 53 | const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options |
31 | 54 | ||
32 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | 55 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) |
33 | 56 | ||
34 | const fileStream = createReadStream(inputPath) | 57 | const fileStream = createReadStream(inputPath) |
35 | 58 | ||
36 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) | 59 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) |
37 | } | 60 | } |
38 | 61 | ||
39 | async function removeObject (filename: string, bucketInfo: BucketInfo) { | 62 | // --------------------------------------------------------------------------- |
40 | const command = new DeleteObjectCommand({ | 63 | |
64 | function updateObjectACL (options: { | ||
65 | objectStorageKey: string | ||
66 | bucketInfo: BucketInfo | ||
67 | isPrivate: boolean | ||
68 | }) { | ||
69 | const { objectStorageKey, bucketInfo, isPrivate } = options | ||
70 | |||
71 | const key = buildKey(objectStorageKey, bucketInfo) | ||
72 | |||
73 | logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) | ||
74 | |||
75 | const command = new PutObjectAclCommand({ | ||
41 | Bucket: bucketInfo.BUCKET_NAME, | 76 | Bucket: bucketInfo.BUCKET_NAME, |
42 | Key: buildKey(filename, bucketInfo) | 77 | Key: key, |
78 | ACL: getACL(isPrivate) | ||
43 | }) | 79 | }) |
44 | 80 | ||
45 | return getClient().send(command) | 81 | return getClient().send(command) |
46 | } | 82 | } |
47 | 83 | ||
48 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | 84 | function updatePrefixACL (options: { |
49 | const s3Client = getClient() | 85 | prefix: string |
50 | 86 | bucketInfo: BucketInfo | |
51 | const commandPrefix = bucketInfo.PREFIX + prefix | 87 | isPrivate: boolean |
52 | const listCommand = new ListObjectsV2Command({ | 88 | }) { |
53 | Bucket: bucketInfo.BUCKET_NAME, | 89 | const { prefix, bucketInfo, isPrivate } = options |
54 | Prefix: commandPrefix | 90 | |
91 | logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
92 | |||
93 | return applyOnPrefix({ | ||
94 | prefix, | ||
95 | bucketInfo, | ||
96 | commandBuilder: obj => { | ||
97 | logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
98 | |||
99 | return new PutObjectAclCommand({ | ||
100 | Bucket: bucketInfo.BUCKET_NAME, | ||
101 | Key: obj.Key, | ||
102 | ACL: getACL(isPrivate) | ||
103 | }) | ||
104 | } | ||
55 | }) | 105 | }) |
106 | } | ||
56 | 107 | ||
57 | const listedObjects = await s3Client.send(listCommand) | 108 | // --------------------------------------------------------------------------- |
58 | |||
59 | // FIXME: use bulk delete when s3ninja will support this operation | ||
60 | // const deleteParams = { | ||
61 | // Bucket: bucketInfo.BUCKET_NAME, | ||
62 | // Delete: { Objects: [] } | ||
63 | // } | ||
64 | 109 | ||
65 | if (isArray(listedObjects.Contents) !== true) { | 110 | function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { |
66 | const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | 111 | const key = buildKey(objectStorageKey, bucketInfo) |
67 | 112 | ||
68 | logger.error(message, { response: listedObjects, ...lTags() }) | 113 | return removeObjectByFullKey(key, bucketInfo) |
69 | throw new Error(message) | 114 | } |
70 | } | ||
71 | 115 | ||
72 | for (const object of listedObjects.Contents) { | 116 | function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { |
73 | const command = new DeleteObjectCommand({ | 117 | logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) |
74 | Bucket: bucketInfo.BUCKET_NAME, | ||
75 | Key: object.Key | ||
76 | }) | ||
77 | 118 | ||
78 | await s3Client.send(command) | 119 | const command = new DeleteObjectCommand({ |
120 | Bucket: bucketInfo.BUCKET_NAME, | ||
121 | Key: fullKey | ||
122 | }) | ||
79 | 123 | ||
80 | // FIXME: use bulk delete when s3ninja will support this operation | 124 | return getClient().send(command) |
81 | // deleteParams.Delete.Objects.push({ Key: object.Key }) | 125 | } |
82 | } | ||
83 | 126 | ||
127 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | ||
84 | // FIXME: use bulk delete when s3ninja will support this operation | 128 | // FIXME: use bulk delete when s3ninja will support this operation |
85 | // const deleteCommand = new DeleteObjectsCommand(deleteParams) | ||
86 | // await s3Client.send(deleteCommand) | ||
87 | 129 | ||
88 | // Repeat if not all objects could be listed at once (limit of 1000?) | 130 | logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) |
89 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) | 131 | |
132 | return applyOnPrefix({ | ||
133 | prefix, | ||
134 | bucketInfo, | ||
135 | commandBuilder: obj => { | ||
136 | logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
137 | |||
138 | return new DeleteObjectCommand({ | ||
139 | Bucket: bucketInfo.BUCKET_NAME, | ||
140 | Key: obj.Key | ||
141 | }) | ||
142 | } | ||
143 | }) | ||
90 | } | 144 | } |
91 | 145 | ||
146 | // --------------------------------------------------------------------------- | ||
147 | |||
92 | async function makeAvailable (options: { | 148 | async function makeAvailable (options: { |
93 | key: string | 149 | key: string |
94 | destination: string | 150 | destination: string |
@@ -116,13 +172,43 @@ function buildKey (key: string, bucketInfo: BucketInfo) { | |||
116 | 172 | ||
117 | // --------------------------------------------------------------------------- | 173 | // --------------------------------------------------------------------------- |
118 | 174 | ||
175 | async function createObjectReadStream (options: { | ||
176 | key: string | ||
177 | bucketInfo: BucketInfo | ||
178 | rangeHeader: string | ||
179 | }) { | ||
180 | const { key, bucketInfo, rangeHeader } = options | ||
181 | |||
182 | const command = new GetObjectCommand({ | ||
183 | Bucket: bucketInfo.BUCKET_NAME, | ||
184 | Key: buildKey(key, bucketInfo), | ||
185 | Range: rangeHeader | ||
186 | }) | ||
187 | |||
188 | const response = await getClient().send(command) | ||
189 | |||
190 | return response.Body as Readable | ||
191 | } | ||
192 | |||
193 | // --------------------------------------------------------------------------- | ||
194 | |||
119 | export { | 195 | export { |
120 | BucketInfo, | 196 | BucketInfo, |
121 | buildKey, | 197 | buildKey, |
198 | |||
122 | storeObject, | 199 | storeObject, |
200 | |||
123 | removeObject, | 201 | removeObject, |
202 | removeObjectByFullKey, | ||
124 | removePrefix, | 203 | removePrefix, |
125 | makeAvailable | 204 | |
205 | makeAvailable, | ||
206 | |||
207 | updateObjectACL, | ||
208 | updatePrefixACL, | ||
209 | |||
210 | listKeysOfPrefix, | ||
211 | createObjectReadStream | ||
126 | } | 212 | } |
127 | 213 | ||
128 | // --------------------------------------------------------------------------- | 214 | // --------------------------------------------------------------------------- |
@@ -131,17 +217,15 @@ async function uploadToStorage (options: { | |||
131 | content: ReadStream | 217 | content: ReadStream |
132 | objectStorageKey: string | 218 | objectStorageKey: string |
133 | bucketInfo: BucketInfo | 219 | bucketInfo: BucketInfo |
220 | isPrivate: boolean | ||
134 | }) { | 221 | }) { |
135 | const { content, objectStorageKey, bucketInfo } = options | 222 | const { content, objectStorageKey, bucketInfo, isPrivate } = options |
136 | 223 | ||
137 | const input: PutObjectCommandInput = { | 224 | const input: PutObjectCommandInput = { |
138 | Body: content, | 225 | Body: content, |
139 | Bucket: bucketInfo.BUCKET_NAME, | 226 | Bucket: bucketInfo.BUCKET_NAME, |
140 | Key: buildKey(objectStorageKey, bucketInfo) | 227 | Key: buildKey(objectStorageKey, bucketInfo), |
141 | } | 228 | ACL: getACL(isPrivate) |
142 | |||
143 | if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) { | ||
144 | input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL | ||
145 | } | 229 | } |
146 | 230 | ||
147 | const parallelUploads3 = new Upload({ | 231 | const parallelUploads3 = new Upload({ |
@@ -171,5 +255,50 @@ async function uploadToStorage (options: { | |||
171 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() | 255 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() |
172 | ) | 256 | ) |
173 | 257 | ||
174 | return getPrivateUrl(bucketInfo, objectStorageKey) | 258 | return getInternalUrl(bucketInfo, objectStorageKey) |
259 | } | ||
260 | |||
261 | async function applyOnPrefix (options: { | ||
262 | prefix: string | ||
263 | bucketInfo: BucketInfo | ||
264 | commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0] | ||
265 | |||
266 | continuationToken?: string | ||
267 | }) { | ||
268 | const { prefix, bucketInfo, commandBuilder, continuationToken } = options | ||
269 | |||
270 | const s3Client = getClient() | ||
271 | |||
272 | const commandPrefix = buildKey(prefix, bucketInfo) | ||
273 | const listCommand = new ListObjectsV2Command({ | ||
274 | Bucket: bucketInfo.BUCKET_NAME, | ||
275 | Prefix: commandPrefix, | ||
276 | ContinuationToken: continuationToken | ||
277 | }) | ||
278 | |||
279 | const listedObjects = await s3Client.send(listCommand) | ||
280 | |||
281 | if (isArray(listedObjects.Contents) !== true) { | ||
282 | const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
283 | |||
284 | logger.error(message, { response: listedObjects, ...lTags() }) | ||
285 | throw new Error(message) | ||
286 | } | ||
287 | |||
288 | await map(listedObjects.Contents, object => { | ||
289 | const command = commandBuilder(object) | ||
290 | |||
291 | return s3Client.send(command) | ||
292 | }, { concurrency: 10 }) | ||
293 | |||
294 | // Repeat if not all objects could be listed at once (limit of 1000?) | ||
295 | if (listedObjects.IsTruncated) { | ||
296 | await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) | ||
297 | } | ||
298 | } | ||
299 | |||
300 | function getACL (isPrivate: boolean) { | ||
301 | return isPrivate | ||
302 | ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE | ||
303 | : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC | ||
175 | } | 304 | } |
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts index 2a889190b..a47a98b98 100644 --- a/server/lib/object-storage/urls.ts +++ b/server/lib/object-storage/urls.ts | |||
@@ -1,10 +1,14 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | 1 | import { CONFIG } from '@server/initializers/config' |
2 | import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
3 | import { MVideoUUID } from '@server/types/models' | ||
2 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' | 4 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' |
3 | 5 | ||
4 | function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { | 6 | function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { |
5 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) | 7 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) |
6 | } | 8 | } |
7 | 9 | ||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
8 | function getWebTorrentPublicFileUrl (fileUrl: string) { | 12 | function getWebTorrentPublicFileUrl (fileUrl: string) { |
9 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL | 13 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL |
10 | if (!baseUrl) return fileUrl | 14 | if (!baseUrl) return fileUrl |
@@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) { | |||
19 | return replaceByBaseUrl(fileUrl, baseUrl) | 23 | return replaceByBaseUrl(fileUrl, baseUrl) |
20 | } | 24 | } |
21 | 25 | ||
26 | // --------------------------------------------------------------------------- | ||
27 | |||
28 | function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { | ||
29 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` | ||
30 | } | ||
31 | |||
32 | function getWebTorrentPrivateFileUrl (filename: string) { | ||
33 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
22 | export { | 38 | export { |
23 | getPrivateUrl, | 39 | getInternalUrl, |
40 | |||
24 | getWebTorrentPublicFileUrl, | 41 | getWebTorrentPublicFileUrl, |
25 | replaceByBaseUrl, | 42 | getHLSPublicFileUrl, |
26 | getHLSPublicFileUrl | 43 | |
44 | getHLSPrivateFileUrl, | ||
45 | getWebTorrentPrivateFileUrl, | ||
46 | |||
47 | replaceByBaseUrl | ||
27 | } | 48 | } |
28 | 49 | ||
29 | // --------------------------------------------------------------------------- | 50 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index 66e738200..b764e4b22 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts | |||
@@ -1,39 +1,102 @@ | |||
1 | import { join } from 'path' | 1 | import { basename, join } from 'path' |
2 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
3 | import { CONFIG } from '@server/initializers/config' | 3 | import { CONFIG } from '@server/initializers/config' |
4 | import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' | 4 | import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' |
5 | import { getHLSDirectory } from '../paths' | 5 | import { getHLSDirectory } from '../paths' |
6 | import { VideoPathManager } from '../video-path-manager' | ||
6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | 7 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' |
7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | 8 | import { |
9 | createObjectReadStream, | ||
10 | listKeysOfPrefix, | ||
11 | lTags, | ||
12 | makeAvailable, | ||
13 | removeObject, | ||
14 | removeObjectByFullKey, | ||
15 | removePrefix, | ||
16 | storeObject, | ||
17 | updateObjectACL, | ||
18 | updatePrefixACL | ||
19 | } from './shared' | ||
20 | |||
21 | function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { | ||
22 | return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
8 | 26 | ||
9 | function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) { | 27 | function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { |
10 | return storeObject({ | 28 | return storeObject({ |
11 | inputPath: path ?? join(getHLSDirectory(playlist.Video), filename), | 29 | inputPath: join(getHLSDirectory(playlist.Video), filename), |
12 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), | 30 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), |
13 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | 31 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, |
32 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
14 | }) | 33 | }) |
15 | } | 34 | } |
16 | 35 | ||
17 | function storeWebTorrentFile (filename: string) { | 36 | function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { |
18 | return storeObject({ | 37 | return storeObject({ |
19 | inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), | 38 | inputPath: path, |
20 | objectStorageKey: generateWebTorrentObjectStorageKey(filename), | 39 | objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), |
21 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | 40 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, |
41 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
42 | }) | ||
43 | } | ||
44 | |||
45 | // --------------------------------------------------------------------------- | ||
46 | |||
47 | function storeWebTorrentFile (video: MVideo, file: MVideoFile) { | ||
48 | return storeObject({ | ||
49 | inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), | ||
50 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), | ||
51 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | ||
52 | isPrivate: video.hasPrivateStaticPath() | ||
53 | }) | ||
54 | } | ||
55 | |||
56 | // --------------------------------------------------------------------------- | ||
57 | |||
58 | function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) { | ||
59 | return updateObjectACL({ | ||
60 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), | ||
61 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | ||
62 | isPrivate: video.hasPrivateStaticPath() | ||
63 | }) | ||
64 | } | ||
65 | |||
66 | function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { | ||
67 | return updatePrefixACL({ | ||
68 | prefix: generateHLSObjectBaseStorageKey(playlist), | ||
69 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
70 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
22 | }) | 71 | }) |
23 | } | 72 | } |
24 | 73 | ||
74 | // --------------------------------------------------------------------------- | ||
75 | |||
25 | function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { | 76 | function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { |
26 | return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | 77 | return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) |
27 | } | 78 | } |
28 | 79 | ||
29 | function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) { | 80 | function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { |
30 | return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | 81 | return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) |
31 | } | 82 | } |
32 | 83 | ||
84 | function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { | ||
85 | return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
86 | } | ||
87 | |||
88 | function removeHLSFileObjectStorageByFullKey (key: string) { | ||
89 | return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
90 | } | ||
91 | |||
92 | // --------------------------------------------------------------------------- | ||
93 | |||
33 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { | 94 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { |
34 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) | 95 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) |
35 | } | 96 | } |
36 | 97 | ||
98 | // --------------------------------------------------------------------------- | ||
99 | |||
37 | async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { | 100 | async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { |
38 | const key = generateHLSObjectStorageKey(playlist, filename) | 101 | const key = generateHLSObjectStorageKey(playlist, filename) |
39 | 102 | ||
@@ -62,14 +125,61 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin | |||
62 | return destination | 125 | return destination |
63 | } | 126 | } |
64 | 127 | ||
128 | // --------------------------------------------------------------------------- | ||
129 | |||
130 | function getWebTorrentFileReadStream (options: { | ||
131 | filename: string | ||
132 | rangeHeader: string | ||
133 | }) { | ||
134 | const { filename, rangeHeader } = options | ||
135 | |||
136 | const key = generateWebTorrentObjectStorageKey(filename) | ||
137 | |||
138 | return createObjectReadStream({ | ||
139 | key, | ||
140 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | ||
141 | rangeHeader | ||
142 | }) | ||
143 | } | ||
144 | |||
145 | function getHLSFileReadStream (options: { | ||
146 | playlist: MStreamingPlaylistVideo | ||
147 | filename: string | ||
148 | rangeHeader: string | ||
149 | }) { | ||
150 | const { playlist, filename, rangeHeader } = options | ||
151 | |||
152 | const key = generateHLSObjectStorageKey(playlist, filename) | ||
153 | |||
154 | return createObjectReadStream({ | ||
155 | key, | ||
156 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
157 | rangeHeader | ||
158 | }) | ||
159 | } | ||
160 | |||
161 | // --------------------------------------------------------------------------- | ||
162 | |||
65 | export { | 163 | export { |
164 | listHLSFileKeysOf, | ||
165 | |||
66 | storeWebTorrentFile, | 166 | storeWebTorrentFile, |
67 | storeHLSFile, | 167 | storeHLSFileFromFilename, |
168 | storeHLSFileFromPath, | ||
169 | |||
170 | updateWebTorrentFileACL, | ||
171 | updateHLSFilesACL, | ||
68 | 172 | ||
69 | removeHLSObjectStorage, | 173 | removeHLSObjectStorage, |
70 | removeHLSFileObjectStorage, | 174 | removeHLSFileObjectStorageByFilename, |
175 | removeHLSFileObjectStorageByPath, | ||
176 | removeHLSFileObjectStorageByFullKey, | ||
177 | |||
71 | removeWebTorrentObjectStorage, | 178 | removeWebTorrentObjectStorage, |
72 | 179 | ||
73 | makeWebTorrentFileAvailable, | 180 | makeWebTorrentFileAvailable, |
74 | makeHLSFileAvailable | 181 | makeHLSFileAvailable, |
182 | |||
183 | getWebTorrentFileReadStream, | ||
184 | getHLSFileReadStream | ||
75 | } | 185 | } |
diff --git a/server/lib/paths.ts b/server/lib/paths.ts index b29854700..470970f55 100644 --- a/server/lib/paths.ts +++ b/server/lib/paths.ts | |||
@@ -1,9 +1,10 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { CONFIG } from '@server/initializers/config' | 2 | import { CONFIG } from '@server/initializers/config' |
3 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, VIDEO_LIVE } from '@server/initializers/constants' | 3 | import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants' |
4 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | 4 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' |
5 | import { removeFragmentedMP4Ext } from '@shared/core-utils' | 5 | import { removeFragmentedMP4Ext } from '@shared/core-utils' |
6 | import { buildUUID } from '@shared/extra-utils' | 6 | import { buildUUID } from '@shared/extra-utils' |
7 | import { isVideoInPrivateDirectory } from './video-privacy' | ||
7 | 8 | ||
8 | // ################## Video file name ################## | 9 | // ################## Video file name ################## |
9 | 10 | ||
@@ -17,20 +18,24 @@ function generateHLSVideoFilename (resolution: number) { | |||
17 | 18 | ||
18 | // ################## Streaming playlist ################## | 19 | // ################## Streaming playlist ################## |
19 | 20 | ||
20 | function getLiveDirectory (video: MVideoUUID) { | 21 | function getLiveDirectory (video: MVideo) { |
21 | return getHLSDirectory(video) | 22 | return getHLSDirectory(video) |
22 | } | 23 | } |
23 | 24 | ||
24 | function getLiveReplayBaseDirectory (video: MVideoUUID) { | 25 | function getLiveReplayBaseDirectory (video: MVideo) { |
25 | return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) | 26 | return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) |
26 | } | 27 | } |
27 | 28 | ||
28 | function getHLSDirectory (video: MVideoUUID) { | 29 | function getHLSDirectory (video: MVideo) { |
29 | return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | 30 | if (isVideoInPrivateDirectory(video.privacy)) { |
31 | return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid) | ||
32 | } | ||
33 | |||
34 | return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid) | ||
30 | } | 35 | } |
31 | 36 | ||
32 | function getHLSRedundancyDirectory (video: MVideoUUID) { | 37 | function getHLSRedundancyDirectory (video: MVideoUUID) { |
33 | return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | 38 | return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) |
34 | } | 39 | } |
35 | 40 | ||
36 | function getHlsResolutionPlaylistFilename (videoFilename: string) { | 41 | function getHlsResolutionPlaylistFilename (videoFilename: string) { |
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts index 4e799b3d4..7b1def6e3 100644 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ b/server/lib/plugins/plugin-helpers-builder.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { Server } from 'http' | ||
2 | import { join } from 'path' | 3 | import { join } from 'path' |
3 | import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' | 4 | import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' |
4 | import { buildLogger } from '@server/helpers/logger' | 5 | import { buildLogger } from '@server/helpers/logger' |
@@ -13,15 +14,16 @@ import { ServerBlocklistModel } from '@server/models/server/server-blocklist' | |||
13 | import { UserModel } from '@server/models/user/user' | 14 | import { UserModel } from '@server/models/user/user' |
14 | import { VideoModel } from '@server/models/video/video' | 15 | import { VideoModel } from '@server/models/video/video' |
15 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' | 16 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' |
16 | import { MPlugin } from '@server/types/models' | 17 | import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' |
17 | import { PeerTubeHelpers } from '@server/types/plugins' | 18 | import { PeerTubeHelpers } from '@server/types/plugins' |
18 | import { VideoBlacklistCreate, VideoStorage } from '@shared/models' | 19 | import { VideoBlacklistCreate, VideoStorage } from '@shared/models' |
19 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' | 20 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' |
21 | import { PeerTubeSocket } from '../peertube-socket' | ||
20 | import { ServerConfigManager } from '../server-config-manager' | 22 | import { ServerConfigManager } from '../server-config-manager' |
21 | import { blacklistVideo, unblacklistVideo } from '../video-blacklist' | 23 | import { blacklistVideo, unblacklistVideo } from '../video-blacklist' |
22 | import { VideoPathManager } from '../video-path-manager' | 24 | import { VideoPathManager } from '../video-path-manager' |
23 | 25 | ||
24 | function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { | 26 | function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers { |
25 | const logger = buildPluginLogger(npmName) | 27 | const logger = buildPluginLogger(npmName) |
26 | 28 | ||
27 | const database = buildDatabaseHelpers() | 29 | const database = buildDatabaseHelpers() |
@@ -29,12 +31,14 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel | |||
29 | 31 | ||
30 | const config = buildConfigHelpers() | 32 | const config = buildConfigHelpers() |
31 | 33 | ||
32 | const server = buildServerHelpers() | 34 | const server = buildServerHelpers(httpServer) |
33 | 35 | ||
34 | const moderation = buildModerationHelpers() | 36 | const moderation = buildModerationHelpers() |
35 | 37 | ||
36 | const plugin = buildPluginRelatedHelpers(pluginModel, npmName) | 38 | const plugin = buildPluginRelatedHelpers(pluginModel, npmName) |
37 | 39 | ||
40 | const socket = buildSocketHelpers() | ||
41 | |||
38 | const user = buildUserHelpers() | 42 | const user = buildUserHelpers() |
39 | 43 | ||
40 | return { | 44 | return { |
@@ -45,6 +49,7 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel | |||
45 | moderation, | 49 | moderation, |
46 | plugin, | 50 | plugin, |
47 | server, | 51 | server, |
52 | socket, | ||
48 | user | 53 | user |
49 | } | 54 | } |
50 | } | 55 | } |
@@ -65,8 +70,10 @@ function buildDatabaseHelpers () { | |||
65 | } | 70 | } |
66 | } | 71 | } |
67 | 72 | ||
68 | function buildServerHelpers () { | 73 | function buildServerHelpers (httpServer: Server) { |
69 | return { | 74 | return { |
75 | getHTTPServer: () => httpServer, | ||
76 | |||
70 | getServerActor: () => getServerActor() | 77 | getServerActor: () => getServerActor() |
71 | } | 78 | } |
72 | } | 79 | } |
@@ -214,10 +221,23 @@ function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) { | |||
214 | 221 | ||
215 | getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, | 222 | getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, |
216 | 223 | ||
224 | getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`, | ||
225 | |||
217 | getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) | 226 | getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) |
218 | } | 227 | } |
219 | } | 228 | } |
220 | 229 | ||
230 | function buildSocketHelpers () { | ||
231 | return { | ||
232 | sendNotification: (userId: number, notification: UserNotificationModelForApi) => { | ||
233 | PeerTubeSocket.Instance.sendNotification(userId, notification) | ||
234 | }, | ||
235 | sendVideoLiveNewState: (video: MVideo) => { | ||
236 | PeerTubeSocket.Instance.sendVideoLiveNewState(video) | ||
237 | } | ||
238 | } | ||
239 | } | ||
240 | |||
221 | function buildUserHelpers () { | 241 | function buildUserHelpers () { |
222 | return { | 242 | return { |
223 | loadById: (id: number) => { | 243 | loadById: (id: number) => { |
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index a46b97fa4..c4d9b6574 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { createReadStream, createWriteStream } from 'fs' | 2 | import { createReadStream, createWriteStream } from 'fs' |
3 | import { ensureDir, outputFile, readJSON } from 'fs-extra' | 3 | import { ensureDir, outputFile, readJSON } from 'fs-extra' |
4 | import { Server } from 'http' | ||
4 | import { basename, join } from 'path' | 5 | import { basename, join } from 'path' |
5 | import { decachePlugin } from '@server/helpers/decache' | 6 | import { decachePlugin } from '@server/helpers/decache' |
6 | import { ApplicationModel } from '@server/models/application/application' | 7 | import { ApplicationModel } from '@server/models/application/application' |
@@ -67,9 +68,37 @@ export class PluginManager implements ServerHook { | |||
67 | private hooks: { [name: string]: HookInformationValue[] } = {} | 68 | private hooks: { [name: string]: HookInformationValue[] } = {} |
68 | private translations: PluginLocalesTranslations = {} | 69 | private translations: PluginLocalesTranslations = {} |
69 | 70 | ||
71 | private server: Server | ||
72 | |||
70 | private constructor () { | 73 | private constructor () { |
71 | } | 74 | } |
72 | 75 | ||
76 | init (server: Server) { | ||
77 | this.server = server | ||
78 | } | ||
79 | |||
80 | registerWebSocketRouter () { | ||
81 | this.server.on('upgrade', (request, socket, head) => { | ||
82 | const url = request.url | ||
83 | |||
84 | const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`) | ||
85 | if (!matched) return | ||
86 | |||
87 | const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN) | ||
88 | const subRoute = matched[3] | ||
89 | |||
90 | const result = this.getRegisteredPluginOrTheme(npmName) | ||
91 | if (!result) return | ||
92 | |||
93 | const routes = result.registerHelpers.getWebSocketRoutes() | ||
94 | |||
95 | const wss = routes.find(r => r.route.startsWith(subRoute)) | ||
96 | if (!wss) return | ||
97 | |||
98 | wss.handler(request, socket, head) | ||
99 | }) | ||
100 | } | ||
101 | |||
73 | // ###################### Getters ###################### | 102 | // ###################### Getters ###################### |
74 | 103 | ||
75 | isRegistered (npmName: string) { | 104 | isRegistered (npmName: string) { |
@@ -581,7 +610,7 @@ export class PluginManager implements ServerHook { | |||
581 | }) | 610 | }) |
582 | } | 611 | } |
583 | 612 | ||
584 | const registerHelpers = new RegisterHelpers(npmName, plugin, onHookAdded.bind(this)) | 613 | const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this)) |
585 | 614 | ||
586 | return { | 615 | return { |
587 | registerStore: registerHelpers, | 616 | registerStore: registerHelpers, |
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts index f4d405676..1aaef3606 100644 --- a/server/lib/plugins/register-helpers.ts +++ b/server/lib/plugins/register-helpers.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { Server } from 'http' | ||
2 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
3 | import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' | 4 | import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' |
4 | import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' | 5 | import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' |
@@ -8,7 +9,8 @@ import { | |||
8 | RegisterServerAuthExternalResult, | 9 | RegisterServerAuthExternalResult, |
9 | RegisterServerAuthPassOptions, | 10 | RegisterServerAuthPassOptions, |
10 | RegisterServerExternalAuthenticatedResult, | 11 | RegisterServerExternalAuthenticatedResult, |
11 | RegisterServerOptions | 12 | RegisterServerOptions, |
13 | RegisterServerWebSocketRouteOptions | ||
12 | } from '@server/types/plugins' | 14 | } from '@server/types/plugins' |
13 | import { | 15 | import { |
14 | EncoderOptionsBuilder, | 16 | EncoderOptionsBuilder, |
@@ -49,12 +51,15 @@ export class RegisterHelpers { | |||
49 | 51 | ||
50 | private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] | 52 | private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] |
51 | 53 | ||
54 | private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = [] | ||
55 | |||
52 | private readonly router: express.Router | 56 | private readonly router: express.Router |
53 | private readonly videoConstantManagerFactory: VideoConstantManagerFactory | 57 | private readonly videoConstantManagerFactory: VideoConstantManagerFactory |
54 | 58 | ||
55 | constructor ( | 59 | constructor ( |
56 | private readonly npmName: string, | 60 | private readonly npmName: string, |
57 | private readonly plugin: PluginModel, | 61 | private readonly plugin: PluginModel, |
62 | private readonly server: Server, | ||
58 | private readonly onHookAdded: (options: RegisterServerHookOptions) => void | 63 | private readonly onHookAdded: (options: RegisterServerHookOptions) => void |
59 | ) { | 64 | ) { |
60 | this.router = express.Router() | 65 | this.router = express.Router() |
@@ -66,6 +71,7 @@ export class RegisterHelpers { | |||
66 | const registerSetting = this.buildRegisterSetting() | 71 | const registerSetting = this.buildRegisterSetting() |
67 | 72 | ||
68 | const getRouter = this.buildGetRouter() | 73 | const getRouter = this.buildGetRouter() |
74 | const registerWebSocketRoute = this.buildRegisterWebSocketRoute() | ||
69 | 75 | ||
70 | const settingsManager = this.buildSettingsManager() | 76 | const settingsManager = this.buildSettingsManager() |
71 | const storageManager = this.buildStorageManager() | 77 | const storageManager = this.buildStorageManager() |
@@ -85,13 +91,14 @@ export class RegisterHelpers { | |||
85 | const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() | 91 | const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() |
86 | const unregisterExternalAuth = this.buildUnregisterExternalAuth() | 92 | const unregisterExternalAuth = this.buildUnregisterExternalAuth() |
87 | 93 | ||
88 | const peertubeHelpers = buildPluginHelpers(this.plugin, this.npmName) | 94 | const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName) |
89 | 95 | ||
90 | return { | 96 | return { |
91 | registerHook, | 97 | registerHook, |
92 | registerSetting, | 98 | registerSetting, |
93 | 99 | ||
94 | getRouter, | 100 | getRouter, |
101 | registerWebSocketRoute, | ||
95 | 102 | ||
96 | settingsManager, | 103 | settingsManager, |
97 | storageManager, | 104 | storageManager, |
@@ -180,10 +187,20 @@ export class RegisterHelpers { | |||
180 | return this.onSettingsChangeCallbacks | 187 | return this.onSettingsChangeCallbacks |
181 | } | 188 | } |
182 | 189 | ||
190 | getWebSocketRoutes () { | ||
191 | return this.webSocketRoutes | ||
192 | } | ||
193 | |||
183 | private buildGetRouter () { | 194 | private buildGetRouter () { |
184 | return () => this.router | 195 | return () => this.router |
185 | } | 196 | } |
186 | 197 | ||
198 | private buildRegisterWebSocketRoute () { | ||
199 | return (options: RegisterServerWebSocketRouteOptions) => { | ||
200 | this.webSocketRoutes.push(options) | ||
201 | } | ||
202 | } | ||
203 | |||
187 | private buildRegisterSetting () { | 204 | private buildRegisterSetting () { |
188 | return (options: RegisterServerSettingOptions) => { | 205 | return (options: RegisterServerSettingOptions) => { |
189 | this.settings.push(options) | 206 | this.settings.push(options) |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 9b3c72300..b7523492a 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -9,6 +9,7 @@ import { | |||
9 | CONTACT_FORM_LIFETIME, | 9 | CONTACT_FORM_LIFETIME, |
10 | RESUMABLE_UPLOAD_SESSION_LIFETIME, | 10 | RESUMABLE_UPLOAD_SESSION_LIFETIME, |
11 | TRACKER_RATE_LIMITS, | 11 | TRACKER_RATE_LIMITS, |
12 | TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, | ||
12 | USER_EMAIL_VERIFY_LIFETIME, | 13 | USER_EMAIL_VERIFY_LIFETIME, |
13 | USER_PASSWORD_CREATE_LIFETIME, | 14 | USER_PASSWORD_CREATE_LIFETIME, |
14 | USER_PASSWORD_RESET_LIFETIME, | 15 | USER_PASSWORD_RESET_LIFETIME, |
@@ -108,10 +109,24 @@ class Redis { | |||
108 | return this.removeValue(this.generateResetPasswordKey(userId)) | 109 | return this.removeValue(this.generateResetPasswordKey(userId)) |
109 | } | 110 | } |
110 | 111 | ||
111 | async getResetPasswordLink (userId: number) { | 112 | async getResetPasswordVerificationString (userId: number) { |
112 | return this.getValue(this.generateResetPasswordKey(userId)) | 113 | return this.getValue(this.generateResetPasswordKey(userId)) |
113 | } | 114 | } |
114 | 115 | ||
116 | /* ************ Two factor auth request ************ */ | ||
117 | |||
118 | async setTwoFactorRequest (userId: number, otpSecret: string) { | ||
119 | const requestToken = await generateRandomString(32) | ||
120 | |||
121 | await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME) | ||
122 | |||
123 | return requestToken | ||
124 | } | ||
125 | |||
126 | async getTwoFactorRequestToken (userId: number, requestToken: string) { | ||
127 | return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken)) | ||
128 | } | ||
129 | |||
115 | /* ************ Email verification ************ */ | 130 | /* ************ Email verification ************ */ |
116 | 131 | ||
117 | async setVerifyEmailVerificationString (userId: number) { | 132 | async setVerifyEmailVerificationString (userId: number) { |
@@ -342,6 +357,10 @@ class Redis { | |||
342 | return 'reset-password-' + userId | 357 | return 'reset-password-' + userId |
343 | } | 358 | } |
344 | 359 | ||
360 | private generateTwoFactorRequestKey (userId: number, token: string) { | ||
361 | return 'two-factor-request-' + userId + '-' + token | ||
362 | } | ||
363 | |||
345 | private generateVerifyEmailKey (userId: number) { | 364 | private generateVerifyEmailKey (userId: number) { |
346 | return 'verify-email-' + userId | 365 | return 'verify-email-' + userId |
347 | } | 366 | } |
@@ -391,8 +410,8 @@ class Redis { | |||
391 | return JSON.parse(value) | 410 | return JSON.parse(value) |
392 | } | 411 | } |
393 | 412 | ||
394 | private setObject (key: string, value: { [ id: string ]: number | string }) { | 413 | private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) { |
395 | return this.setValue(key, JSON.stringify(value)) | 414 | return this.setValue(key, JSON.stringify(value), expirationMilliseconds) |
396 | } | 415 | } |
397 | 416 | ||
398 | private async setValue (key: string, value: string, expirationMilliseconds?: number) { | 417 | private async setValue (key: string, value: string, expirationMilliseconds?: number) { |
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 5bfbc3cd2..e38685c04 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts | |||
@@ -1,11 +1,14 @@ | |||
1 | import { VideoModel } from '@server/models/video/video' | 1 | import { VideoModel } from '@server/models/video/video' |
2 | import { MVideoFullLight } from '@server/types/models' | 2 | import { MScheduleVideoUpdate } from '@server/types/models' |
3 | import { VideoPrivacy, VideoState } from '@shared/models' | ||
3 | import { logger } from '../../helpers/logger' | 4 | import { logger } from '../../helpers/logger' |
4 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
5 | import { sequelizeTypescript } from '../../initializers/database' | 6 | import { sequelizeTypescript } from '../../initializers/database' |
6 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' | 7 | import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' |
7 | import { federateVideoIfNeeded } from '../activitypub/videos' | ||
8 | import { Notifier } from '../notifier' | 8 | import { Notifier } from '../notifier' |
9 | import { addVideoJobsAfterUpdate } from '../video' | ||
10 | import { VideoPathManager } from '../video-path-manager' | ||
11 | import { setVideoPrivacy } from '../video-privacy' | ||
9 | import { AbstractScheduler } from './abstract-scheduler' | 12 | import { AbstractScheduler } from './abstract-scheduler' |
10 | 13 | ||
11 | export class UpdateVideosScheduler extends AbstractScheduler { | 14 | export class UpdateVideosScheduler extends AbstractScheduler { |
@@ -26,35 +29,58 @@ export class UpdateVideosScheduler extends AbstractScheduler { | |||
26 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined | 29 | if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined |
27 | 30 | ||
28 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() | 31 | const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() |
29 | const publishedVideos: MVideoFullLight[] = [] | ||
30 | 32 | ||
31 | for (const schedule of schedules) { | 33 | for (const schedule of schedules) { |
32 | await sequelizeTypescript.transaction(async t => { | 34 | const videoOnly = await VideoModel.load(schedule.videoId) |
33 | const video = await VideoModel.loadFull(schedule.videoId, t) | 35 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid) |
34 | 36 | ||
35 | logger.info('Executing scheduled video update on %s.', video.uuid) | 37 | try { |
38 | const { video, published } = await this.updateAVideo(schedule) | ||
36 | 39 | ||
37 | if (schedule.privacy) { | 40 | if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video) |
38 | const wasConfidentialVideo = video.isConfidential() | 41 | } catch (err) { |
39 | const isNewVideo = video.isNewVideo(schedule.privacy) | 42 | logger.error('Cannot update video', { err }) |
43 | } | ||
40 | 44 | ||
41 | video.setPrivacy(schedule.privacy) | 45 | mutexReleaser() |
42 | await video.save({ transaction: t }) | 46 | } |
43 | await federateVideoIfNeeded(video, isNewVideo, t) | 47 | } |
48 | |||
49 | private async updateAVideo (schedule: MScheduleVideoUpdate) { | ||
50 | let oldPrivacy: VideoPrivacy | ||
51 | let isNewVideo: boolean | ||
52 | let published = false | ||
53 | |||
54 | const video = await sequelizeTypescript.transaction(async t => { | ||
55 | const video = await VideoModel.loadFull(schedule.videoId, t) | ||
56 | if (video.state === VideoState.TO_TRANSCODE) return null | ||
57 | |||
58 | logger.info('Executing scheduled video update on %s.', video.uuid) | ||
59 | |||
60 | if (schedule.privacy) { | ||
61 | isNewVideo = video.isNewVideo(schedule.privacy) | ||
62 | oldPrivacy = video.privacy | ||
44 | 63 | ||
45 | if (wasConfidentialVideo) { | 64 | setVideoPrivacy(video, schedule.privacy) |
46 | publishedVideos.push(video) | 65 | await video.save({ transaction: t }) |
47 | } | 66 | |
67 | if (oldPrivacy === VideoPrivacy.PRIVATE) { | ||
68 | published = true | ||
48 | } | 69 | } |
70 | } | ||
49 | 71 | ||
50 | await schedule.destroy({ transaction: t }) | 72 | await schedule.destroy({ transaction: t }) |
51 | }) | 73 | |
52 | } | 74 | return video |
75 | }) | ||
53 | 76 | ||
54 | for (const v of publishedVideos) { | 77 | if (!video) { |
55 | Notifier.Instance.notifyOnNewVideoIfNeeded(v) | 78 | return { video, published: false } |
56 | Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v) | ||
57 | } | 79 | } |
80 | |||
81 | await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false }) | ||
82 | |||
83 | return { video, published } | ||
58 | } | 84 | } |
59 | 85 | ||
60 | static get Instance () { | 86 | static get Instance () { |
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts index a527f68b5..efb957fac 100644 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts | |||
@@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger' | |||
2 | import { CONFIG } from '@server/initializers/config' | 2 | import { CONFIG } from '@server/initializers/config' |
3 | import { VideoChannelModel } from '@server/models/video/video-channel' | 3 | import { VideoChannelModel } from '@server/models/video/video-channel' |
4 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | 4 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' |
5 | import { VideoChannelSyncState } from '@shared/models' | ||
6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
7 | import { synchronizeChannel } from '../sync-channel' | 6 | import { synchronizeChannel } from '../sync-channel' |
8 | import { AbstractScheduler } from './abstract-scheduler' | 7 | import { AbstractScheduler } from './abstract-scheduler' |
@@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler { | |||
28 | for (const sync of channelSyncs) { | 27 | for (const sync of channelSyncs) { |
29 | const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) | 28 | const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) |
30 | 29 | ||
31 | try { | 30 | logger.info( |
32 | logger.info( | 31 | 'Creating video import jobs for "%s" sync with external channel "%s"', |
33 | 'Creating video import jobs for "%s" sync with external channel "%s"', | 32 | channel.Actor.preferredUsername, sync.externalChannelUrl |
34 | channel.Actor.preferredUsername, sync.externalChannelUrl | 33 | ) |
35 | ) | 34 | |
36 | 35 | const onlyAfter = sync.lastSyncAt || sync.createdAt | |
37 | const onlyAfter = sync.lastSyncAt || sync.createdAt | 36 | |
38 | 37 | await synchronizeChannel({ | |
39 | await synchronizeChannel({ | 38 | channel, |
40 | channel, | 39 | externalChannelUrl: sync.externalChannelUrl, |
41 | externalChannelUrl: sync.externalChannelUrl, | 40 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, |
42 | videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, | 41 | channelSync: sync, |
43 | channelSync: sync, | 42 | onlyAfter |
44 | onlyAfter | 43 | }) |
45 | }) | ||
46 | } catch (err) { | ||
47 | logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err }) | ||
48 | sync.state = VideoChannelSyncState.FAILED | ||
49 | await sync.save() | ||
50 | } | ||
51 | } | 44 | } |
52 | } | 45 | } |
53 | 46 | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 91c217615..dc450c338 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -16,7 +16,7 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' | |||
16 | import { logger, loggerTagsFactory } from '../../helpers/logger' | 16 | import { logger, loggerTagsFactory } from '../../helpers/logger' |
17 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | 17 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' |
18 | import { CONFIG } from '../../initializers/config' | 18 | import { CONFIG } from '../../initializers/config' |
19 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' | 19 | import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' |
20 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 20 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
21 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 21 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
22 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' | 22 | import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
@@ -115,16 +115,29 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
115 | for (const redundancyModel of expired) { | 115 | for (const redundancyModel of expired) { |
116 | try { | 116 | try { |
117 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 117 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
118 | |||
119 | // If the admin disabled the redundancy, remove this redundancy instead of extending it | ||
120 | if (!redundancyConfig) { | ||
121 | logger.info( | ||
122 | 'Destroying redundancy %s because the redundancy %s does not exist anymore.', | ||
123 | redundancyModel.url, redundancyModel.strategy | ||
124 | ) | ||
125 | |||
126 | await removeVideoRedundancy(redundancyModel) | ||
127 | continue | ||
128 | } | ||
129 | |||
118 | const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) | 130 | const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) |
119 | 131 | ||
120 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | 132 | // If the admin decreased the cache size, remove this redundancy instead of extending it |
121 | if (!redundancyConfig || totalUsed > redundancyConfig.size) { | 133 | if (totalUsed > redundancyConfig.size) { |
122 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | 134 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) |
123 | 135 | ||
124 | await removeVideoRedundancy(redundancyModel) | 136 | await removeVideoRedundancy(redundancyModel) |
125 | } else { | 137 | continue |
126 | await this.extendsRedundancy(redundancyModel) | ||
127 | } | 138 | } |
139 | |||
140 | await this.extendsRedundancy(redundancyModel) | ||
128 | } catch (err) { | 141 | } catch (err) { |
129 | logger.error( | 142 | logger.error( |
130 | 'Cannot extend or remove expiration of %s video from our redundancy system.', | 143 | 'Cannot extend or remove expiration of %s video from our redundancy system.', |
@@ -262,7 +275,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
262 | 275 | ||
263 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) | 276 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) |
264 | 277 | ||
265 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | 278 | const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) |
266 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) | 279 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) |
267 | 280 | ||
268 | const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 | 281 | const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 |
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index f91599c14..10167ee38 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { logger } from '@server/helpers/logger' | 1 | import { logger } from '@server/helpers/logger' |
2 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' | 2 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' |
3 | import { CONFIG } from '@server/initializers/config' | 3 | import { CONFIG } from '@server/initializers/config' |
4 | import { buildYoutubeDLImport } from '@server/lib/video-import' | 4 | import { buildYoutubeDLImport } from '@server/lib/video-pre-import' |
5 | import { UserModel } from '@server/models/user/user' | 5 | import { UserModel } from '@server/models/user/user' |
6 | import { VideoImportModel } from '@server/models/video/video-import' | 6 | import { VideoImportModel } from '@server/models/video/video-import' |
7 | import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' | 7 | import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' |
@@ -12,8 +12,8 @@ import { ServerConfigManager } from './server-config-manager' | |||
12 | export async function synchronizeChannel (options: { | 12 | export async function synchronizeChannel (options: { |
13 | channel: MChannelAccountDefault | 13 | channel: MChannelAccountDefault |
14 | externalChannelUrl: string | 14 | externalChannelUrl: string |
15 | videosCountLimit: number | ||
15 | channelSync?: MChannelSync | 16 | channelSync?: MChannelSync |
16 | videosCountLimit?: number | ||
17 | onlyAfter?: Date | 17 | onlyAfter?: Date |
18 | }) { | 18 | }) { |
19 | const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options | 19 | const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options |
@@ -24,56 +24,62 @@ export async function synchronizeChannel (options: { | |||
24 | await channelSync.save() | 24 | await channelSync.save() |
25 | } | 25 | } |
26 | 26 | ||
27 | const user = await UserModel.loadByChannelActorId(channel.actorId) | 27 | try { |
28 | const youtubeDL = new YoutubeDLWrapper( | 28 | const user = await UserModel.loadByChannelActorId(channel.actorId) |
29 | externalChannelUrl, | 29 | const youtubeDL = new YoutubeDLWrapper( |
30 | ServerConfigManager.Instance.getEnabledResolutions('vod'), | 30 | externalChannelUrl, |
31 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION | 31 | ServerConfigManager.Instance.getEnabledResolutions('vod'), |
32 | ) | 32 | CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION |
33 | ) | ||
33 | 34 | ||
34 | const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) | 35 | const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) |
35 | 36 | ||
36 | logger.info( | 37 | logger.info( |
37 | 'Fetched %d candidate URLs for sync channel %s.', | 38 | 'Fetched %d candidate URLs for sync channel %s.', |
38 | targetUrls.length, channel.Actor.preferredUsername, { targetUrls } | 39 | targetUrls.length, channel.Actor.preferredUsername, { targetUrls } |
39 | ) | 40 | ) |
40 | 41 | ||
41 | if (targetUrls.length === 0) { | 42 | if (targetUrls.length === 0) { |
42 | if (channelSync) { | 43 | if (channelSync) { |
43 | channelSync.state = VideoChannelSyncState.SYNCED | 44 | channelSync.state = VideoChannelSyncState.SYNCED |
44 | await channelSync.save() | 45 | await channelSync.save() |
46 | } | ||
47 | |||
48 | return | ||
45 | } | 49 | } |
46 | 50 | ||
47 | return | 51 | const children: CreateJobArgument[] = [] |
48 | } | ||
49 | 52 | ||
50 | const children: CreateJobArgument[] = [] | 53 | for (const targetUrl of targetUrls) { |
54 | if (await skipImport(channel, targetUrl, onlyAfter)) continue | ||
51 | 55 | ||
52 | for (const targetUrl of targetUrls) { | 56 | const { job } = await buildYoutubeDLImport({ |
53 | if (await skipImport(channel, targetUrl, onlyAfter)) continue | 57 | user, |
58 | channel, | ||
59 | targetUrl, | ||
60 | channelSync, | ||
61 | importDataOverride: { | ||
62 | privacy: VideoPrivacy.PUBLIC | ||
63 | } | ||
64 | }) | ||
54 | 65 | ||
55 | const { job } = await buildYoutubeDLImport({ | 66 | children.push(job) |
56 | user, | 67 | } |
57 | channel, | ||
58 | targetUrl, | ||
59 | channelSync, | ||
60 | importDataOverride: { | ||
61 | privacy: VideoPrivacy.PUBLIC | ||
62 | } | ||
63 | }) | ||
64 | |||
65 | children.push(job) | ||
66 | } | ||
67 | 68 | ||
68 | // Will update the channel sync status | 69 | // Will update the channel sync status |
69 | const parent: CreateJobArgument = { | 70 | const parent: CreateJobArgument = { |
70 | type: 'after-video-channel-import', | 71 | type: 'after-video-channel-import', |
71 | payload: { | 72 | payload: { |
72 | channelSyncId: channelSync?.id | 73 | channelSyncId: channelSync?.id |
74 | } | ||
73 | } | 75 | } |
74 | } | ||
75 | 76 | ||
76 | await JobQueue.Instance.createJobWithChildren(parent, children) | 77 | await JobQueue.Instance.createJobWithChildren(parent, children) |
78 | } catch (err) { | ||
79 | logger.error(`Failed to import channel ${channel.name}`, { err }) | ||
80 | channelSync.state = VideoChannelSyncState.FAILED | ||
81 | await channelSync.save() | ||
82 | } | ||
77 | } | 83 | } |
78 | 84 | ||
79 | // --------------------------------------------------------------------------- | 85 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts index 44e26754d..c7b61e9ba 100644 --- a/server/lib/transcoding/transcoding.ts +++ b/server/lib/transcoding/transcoding.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import { MutexInterface } from 'async-mutex' | ||
1 | import { Job } from 'bullmq' | 2 | import { Job } from 'bullmq' |
2 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' | 3 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' |
3 | import { basename, extname as extnameUtil, join } from 'path' | 4 | import { basename, extname as extnameUtil, join } from 'path' |
@@ -6,11 +7,13 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils' | |||
6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
7 | import { sequelizeTypescript } from '@server/initializers/database' | 8 | import { sequelizeTypescript } from '@server/initializers/database' |
8 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 9 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
10 | import { pick } from '@shared/core-utils' | ||
9 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' | 11 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' |
10 | import { | 12 | import { |
11 | buildFileMetadata, | 13 | buildFileMetadata, |
12 | canDoQuickTranscode, | 14 | canDoQuickTranscode, |
13 | computeResolutionsToTranscode, | 15 | computeResolutionsToTranscode, |
16 | ffprobePromise, | ||
14 | getVideoStreamDuration, | 17 | getVideoStreamDuration, |
15 | getVideoStreamFPS, | 18 | getVideoStreamFPS, |
16 | transcodeVOD, | 19 | transcodeVOD, |
@@ -33,7 +36,7 @@ import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' | |||
33 | */ | 36 | */ |
34 | 37 | ||
35 | // Optimize the original video file and replace it. The resolution is not changed. | 38 | // Optimize the original video file and replace it. The resolution is not changed. |
36 | function optimizeOriginalVideofile (options: { | 39 | async function optimizeOriginalVideofile (options: { |
37 | video: MVideoFullLight | 40 | video: MVideoFullLight |
38 | inputVideoFile: MVideoFile | 41 | inputVideoFile: MVideoFile |
39 | job: Job | 42 | job: Job |
@@ -43,49 +46,62 @@ function optimizeOriginalVideofile (options: { | |||
43 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 46 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
44 | const newExtname = '.mp4' | 47 | const newExtname = '.mp4' |
45 | 48 | ||
46 | return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => { | 49 | // Will be released by our transcodeVOD function once ffmpeg is ran |
47 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 50 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
48 | 51 | ||
49 | const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) | 52 | try { |
50 | ? 'quick-transcode' | 53 | await video.reload() |
51 | : 'video' | ||
52 | 54 | ||
53 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) | 55 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) |
54 | 56 | ||
55 | const transcodeOptions: TranscodeVODOptions = { | 57 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { |
56 | type: transcodeType, | 58 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
57 | 59 | ||
58 | inputPath: videoInputPath, | 60 | const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) |
59 | outputPath: videoTranscodedPath, | 61 | ? 'quick-transcode' |
62 | : 'video' | ||
60 | 63 | ||
61 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 64 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) |
62 | profile: CONFIG.TRANSCODING.PROFILE, | ||
63 | 65 | ||
64 | resolution, | 66 | const transcodeOptions: TranscodeVODOptions = { |
67 | type: transcodeType, | ||
65 | 68 | ||
66 | job | 69 | inputPath: videoInputPath, |
67 | } | 70 | outputPath: videoTranscodedPath, |
68 | 71 | ||
69 | // Could be very long! | 72 | inputFileMutexReleaser, |
70 | await transcodeVOD(transcodeOptions) | ||
71 | 73 | ||
72 | // Important to do this before getVideoFilename() to take in account the new filename | 74 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
73 | inputVideoFile.resolution = resolution | 75 | profile: CONFIG.TRANSCODING.PROFILE, |
74 | inputVideoFile.extname = newExtname | ||
75 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | ||
76 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
77 | 76 | ||
78 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) | 77 | resolution, |
79 | 78 | ||
80 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 79 | job |
81 | await remove(videoInputPath) | 80 | } |
82 | 81 | ||
83 | return { transcodeType, videoFile } | 82 | // Could be very long! |
84 | }) | 83 | await transcodeVOD(transcodeOptions) |
84 | |||
85 | // Important to do this before getVideoFilename() to take in account the new filename | ||
86 | inputVideoFile.resolution = resolution | ||
87 | inputVideoFile.extname = newExtname | ||
88 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | ||
89 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
90 | |||
91 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | ||
92 | await remove(videoInputPath) | ||
93 | |||
94 | return { transcodeType, videoFile } | ||
95 | }) | ||
96 | |||
97 | return result | ||
98 | } finally { | ||
99 | inputFileMutexReleaser() | ||
100 | } | ||
85 | } | 101 | } |
86 | 102 | ||
87 | // Transcode the original video file to a lower resolution compatible with WebTorrent | 103 | // Transcode the original video file to a lower resolution compatible with WebTorrent |
88 | function transcodeNewWebTorrentResolution (options: { | 104 | async function transcodeNewWebTorrentResolution (options: { |
89 | video: MVideoFullLight | 105 | video: MVideoFullLight |
90 | resolution: VideoResolution | 106 | resolution: VideoResolution |
91 | job: Job | 107 | job: Job |
@@ -95,53 +111,68 @@ function transcodeNewWebTorrentResolution (options: { | |||
95 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 111 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
96 | const newExtname = '.mp4' | 112 | const newExtname = '.mp4' |
97 | 113 | ||
98 | return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => { | 114 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
99 | const newVideoFile = new VideoFileModel({ | ||
100 | resolution, | ||
101 | extname: newExtname, | ||
102 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | ||
103 | size: 0, | ||
104 | videoId: video.id | ||
105 | }) | ||
106 | 115 | ||
107 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) | 116 | try { |
108 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) | 117 | await video.reload() |
109 | 118 | ||
110 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO | 119 | const file = video.getMaxQualityFile().withVideoOrPlaylist(video) |
111 | ? { | ||
112 | type: 'only-audio' as 'only-audio', | ||
113 | 120 | ||
114 | inputPath: videoInputPath, | 121 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { |
115 | outputPath: videoTranscodedPath, | 122 | const newVideoFile = new VideoFileModel({ |
123 | resolution, | ||
124 | extname: newExtname, | ||
125 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | ||
126 | size: 0, | ||
127 | videoId: video.id | ||
128 | }) | ||
116 | 129 | ||
117 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 130 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) |
118 | profile: CONFIG.TRANSCODING.PROFILE, | ||
119 | 131 | ||
120 | resolution, | 132 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO |
133 | ? { | ||
134 | type: 'only-audio' as 'only-audio', | ||
121 | 135 | ||
122 | job | 136 | inputPath: videoInputPath, |
123 | } | 137 | outputPath: videoTranscodedPath, |
124 | : { | ||
125 | type: 'video' as 'video', | ||
126 | inputPath: videoInputPath, | ||
127 | outputPath: videoTranscodedPath, | ||
128 | 138 | ||
129 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 139 | inputFileMutexReleaser, |
130 | profile: CONFIG.TRANSCODING.PROFILE, | ||
131 | 140 | ||
132 | resolution, | 141 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
142 | profile: CONFIG.TRANSCODING.PROFILE, | ||
133 | 143 | ||
134 | job | 144 | resolution, |
135 | } | ||
136 | 145 | ||
137 | await transcodeVOD(transcodeOptions) | 146 | job |
147 | } | ||
148 | : { | ||
149 | type: 'video' as 'video', | ||
150 | inputPath: videoInputPath, | ||
151 | outputPath: videoTranscodedPath, | ||
138 | 152 | ||
139 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) | 153 | inputFileMutexReleaser, |
140 | }) | 154 | |
155 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
156 | profile: CONFIG.TRANSCODING.PROFILE, | ||
157 | |||
158 | resolution, | ||
159 | |||
160 | job | ||
161 | } | ||
162 | |||
163 | await transcodeVOD(transcodeOptions) | ||
164 | |||
165 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile) | ||
166 | }) | ||
167 | |||
168 | return result | ||
169 | } finally { | ||
170 | inputFileMutexReleaser() | ||
171 | } | ||
141 | } | 172 | } |
142 | 173 | ||
143 | // Merge an image with an audio file to create a video | 174 | // Merge an image with an audio file to create a video |
144 | function mergeAudioVideofile (options: { | 175 | async function mergeAudioVideofile (options: { |
145 | video: MVideoFullLight | 176 | video: MVideoFullLight |
146 | resolution: VideoResolution | 177 | resolution: VideoResolution |
147 | job: Job | 178 | job: Job |
@@ -151,54 +182,67 @@ function mergeAudioVideofile (options: { | |||
151 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 182 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
152 | const newExtname = '.mp4' | 183 | const newExtname = '.mp4' |
153 | 184 | ||
154 | const inputVideoFile = video.getMinQualityFile() | 185 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
155 | 186 | ||
156 | return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => { | 187 | try { |
157 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 188 | await video.reload() |
158 | 189 | ||
159 | // If the user updates the video preview during transcoding | 190 | const inputVideoFile = video.getMinQualityFile() |
160 | const previewPath = video.getPreview().getPath() | ||
161 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | ||
162 | await copyFile(previewPath, tmpPreviewPath) | ||
163 | 191 | ||
164 | const transcodeOptions = { | 192 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) |
165 | type: 'merge-audio' as 'merge-audio', | ||
166 | 193 | ||
167 | inputPath: tmpPreviewPath, | 194 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { |
168 | outputPath: videoTranscodedPath, | 195 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
169 | 196 | ||
170 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 197 | // If the user updates the video preview during transcoding |
171 | profile: CONFIG.TRANSCODING.PROFILE, | 198 | const previewPath = video.getPreview().getPath() |
199 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | ||
200 | await copyFile(previewPath, tmpPreviewPath) | ||
172 | 201 | ||
173 | audioPath: audioInputPath, | 202 | const transcodeOptions = { |
174 | resolution, | 203 | type: 'merge-audio' as 'merge-audio', |
175 | 204 | ||
176 | job | 205 | inputPath: tmpPreviewPath, |
177 | } | 206 | outputPath: videoTranscodedPath, |
178 | 207 | ||
179 | try { | 208 | inputFileMutexReleaser, |
180 | await transcodeVOD(transcodeOptions) | ||
181 | 209 | ||
182 | await remove(audioInputPath) | 210 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
183 | await remove(tmpPreviewPath) | 211 | profile: CONFIG.TRANSCODING.PROFILE, |
184 | } catch (err) { | ||
185 | await remove(tmpPreviewPath) | ||
186 | throw err | ||
187 | } | ||
188 | 212 | ||
189 | // Important to do this before getVideoFilename() to take in account the new file extension | 213 | audioPath: audioInputPath, |
190 | inputVideoFile.extname = newExtname | 214 | resolution, |
191 | inputVideoFile.resolution = resolution | ||
192 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | ||
193 | 215 | ||
194 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) | 216 | job |
195 | // ffmpeg generated a new video file, so update the video duration | 217 | } |
196 | // See https://trac.ffmpeg.org/ticket/5456 | ||
197 | video.duration = await getVideoStreamDuration(videoTranscodedPath) | ||
198 | await video.save() | ||
199 | 218 | ||
200 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 219 | try { |
201 | }) | 220 | await transcodeVOD(transcodeOptions) |
221 | |||
222 | await remove(audioInputPath) | ||
223 | await remove(tmpPreviewPath) | ||
224 | } catch (err) { | ||
225 | await remove(tmpPreviewPath) | ||
226 | throw err | ||
227 | } | ||
228 | |||
229 | // Important to do this before getVideoFilename() to take in account the new file extension | ||
230 | inputVideoFile.extname = newExtname | ||
231 | inputVideoFile.resolution = resolution | ||
232 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | ||
233 | |||
234 | // ffmpeg generated a new video file, so update the video duration | ||
235 | // See https://trac.ffmpeg.org/ticket/5456 | ||
236 | video.duration = await getVideoStreamDuration(videoTranscodedPath) | ||
237 | await video.save() | ||
238 | |||
239 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | ||
240 | }) | ||
241 | |||
242 | return result | ||
243 | } finally { | ||
244 | inputFileMutexReleaser() | ||
245 | } | ||
202 | } | 246 | } |
203 | 247 | ||
204 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | 248 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist |
@@ -207,13 +251,13 @@ async function generateHlsPlaylistResolutionFromTS (options: { | |||
207 | concatenatedTsFilePath: string | 251 | concatenatedTsFilePath: string |
208 | resolution: VideoResolution | 252 | resolution: VideoResolution |
209 | isAAC: boolean | 253 | isAAC: boolean |
254 | inputFileMutexReleaser: MutexInterface.Releaser | ||
210 | }) { | 255 | }) { |
211 | return generateHlsPlaylistCommon({ | 256 | return generateHlsPlaylistCommon({ |
212 | video: options.video, | ||
213 | resolution: options.resolution, | ||
214 | inputPath: options.concatenatedTsFilePath, | ||
215 | type: 'hls-from-ts' as 'hls-from-ts', | 257 | type: 'hls-from-ts' as 'hls-from-ts', |
216 | isAAC: options.isAAC | 258 | inputPath: options.concatenatedTsFilePath, |
259 | |||
260 | ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ]) | ||
217 | }) | 261 | }) |
218 | } | 262 | } |
219 | 263 | ||
@@ -223,15 +267,14 @@ function generateHlsPlaylistResolution (options: { | |||
223 | videoInputPath: string | 267 | videoInputPath: string |
224 | resolution: VideoResolution | 268 | resolution: VideoResolution |
225 | copyCodecs: boolean | 269 | copyCodecs: boolean |
270 | inputFileMutexReleaser: MutexInterface.Releaser | ||
226 | job?: Job | 271 | job?: Job |
227 | }) { | 272 | }) { |
228 | return generateHlsPlaylistCommon({ | 273 | return generateHlsPlaylistCommon({ |
229 | video: options.video, | ||
230 | resolution: options.resolution, | ||
231 | copyCodecs: options.copyCodecs, | ||
232 | inputPath: options.videoInputPath, | ||
233 | type: 'hls' as 'hls', | 274 | type: 'hls' as 'hls', |
234 | job: options.job | 275 | inputPath: options.videoInputPath, |
276 | |||
277 | ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) | ||
235 | }) | 278 | }) |
236 | } | 279 | } |
237 | 280 | ||
@@ -251,27 +294,39 @@ async function onWebTorrentVideoFileTranscoding ( | |||
251 | video: MVideoFullLight, | 294 | video: MVideoFullLight, |
252 | videoFile: MVideoFile, | 295 | videoFile: MVideoFile, |
253 | transcodingPath: string, | 296 | transcodingPath: string, |
254 | outputPath: string | 297 | newVideoFile: MVideoFile |
255 | ) { | 298 | ) { |
256 | const stats = await stat(transcodingPath) | 299 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
257 | const fps = await getVideoStreamFPS(transcodingPath) | ||
258 | const metadata = await buildFileMetadata(transcodingPath) | ||
259 | 300 | ||
260 | await move(transcodingPath, outputPath, { overwrite: true }) | 301 | try { |
302 | await video.reload() | ||
261 | 303 | ||
262 | videoFile.size = stats.size | 304 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
263 | videoFile.fps = fps | ||
264 | videoFile.metadata = metadata | ||
265 | 305 | ||
266 | await createTorrentAndSetInfoHash(video, videoFile) | 306 | const stats = await stat(transcodingPath) |
267 | 307 | ||
268 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | 308 | const probe = await ffprobePromise(transcodingPath) |
269 | if (oldFile) await video.removeWebTorrentFile(oldFile) | 309 | const fps = await getVideoStreamFPS(transcodingPath, probe) |
310 | const metadata = await buildFileMetadata(transcodingPath, probe) | ||
270 | 311 | ||
271 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | 312 | await move(transcodingPath, outputPath, { overwrite: true }) |
272 | video.VideoFiles = await video.$get('VideoFiles') | ||
273 | 313 | ||
274 | return { video, videoFile } | 314 | videoFile.size = stats.size |
315 | videoFile.fps = fps | ||
316 | videoFile.metadata = metadata | ||
317 | |||
318 | await createTorrentAndSetInfoHash(video, videoFile) | ||
319 | |||
320 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | ||
321 | if (oldFile) await video.removeWebTorrentFile(oldFile) | ||
322 | |||
323 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | ||
324 | video.VideoFiles = await video.$get('VideoFiles') | ||
325 | |||
326 | return { video, videoFile } | ||
327 | } finally { | ||
328 | mutexReleaser() | ||
329 | } | ||
275 | } | 330 | } |
276 | 331 | ||
277 | async function generateHlsPlaylistCommon (options: { | 332 | async function generateHlsPlaylistCommon (options: { |
@@ -279,12 +334,15 @@ async function generateHlsPlaylistCommon (options: { | |||
279 | video: MVideo | 334 | video: MVideo |
280 | inputPath: string | 335 | inputPath: string |
281 | resolution: VideoResolution | 336 | resolution: VideoResolution |
337 | |||
338 | inputFileMutexReleaser: MutexInterface.Releaser | ||
339 | |||
282 | copyCodecs?: boolean | 340 | copyCodecs?: boolean |
283 | isAAC?: boolean | 341 | isAAC?: boolean |
284 | 342 | ||
285 | job?: Job | 343 | job?: Job |
286 | }) { | 344 | }) { |
287 | const { type, video, inputPath, resolution, copyCodecs, isAAC, job } = options | 345 | const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options |
288 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 346 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
289 | 347 | ||
290 | const videoTranscodedBasePath = join(transcodeDirectory, type) | 348 | const videoTranscodedBasePath = join(transcodeDirectory, type) |
@@ -308,6 +366,8 @@ async function generateHlsPlaylistCommon (options: { | |||
308 | 366 | ||
309 | isAAC, | 367 | isAAC, |
310 | 368 | ||
369 | inputFileMutexReleaser, | ||
370 | |||
311 | hlsPlaylist: { | 371 | hlsPlaylist: { |
312 | videoFilename | 372 | videoFilename |
313 | }, | 373 | }, |
@@ -333,47 +393,73 @@ async function generateHlsPlaylistCommon (options: { | |||
333 | videoStreamingPlaylistId: playlist.id | 393 | videoStreamingPlaylistId: playlist.id |
334 | }) | 394 | }) |
335 | 395 | ||
336 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) | 396 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
337 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | ||
338 | 397 | ||
339 | // Move playlist file | 398 | try { |
340 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) | 399 | // VOD transcoding is a long task, refresh video attributes |
341 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | 400 | await video.reload() |
342 | // Move video file | ||
343 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | ||
344 | 401 | ||
345 | // Update video duration if it was not set (in case of a live for example) | 402 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) |
346 | if (!video.duration) { | 403 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) |
347 | video.duration = await getVideoStreamDuration(videoFilePath) | ||
348 | await video.save() | ||
349 | } | ||
350 | 404 | ||
351 | const stats = await stat(videoFilePath) | 405 | // Move playlist file |
406 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) | ||
407 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | ||
408 | // Move video file | ||
409 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | ||
352 | 410 | ||
353 | newVideoFile.size = stats.size | 411 | // Update video duration if it was not set (in case of a live for example) |
354 | newVideoFile.fps = await getVideoStreamFPS(videoFilePath) | 412 | if (!video.duration) { |
355 | newVideoFile.metadata = await buildFileMetadata(videoFilePath) | 413 | video.duration = await getVideoStreamDuration(videoFilePath) |
414 | await video.save() | ||
415 | } | ||
356 | 416 | ||
357 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | 417 | const stats = await stat(videoFilePath) |
358 | 418 | ||
359 | const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) | 419 | newVideoFile.size = stats.size |
360 | if (oldFile) { | 420 | newVideoFile.fps = await getVideoStreamFPS(videoFilePath) |
361 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | 421 | newVideoFile.metadata = await buildFileMetadata(videoFilePath) |
362 | await oldFile.destroy() | ||
363 | } | ||
364 | 422 | ||
365 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | 423 | await createTorrentAndSetInfoHash(playlist, newVideoFile) |
366 | 424 | ||
367 | await updatePlaylistAfterFileChange(video, playlist) | 425 | const oldFile = await VideoFileModel.loadHLSFile({ |
426 | playlistId: playlist.id, | ||
427 | fps: newVideoFile.fps, | ||
428 | resolution: newVideoFile.resolution | ||
429 | }) | ||
430 | |||
431 | if (oldFile) { | ||
432 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | ||
433 | await oldFile.destroy() | ||
434 | } | ||
435 | |||
436 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | ||
437 | |||
438 | await updatePlaylistAfterFileChange(video, playlist) | ||
368 | 439 | ||
369 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | 440 | return { resolutionPlaylistPath, videoFile: savedVideoFile } |
441 | } finally { | ||
442 | mutexReleaser() | ||
443 | } | ||
370 | } | 444 | } |
371 | 445 | ||
372 | function buildOriginalFileResolution (inputResolution: number) { | 446 | function buildOriginalFileResolution (inputResolution: number) { |
373 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution) | 447 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { |
448 | return toEven(inputResolution) | ||
449 | } | ||
374 | 450 | ||
375 | const resolutions = computeResolutionsToTranscode({ input: inputResolution, type: 'vod', includeInput: false, strictLower: false }) | 451 | const resolutions = computeResolutionsToTranscode({ |
376 | if (resolutions.length === 0) return toEven(inputResolution) | 452 | input: inputResolution, |
453 | type: 'vod', | ||
454 | includeInput: false, | ||
455 | strictLower: false, | ||
456 | // We don't really care about the audio resolution in this context | ||
457 | hasAudio: true | ||
458 | }) | ||
459 | |||
460 | if (resolutions.length === 0) { | ||
461 | return toEven(inputResolution) | ||
462 | } | ||
377 | 463 | ||
378 | return Math.max(...resolutions) | 464 | return Math.max(...resolutions) |
379 | } | 465 | } |
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts index 9484eff75..58040cb6d 100644 --- a/server/lib/uploadx.ts +++ b/server/lib/uploadx.ts | |||
@@ -1,6 +1,10 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { buildLogger } from '@server/helpers/logger' | ||
2 | import { getResumableUploadPath } from '@server/helpers/upload' | 3 | import { getResumableUploadPath } from '@server/helpers/upload' |
3 | import { Uploadx } from '@uploadx/core' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { LogLevel, Uploadx } from '@uploadx/core' | ||
6 | |||
7 | const logger = buildLogger('uploadx') | ||
4 | 8 | ||
5 | const uploadx = new Uploadx({ | 9 | const uploadx = new Uploadx({ |
6 | directory: getResumableUploadPath(), | 10 | directory: getResumableUploadPath(), |
@@ -10,6 +14,14 @@ const uploadx = new Uploadx({ | |||
10 | // Could be big with thumbnails/previews | 14 | // Could be big with thumbnails/previews |
11 | maxMetadataSize: '10MB', | 15 | maxMetadataSize: '10MB', |
12 | 16 | ||
17 | logger: { | ||
18 | logLevel: CONFIG.LOG.LEVEL as LogLevel, | ||
19 | debug: logger.debug.bind(logger), | ||
20 | info: logger.info.bind(logger), | ||
21 | warn: logger.warn.bind(logger), | ||
22 | error: logger.error.bind(logger) | ||
23 | }, | ||
24 | |||
13 | userIdentifier: (_, res: express.Response) => { | 25 | userIdentifier: (_, res: express.Response) => { |
14 | if (!res.locals.oauth) return undefined | 26 | if (!res.locals.oauth) return undefined |
15 | 27 | ||
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts index c3f55fd95..9953cae5d 100644 --- a/server/lib/video-path-manager.ts +++ b/server/lib/video-path-manager.ts | |||
@@ -1,29 +1,31 @@ | |||
1 | import { Mutex } from 'async-mutex' | ||
1 | import { remove } from 'fs-extra' | 2 | import { remove } from 'fs-extra' |
2 | import { extname, join } from 'path' | 3 | import { extname, join } from 'path' |
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { extractVideo } from '@server/helpers/video' | 5 | import { extractVideo } from '@server/helpers/video' |
4 | import { CONFIG } from '@server/initializers/config' | 6 | import { CONFIG } from '@server/initializers/config' |
5 | import { | 7 | import { DIRECTORIES } from '@server/initializers/constants' |
6 | MStreamingPlaylistVideo, | 8 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models' |
7 | MVideo, | ||
8 | MVideoFile, | ||
9 | MVideoFileStreamingPlaylistVideo, | ||
10 | MVideoFileVideo, | ||
11 | MVideoUUID | ||
12 | } from '@server/types/models' | ||
13 | import { buildUUID } from '@shared/extra-utils' | 9 | import { buildUUID } from '@shared/extra-utils' |
14 | import { VideoStorage } from '@shared/models' | 10 | import { VideoStorage } from '@shared/models' |
15 | import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' | 11 | import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' |
16 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' | 12 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' |
13 | import { isVideoInPrivateDirectory } from './video-privacy' | ||
17 | 14 | ||
18 | type MakeAvailableCB <T> = (path: string) => Promise<T> | T | 15 | type MakeAvailableCB <T> = (path: string) => Promise<T> | T |
19 | 16 | ||
17 | const lTags = loggerTagsFactory('video-path-manager') | ||
18 | |||
20 | class VideoPathManager { | 19 | class VideoPathManager { |
21 | 20 | ||
22 | private static instance: VideoPathManager | 21 | private static instance: VideoPathManager |
23 | 22 | ||
23 | // Key is a video UUID | ||
24 | private readonly videoFileMutexStore = new Map<string, Mutex>() | ||
25 | |||
24 | private constructor () {} | 26 | private constructor () {} |
25 | 27 | ||
26 | getFSHLSOutputPath (video: MVideoUUID, filename?: string) { | 28 | getFSHLSOutputPath (video: MVideo, filename?: string) { |
27 | const base = getHLSDirectory(video) | 29 | const base = getHLSDirectory(video) |
28 | if (!filename) return base | 30 | if (!filename) return base |
29 | 31 | ||
@@ -41,13 +43,17 @@ class VideoPathManager { | |||
41 | } | 43 | } |
42 | 44 | ||
43 | getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | 45 | getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { |
44 | if (videoFile.isHLS()) { | 46 | const video = extractVideo(videoOrPlaylist) |
45 | const video = extractVideo(videoOrPlaylist) | ||
46 | 47 | ||
48 | if (videoFile.isHLS()) { | ||
47 | return join(getHLSDirectory(video), videoFile.filename) | 49 | return join(getHLSDirectory(video), videoFile.filename) |
48 | } | 50 | } |
49 | 51 | ||
50 | return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) | 52 | if (isVideoInPrivateDirectory(video.privacy)) { |
53 | return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename) | ||
54 | } | ||
55 | |||
56 | return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename) | ||
51 | } | 57 | } |
52 | 58 | ||
53 | async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { | 59 | async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) { |
@@ -113,6 +119,27 @@ class VideoPathManager { | |||
113 | ) | 119 | ) |
114 | } | 120 | } |
115 | 121 | ||
122 | async lockFiles (videoUUID: string) { | ||
123 | if (!this.videoFileMutexStore.has(videoUUID)) { | ||
124 | this.videoFileMutexStore.set(videoUUID, new Mutex()) | ||
125 | } | ||
126 | |||
127 | const mutex = this.videoFileMutexStore.get(videoUUID) | ||
128 | const releaser = await mutex.acquire() | ||
129 | |||
130 | logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID)) | ||
131 | |||
132 | return releaser | ||
133 | } | ||
134 | |||
135 | unlockFiles (videoUUID: string) { | ||
136 | const mutex = this.videoFileMutexStore.get(videoUUID) | ||
137 | |||
138 | mutex.release() | ||
139 | |||
140 | logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID)) | ||
141 | } | ||
142 | |||
116 | private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { | 143 | private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { |
117 | let result: T | 144 | let result: T |
118 | 145 | ||
diff --git a/server/lib/video-import.ts b/server/lib/video-pre-import.ts index 796079875..796079875 100644 --- a/server/lib/video-import.ts +++ b/server/lib/video-pre-import.ts | |||
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts new file mode 100644 index 000000000..41f9d62b3 --- /dev/null +++ b/server/lib/video-privacy.ts | |||
@@ -0,0 +1,127 @@ | |||
1 | import { move } from 'fs-extra' | ||
2 | import { join } from 'path' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { DIRECTORIES } from '@server/initializers/constants' | ||
5 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
6 | import { VideoPrivacy, VideoStorage } from '@shared/models' | ||
7 | import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage' | ||
8 | |||
9 | function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { | ||
10 | if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { | ||
11 | video.publishedAt = new Date() | ||
12 | } | ||
13 | |||
14 | video.privacy = newPrivacy | ||
15 | } | ||
16 | |||
17 | function isVideoInPrivateDirectory (privacy: VideoPrivacy) { | ||
18 | return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL | ||
19 | } | ||
20 | |||
21 | function isVideoInPublicDirectory (privacy: VideoPrivacy) { | ||
22 | return !isVideoInPrivateDirectory(privacy) | ||
23 | } | ||
24 | |||
25 | async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) { | ||
26 | // Now public, previously private | ||
27 | if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) { | ||
28 | await moveFiles({ type: 'private-to-public', video }) | ||
29 | |||
30 | return true | ||
31 | } | ||
32 | |||
33 | // Now private, previously public | ||
34 | if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) { | ||
35 | await moveFiles({ type: 'public-to-private', video }) | ||
36 | |||
37 | return true | ||
38 | } | ||
39 | |||
40 | return false | ||
41 | } | ||
42 | |||
43 | export { | ||
44 | setVideoPrivacy, | ||
45 | |||
46 | isVideoInPrivateDirectory, | ||
47 | isVideoInPublicDirectory, | ||
48 | |||
49 | moveFilesIfPrivacyChanged | ||
50 | } | ||
51 | |||
52 | // --------------------------------------------------------------------------- | ||
53 | |||
54 | type MoveType = 'private-to-public' | 'public-to-private' | ||
55 | |||
56 | async function moveFiles (options: { | ||
57 | type: MoveType | ||
58 | video: MVideoFullLight | ||
59 | }) { | ||
60 | const { type, video } = options | ||
61 | |||
62 | for (const file of video.VideoFiles) { | ||
63 | if (file.storage === VideoStorage.FILE_SYSTEM) { | ||
64 | await moveWebTorrentFileOnFS(type, video, file) | ||
65 | } else { | ||
66 | await updateWebTorrentFileACL(video, file) | ||
67 | } | ||
68 | } | ||
69 | |||
70 | const hls = video.getHLSPlaylist() | ||
71 | |||
72 | if (hls) { | ||
73 | if (hls.storage === VideoStorage.FILE_SYSTEM) { | ||
74 | await moveHLSFilesOnFS(type, video) | ||
75 | } else { | ||
76 | await updateHLSFilesACL(hls) | ||
77 | } | ||
78 | } | ||
79 | } | ||
80 | |||
81 | async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { | ||
82 | const directories = getWebTorrentDirectories(type) | ||
83 | |||
84 | const source = join(directories.old, file.filename) | ||
85 | const destination = join(directories.new, file.filename) | ||
86 | |||
87 | try { | ||
88 | logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | ||
89 | |||
90 | await move(source, destination) | ||
91 | } catch (err) { | ||
92 | logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) | ||
93 | } | ||
94 | } | ||
95 | |||
96 | function getWebTorrentDirectories (moveType: MoveType) { | ||
97 | if (moveType === 'private-to-public') { | ||
98 | return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } | ||
99 | } | ||
100 | |||
101 | return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } | ||
102 | } | ||
103 | |||
104 | // --------------------------------------------------------------------------- | ||
105 | |||
106 | async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { | ||
107 | const directories = getHLSDirectories(type) | ||
108 | |||
109 | const source = join(directories.old, video.uuid) | ||
110 | const destination = join(directories.new, video.uuid) | ||
111 | |||
112 | try { | ||
113 | logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | ||
114 | |||
115 | await move(source, destination) | ||
116 | } catch (err) { | ||
117 | logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) | ||
118 | } | ||
119 | } | ||
120 | |||
121 | function getHLSDirectories (moveType: MoveType) { | ||
122 | if (moveType === 'private-to-public') { | ||
123 | return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } | ||
124 | } | ||
125 | |||
126 | return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } | ||
127 | } | ||
diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts new file mode 100644 index 000000000..c43085d16 --- /dev/null +++ b/server/lib/video-tokens-manager.ts | |||
@@ -0,0 +1,49 @@ | |||
1 | import LRUCache from 'lru-cache' | ||
2 | import { LRU_CACHE } from '@server/initializers/constants' | ||
3 | import { buildUUID } from '@shared/extra-utils' | ||
4 | |||
5 | // --------------------------------------------------------------------------- | ||
6 | // Create temporary tokens that can be used as URL query parameters to access video static files | ||
7 | // --------------------------------------------------------------------------- | ||
8 | |||
9 | class VideoTokensManager { | ||
10 | |||
11 | private static instance: VideoTokensManager | ||
12 | |||
13 | private readonly lruCache = new LRUCache<string, string>({ | ||
14 | max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, | ||
15 | ttl: LRU_CACHE.VIDEO_TOKENS.TTL | ||
16 | }) | ||
17 | |||
18 | private constructor () {} | ||
19 | |||
20 | create (videoUUID: string) { | ||
21 | const token = buildUUID() | ||
22 | |||
23 | const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) | ||
24 | |||
25 | this.lruCache.set(token, videoUUID) | ||
26 | |||
27 | return { token, expires } | ||
28 | } | ||
29 | |||
30 | hasToken (options: { | ||
31 | token: string | ||
32 | videoUUID: string | ||
33 | }) { | ||
34 | const value = this.lruCache.get(options.token) | ||
35 | if (!value) return false | ||
36 | |||
37 | return value === options.videoUUID | ||
38 | } | ||
39 | |||
40 | static get Instance () { | ||
41 | return this.instance || (this.instance = new this()) | ||
42 | } | ||
43 | } | ||
44 | |||
45 | // --------------------------------------------------------------------------- | ||
46 | |||
47 | export { | ||
48 | VideoTokensManager | ||
49 | } | ||
diff --git a/server/lib/video.ts b/server/lib/video.ts index 6c4f3ce7b..aacc41a7a 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -7,10 +7,11 @@ import { TagModel } from '@server/models/video/tag' | |||
7 | import { VideoModel } from '@server/models/video/video' | 7 | import { VideoModel } from '@server/models/video/video' |
8 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 8 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
9 | import { FilteredModelAttributes } from '@server/types' | 9 | import { FilteredModelAttributes } from '@server/types' |
10 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | 10 | import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' |
11 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' | 11 | import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' |
12 | import { CreateJobOptions } from './job-queue/job-queue' | 12 | import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue' |
13 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 13 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
14 | import { moveFilesIfPrivacyChanged } from './video-privacy' | ||
14 | 15 | ||
15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 16 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
16 | return { | 17 | return { |
@@ -177,6 +178,59 @@ const getCachedVideoDuration = memoizee(getVideoDuration, { | |||
177 | 178 | ||
178 | // --------------------------------------------------------------------------- | 179 | // --------------------------------------------------------------------------- |
179 | 180 | ||
181 | async function addVideoJobsAfterUpdate (options: { | ||
182 | video: MVideoFullLight | ||
183 | isNewVideo: boolean | ||
184 | |||
185 | nameChanged: boolean | ||
186 | oldPrivacy: VideoPrivacy | ||
187 | }) { | ||
188 | const { video, nameChanged, oldPrivacy, isNewVideo } = options | ||
189 | const jobs: CreateJobArgument[] = [] | ||
190 | |||
191 | const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) | ||
192 | |||
193 | if (!video.isLive && (nameChanged || filePathChanged)) { | ||
194 | for (const file of (video.VideoFiles || [])) { | ||
195 | const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } | ||
196 | |||
197 | jobs.push({ type: 'manage-video-torrent', payload }) | ||
198 | } | ||
199 | |||
200 | const hls = video.getHLSPlaylist() | ||
201 | |||
202 | for (const file of (hls?.VideoFiles || [])) { | ||
203 | const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } | ||
204 | |||
205 | jobs.push({ type: 'manage-video-torrent', payload }) | ||
206 | } | ||
207 | } | ||
208 | |||
209 | jobs.push({ | ||
210 | type: 'federate-video', | ||
211 | payload: { | ||
212 | videoUUID: video.uuid, | ||
213 | isNewVideo | ||
214 | } | ||
215 | }) | ||
216 | |||
217 | const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy) | ||
218 | |||
219 | if (wasConfidentialVideo) { | ||
220 | jobs.push({ | ||
221 | type: 'notify', | ||
222 | payload: { | ||
223 | action: 'new-video', | ||
224 | videoUUID: video.uuid | ||
225 | } | ||
226 | }) | ||
227 | } | ||
228 | |||
229 | return JobQueue.Instance.createSequentialJobFlow(...jobs) | ||
230 | } | ||
231 | |||
232 | // --------------------------------------------------------------------------- | ||
233 | |||
180 | export { | 234 | export { |
181 | buildLocalVideoFromReq, | 235 | buildLocalVideoFromReq, |
182 | buildVideoThumbnailsFromReq, | 236 | buildVideoThumbnailsFromReq, |
@@ -185,5 +239,6 @@ export { | |||
185 | buildTranscodingJob, | 239 | buildTranscodingJob, |
186 | buildMoveToObjectStorageJob, | 240 | buildMoveToObjectStorageJob, |
187 | getTranscodingJobPriority, | 241 | getTranscodingJobPriority, |
242 | addVideoJobsAfterUpdate, | ||
188 | getCachedVideoDuration | 243 | getCachedVideoDuration |
189 | } | 244 | } |
diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts index 904d47efd..e6025c8ce 100644 --- a/server/middlewares/auth.ts +++ b/server/middlewares/auth.ts | |||
@@ -5,8 +5,8 @@ import { HttpStatusCode } from '../../shared/models/http/http-error-codes' | |||
5 | import { logger } from '../helpers/logger' | 5 | import { logger } from '../helpers/logger' |
6 | import { handleOAuthAuthenticate } from '../lib/auth/oauth' | 6 | import { handleOAuthAuthenticate } from '../lib/auth/oauth' |
7 | 7 | ||
8 | function authenticate (req: express.Request, res: express.Response, next: express.NextFunction, authenticateInQuery = false) { | 8 | function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) { |
9 | handleOAuthAuthenticate(req, res, authenticateInQuery) | 9 | handleOAuthAuthenticate(req, res) |
10 | .then((token: any) => { | 10 | .then((token: any) => { |
11 | res.locals.oauth = { token } | 11 | res.locals.oauth = { token } |
12 | res.locals.authenticated = true | 12 | res.locals.authenticated = true |
@@ -47,7 +47,7 @@ function authenticateSocket (socket: Socket, next: (err?: any) => void) { | |||
47 | .catch(err => logger.error('Cannot get access token.', { err })) | 47 | .catch(err => logger.error('Cannot get access token.', { err })) |
48 | } | 48 | } |
49 | 49 | ||
50 | function authenticatePromise (req: express.Request, res: express.Response, authenticateInQuery = false) { | 50 | function authenticatePromise (req: express.Request, res: express.Response) { |
51 | return new Promise<void>(resolve => { | 51 | return new Promise<void>(resolve => { |
52 | // Already authenticated? (or tried to) | 52 | // Already authenticated? (or tried to) |
53 | if (res.locals.oauth?.token.User) return resolve() | 53 | if (res.locals.oauth?.token.User) return resolve() |
@@ -59,7 +59,7 @@ function authenticatePromise (req: express.Request, res: express.Response, authe | |||
59 | }) | 59 | }) |
60 | } | 60 | } |
61 | 61 | ||
62 | authenticate(req, res, () => resolve(), authenticateInQuery) | 62 | authenticate(req, res, () => resolve()) |
63 | }) | 63 | }) |
64 | } | 64 | } |
65 | 65 | ||
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index ffadb3b49..9bc8887ff 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | export * from './activitypub' | ||
2 | export * from './videos' | ||
3 | export * from './abuse' | 1 | export * from './abuse' |
4 | export * from './account' | 2 | export * from './account' |
3 | export * from './activitypub' | ||
5 | export * from './actor-image' | 4 | export * from './actor-image' |
6 | export * from './blocklist' | 5 | export * from './blocklist' |
7 | export * from './bulk' | 6 | export * from './bulk' |
@@ -10,8 +9,9 @@ export * from './express' | |||
10 | export * from './feeds' | 9 | export * from './feeds' |
11 | export * from './follows' | 10 | export * from './follows' |
12 | export * from './jobs' | 11 | export * from './jobs' |
13 | export * from './metrics' | ||
14 | export * from './logs' | 12 | export * from './logs' |
13 | export * from './metrics' | ||
14 | export * from './object-storage-proxy' | ||
15 | export * from './oembed' | 15 | export * from './oembed' |
16 | export * from './pagination' | 16 | export * from './pagination' |
17 | export * from './plugins' | 17 | export * from './plugins' |
@@ -19,9 +19,11 @@ export * from './redundancy' | |||
19 | export * from './search' | 19 | export * from './search' |
20 | export * from './server' | 20 | export * from './server' |
21 | export * from './sort' | 21 | export * from './sort' |
22 | export * from './static' | ||
22 | export * from './themes' | 23 | export * from './themes' |
23 | export * from './user-history' | 24 | export * from './user-history' |
24 | export * from './user-notifications' | 25 | export * from './user-notifications' |
25 | export * from './user-subscriptions' | 26 | export * from './user-subscriptions' |
26 | export * from './users' | 27 | export * from './users' |
28 | export * from './videos' | ||
27 | export * from './webfinger' | 29 | export * from './webfinger' |
diff --git a/server/middlewares/validators/object-storage-proxy.ts b/server/middlewares/validators/object-storage-proxy.ts new file mode 100644 index 000000000..bbd77f262 --- /dev/null +++ b/server/middlewares/validators/object-storage-proxy.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import express from 'express' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { HttpStatusCode } from '@shared/models' | ||
4 | |||
5 | const ensurePrivateObjectStorageProxyIsEnabled = [ | ||
6 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
7 | if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) { | ||
8 | return res.fail({ | ||
9 | message: 'Private object storage proxy is not enabled', | ||
10 | status: HttpStatusCode.BAD_REQUEST_400 | ||
11 | }) | ||
12 | } | ||
13 | |||
14 | return next() | ||
15 | } | ||
16 | ] | ||
17 | |||
18 | export { | ||
19 | ensurePrivateObjectStorageProxyIsEnabled | ||
20 | } | ||
diff --git a/server/middlewares/validators/shared/index.ts b/server/middlewares/validators/shared/index.ts index bbd03b248..de98cd442 100644 --- a/server/middlewares/validators/shared/index.ts +++ b/server/middlewares/validators/shared/index.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | export * from './abuses' | 1 | export * from './abuses' |
2 | export * from './accounts' | 2 | export * from './accounts' |
3 | export * from './users' | ||
3 | export * from './utils' | 4 | export * from './utils' |
4 | export * from './video-blacklists' | 5 | export * from './video-blacklists' |
5 | export * from './video-captions' | 6 | export * from './video-captions' |
diff --git a/server/middlewares/validators/shared/users.ts b/server/middlewares/validators/shared/users.ts new file mode 100644 index 000000000..fbaa7db0e --- /dev/null +++ b/server/middlewares/validators/shared/users.ts | |||
@@ -0,0 +1,62 @@ | |||
1 | import express from 'express' | ||
2 | import { ActorModel } from '@server/models/actor/actor' | ||
3 | import { UserModel } from '@server/models/user/user' | ||
4 | import { MUserDefault } from '@server/types/models' | ||
5 | import { HttpStatusCode } from '@shared/models' | ||
6 | |||
7 | function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { | ||
8 | const id = parseInt(idArg + '', 10) | ||
9 | return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res) | ||
10 | } | ||
11 | |||
12 | function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { | ||
13 | return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse) | ||
14 | } | ||
15 | |||
16 | async function checkUserNameOrEmailDoesNotAlreadyExist (username: string, email: string, res: express.Response) { | ||
17 | const user = await UserModel.loadByUsernameOrEmail(username, email) | ||
18 | |||
19 | if (user) { | ||
20 | res.fail({ | ||
21 | status: HttpStatusCode.CONFLICT_409, | ||
22 | message: 'User with this username or email already exists.' | ||
23 | }) | ||
24 | return false | ||
25 | } | ||
26 | |||
27 | const actor = await ActorModel.loadLocalByName(username) | ||
28 | if (actor) { | ||
29 | res.fail({ | ||
30 | status: HttpStatusCode.CONFLICT_409, | ||
31 | message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.' | ||
32 | }) | ||
33 | return false | ||
34 | } | ||
35 | |||
36 | return true | ||
37 | } | ||
38 | |||
39 | async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) { | ||
40 | const user = await finder() | ||
41 | |||
42 | if (!user) { | ||
43 | if (abortResponse === true) { | ||
44 | res.fail({ | ||
45 | status: HttpStatusCode.NOT_FOUND_404, | ||
46 | message: 'User not found' | ||
47 | }) | ||
48 | } | ||
49 | |||
50 | return false | ||
51 | } | ||
52 | |||
53 | res.locals.user = user | ||
54 | return true | ||
55 | } | ||
56 | |||
57 | export { | ||
58 | checkUserIdExist, | ||
59 | checkUserEmailExist, | ||
60 | checkUserNameOrEmailDoesNotAlreadyExist, | ||
61 | checkUserExist | ||
62 | } | ||
diff --git a/server/middlewares/validators/shared/videos.ts b/server/middlewares/validators/shared/videos.ts index e3a98c58f..ebbfc0a0a 100644 --- a/server/middlewares/validators/shared/videos.ts +++ b/server/middlewares/validators/shared/videos.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { Request, Response } from 'express' | 1 | import { Request, Response } from 'express' |
2 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' | ||
3 | import { loadVideo, VideoLoadType } from '@server/lib/model-loaders' | 2 | import { loadVideo, VideoLoadType } from '@server/lib/model-loaders' |
4 | import { isAbleToUploadVideo } from '@server/lib/user' | 3 | import { isAbleToUploadVideo } from '@server/lib/user' |
4 | import { VideoTokensManager } from '@server/lib/video-tokens-manager' | ||
5 | import { authenticatePromise } from '@server/middlewares/auth' | 5 | import { authenticatePromise } from '@server/middlewares/auth' |
6 | import { VideoModel } from '@server/models/video/video' | 6 | import { VideoModel } from '@server/models/video/video' |
7 | import { VideoChannelModel } from '@server/models/video/video-channel' | 7 | import { VideoChannelModel } from '@server/models/video/video-channel' |
@@ -108,26 +108,21 @@ async function checkCanSeeVideo (options: { | |||
108 | res: Response | 108 | res: Response |
109 | paramId: string | 109 | paramId: string |
110 | video: MVideo | 110 | video: MVideo |
111 | authenticateInQuery?: boolean // default false | ||
112 | }) { | 111 | }) { |
113 | const { req, res, video, paramId, authenticateInQuery = false } = options | 112 | const { req, res, video, paramId } = options |
114 | 113 | ||
115 | if (video.requiresAuth()) { | 114 | if (video.requiresAuth({ urlParamId: paramId, checkBlacklist: true })) { |
116 | return checkCanSeeAuthVideo(req, res, video, authenticateInQuery) | 115 | return checkCanSeeAuthVideo(req, res, video) |
117 | } | 116 | } |
118 | 117 | ||
119 | if (video.privacy === VideoPrivacy.UNLISTED) { | 118 | if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) { |
120 | if (isUUIDValid(paramId)) return true | 119 | return true |
121 | |||
122 | return checkCanSeeAuthVideo(req, res, video, authenticateInQuery) | ||
123 | } | 120 | } |
124 | 121 | ||
125 | if (video.privacy === VideoPrivacy.PUBLIC) return true | 122 | throw new Error('Unknown video privacy when checking video right ' + video.url) |
126 | |||
127 | throw new Error('Fatal error when checking video right ' + video.url) | ||
128 | } | 123 | } |
129 | 124 | ||
130 | async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights, authenticateInQuery = false) { | 125 | async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights) { |
131 | const fail = () => { | 126 | const fail = () => { |
132 | res.fail({ | 127 | res.fail({ |
133 | status: HttpStatusCode.FORBIDDEN_403, | 128 | status: HttpStatusCode.FORBIDDEN_403, |
@@ -137,7 +132,7 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI | |||
137 | return false | 132 | return false |
138 | } | 133 | } |
139 | 134 | ||
140 | await authenticatePromise(req, res, authenticateInQuery) | 135 | await authenticatePromise(req, res) |
141 | 136 | ||
142 | const user = res.locals.oauth?.token.User | 137 | const user = res.locals.oauth?.token.User |
143 | if (!user) return fail() | 138 | if (!user) return fail() |
@@ -173,6 +168,36 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI | |||
173 | 168 | ||
174 | // --------------------------------------------------------------------------- | 169 | // --------------------------------------------------------------------------- |
175 | 170 | ||
171 | async function checkCanAccessVideoStaticFiles (options: { | ||
172 | video: MVideo | ||
173 | req: Request | ||
174 | res: Response | ||
175 | paramId: string | ||
176 | }) { | ||
177 | const { video, req, res } = options | ||
178 | |||
179 | if (res.locals.oauth?.token.User) { | ||
180 | return checkCanSeeVideo(options) | ||
181 | } | ||
182 | |||
183 | if (!video.hasPrivateStaticPath()) return true | ||
184 | |||
185 | const videoFileToken = req.query.videoFileToken | ||
186 | if (!videoFileToken) { | ||
187 | res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
188 | return false | ||
189 | } | ||
190 | |||
191 | if (VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) { | ||
192 | return true | ||
193 | } | ||
194 | |||
195 | res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
196 | return false | ||
197 | } | ||
198 | |||
199 | // --------------------------------------------------------------------------- | ||
200 | |||
176 | function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) { | 201 | function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) { |
177 | // Retrieve the user who did the request | 202 | // Retrieve the user who did the request |
178 | if (onlyOwned && video.isOwned() === false) { | 203 | if (onlyOwned && video.isOwned() === false) { |
@@ -220,6 +245,7 @@ export { | |||
220 | doesVideoExist, | 245 | doesVideoExist, |
221 | doesVideoFileOfVideoExist, | 246 | doesVideoFileOfVideoExist, |
222 | 247 | ||
248 | checkCanAccessVideoStaticFiles, | ||
223 | checkUserCanManageVideo, | 249 | checkUserCanManageVideo, |
224 | checkCanSeeVideo, | 250 | checkCanSeeVideo, |
225 | checkUserQuota | 251 | checkUserQuota |
diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts new file mode 100644 index 000000000..13fde6dd1 --- /dev/null +++ b/server/middlewares/validators/static.ts | |||
@@ -0,0 +1,169 @@ | |||
1 | import express from 'express' | ||
2 | import { query } from 'express-validator' | ||
3 | import LRUCache from 'lru-cache' | ||
4 | import { basename, dirname } from 'path' | ||
5 | import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc' | ||
6 | import { logger } from '@server/helpers/logger' | ||
7 | import { LRU_CACHE } from '@server/initializers/constants' | ||
8 | import { VideoModel } from '@server/models/video/video' | ||
9 | import { VideoFileModel } from '@server/models/video/video-file' | ||
10 | import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models' | ||
11 | import { HttpStatusCode } from '@shared/models' | ||
12 | import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared' | ||
13 | |||
14 | type LRUValue = { | ||
15 | allowed: boolean | ||
16 | video?: MVideoThumbnail | ||
17 | file?: MVideoFile | ||
18 | playlist?: MStreamingPlaylist } | ||
19 | |||
20 | const staticFileTokenBypass = new LRUCache<string, LRUValue>({ | ||
21 | max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE, | ||
22 | ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL | ||
23 | }) | ||
24 | |||
25 | const ensureCanAccessVideoPrivateWebTorrentFiles = [ | ||
26 | query('videoFileToken').optional().custom(exists), | ||
27 | |||
28 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
29 | if (areValidationErrors(req, res)) return | ||
30 | |||
31 | const token = extractTokenOrDie(req, res) | ||
32 | if (!token) return | ||
33 | |||
34 | const cacheKey = token + '-' + req.originalUrl | ||
35 | |||
36 | if (staticFileTokenBypass.has(cacheKey)) { | ||
37 | const { allowed, file, video } = staticFileTokenBypass.get(cacheKey) | ||
38 | |||
39 | if (allowed === true) { | ||
40 | res.locals.onlyVideo = video | ||
41 | res.locals.videoFile = file | ||
42 | |||
43 | return next() | ||
44 | } | ||
45 | |||
46 | return res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
47 | } | ||
48 | |||
49 | const result = await isWebTorrentAllowed(req, res) | ||
50 | |||
51 | staticFileTokenBypass.set(cacheKey, result) | ||
52 | |||
53 | if (result.allowed !== true) return | ||
54 | |||
55 | res.locals.onlyVideo = result.video | ||
56 | res.locals.videoFile = result.file | ||
57 | |||
58 | return next() | ||
59 | } | ||
60 | ] | ||
61 | |||
62 | const ensureCanAccessPrivateVideoHLSFiles = [ | ||
63 | query('videoFileToken').optional().custom(exists), | ||
64 | |||
65 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
66 | if (areValidationErrors(req, res)) return | ||
67 | |||
68 | const videoUUID = basename(dirname(req.originalUrl)) | ||
69 | |||
70 | if (!isUUIDValid(videoUUID)) { | ||
71 | logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl) | ||
72 | |||
73 | return res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
74 | } | ||
75 | |||
76 | const token = extractTokenOrDie(req, res) | ||
77 | if (!token) return | ||
78 | |||
79 | const cacheKey = token + '-' + videoUUID | ||
80 | |||
81 | if (staticFileTokenBypass.has(cacheKey)) { | ||
82 | const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey) | ||
83 | |||
84 | if (allowed === true) { | ||
85 | res.locals.onlyVideo = video | ||
86 | res.locals.videoFile = file | ||
87 | res.locals.videoStreamingPlaylist = playlist | ||
88 | |||
89 | return next() | ||
90 | } | ||
91 | |||
92 | return res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
93 | } | ||
94 | |||
95 | const result = await isHLSAllowed(req, res, videoUUID) | ||
96 | |||
97 | staticFileTokenBypass.set(cacheKey, result) | ||
98 | |||
99 | if (result.allowed !== true) return | ||
100 | |||
101 | res.locals.onlyVideo = result.video | ||
102 | res.locals.videoFile = result.file | ||
103 | res.locals.videoStreamingPlaylist = result.playlist | ||
104 | |||
105 | return next() | ||
106 | } | ||
107 | ] | ||
108 | |||
109 | export { | ||
110 | ensureCanAccessVideoPrivateWebTorrentFiles, | ||
111 | ensureCanAccessPrivateVideoHLSFiles | ||
112 | } | ||
113 | |||
114 | // --------------------------------------------------------------------------- | ||
115 | |||
116 | async function isWebTorrentAllowed (req: express.Request, res: express.Response) { | ||
117 | const filename = basename(req.path) | ||
118 | |||
119 | const file = await VideoFileModel.loadWithVideoByFilename(filename) | ||
120 | if (!file) { | ||
121 | logger.debug('Unknown static file %s to serve', req.originalUrl, { filename }) | ||
122 | |||
123 | res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
124 | return { allowed: false } | ||
125 | } | ||
126 | |||
127 | const video = await VideoModel.load(file.getVideo().id) | ||
128 | |||
129 | return { | ||
130 | file, | ||
131 | video, | ||
132 | allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) | ||
133 | } | ||
134 | } | ||
135 | |||
136 | async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) { | ||
137 | const filename = basename(req.path) | ||
138 | |||
139 | const video = await VideoModel.loadWithFiles(videoUUID) | ||
140 | |||
141 | if (!video) { | ||
142 | logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID }) | ||
143 | |||
144 | res.sendStatus(HttpStatusCode.FORBIDDEN_403) | ||
145 | return { allowed: false } | ||
146 | } | ||
147 | |||
148 | const file = await VideoFileModel.loadByFilename(filename) | ||
149 | |||
150 | return { | ||
151 | file, | ||
152 | video, | ||
153 | playlist: video.getHLSPlaylist(), | ||
154 | allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) | ||
155 | } | ||
156 | } | ||
157 | |||
158 | function extractTokenOrDie (req: express.Request, res: express.Response) { | ||
159 | const token = res.locals.oauth?.token.accessToken || req.query.videoFileToken | ||
160 | |||
161 | if (!token) { | ||
162 | return res.fail({ | ||
163 | message: 'Bearer token is missing in headers or video file token is missing in URL query parameters', | ||
164 | status: HttpStatusCode.FORBIDDEN_403 | ||
165 | }) | ||
166 | } | ||
167 | |||
168 | return token | ||
169 | } | ||
diff --git a/server/middlewares/validators/two-factor.ts b/server/middlewares/validators/two-factor.ts new file mode 100644 index 000000000..106b579b5 --- /dev/null +++ b/server/middlewares/validators/two-factor.ts | |||
@@ -0,0 +1,81 @@ | |||
1 | import express from 'express' | ||
2 | import { body, param } from 'express-validator' | ||
3 | import { HttpStatusCode, UserRight } from '@shared/models' | ||
4 | import { exists, isIdValid } from '../../helpers/custom-validators/misc' | ||
5 | import { areValidationErrors, checkUserIdExist } from './shared' | ||
6 | |||
7 | const requestOrConfirmTwoFactorValidator = [ | ||
8 | param('id').custom(isIdValid), | ||
9 | |||
10 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
11 | if (areValidationErrors(req, res)) return | ||
12 | |||
13 | if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return | ||
14 | |||
15 | if (res.locals.user.otpSecret) { | ||
16 | return res.fail({ | ||
17 | status: HttpStatusCode.BAD_REQUEST_400, | ||
18 | message: `Two factor is already enabled.` | ||
19 | }) | ||
20 | } | ||
21 | |||
22 | return next() | ||
23 | } | ||
24 | ] | ||
25 | |||
26 | const confirmTwoFactorValidator = [ | ||
27 | body('requestToken').custom(exists), | ||
28 | body('otpToken').custom(exists), | ||
29 | |||
30 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
31 | if (areValidationErrors(req, res)) return | ||
32 | |||
33 | return next() | ||
34 | } | ||
35 | ] | ||
36 | |||
37 | const disableTwoFactorValidator = [ | ||
38 | param('id').custom(isIdValid), | ||
39 | |||
40 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
41 | if (areValidationErrors(req, res)) return | ||
42 | |||
43 | if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return | ||
44 | |||
45 | if (!res.locals.user.otpSecret) { | ||
46 | return res.fail({ | ||
47 | status: HttpStatusCode.BAD_REQUEST_400, | ||
48 | message: `Two factor is already disabled.` | ||
49 | }) | ||
50 | } | ||
51 | |||
52 | return next() | ||
53 | } | ||
54 | ] | ||
55 | |||
56 | // --------------------------------------------------------------------------- | ||
57 | |||
58 | export { | ||
59 | requestOrConfirmTwoFactorValidator, | ||
60 | confirmTwoFactorValidator, | ||
61 | disableTwoFactorValidator | ||
62 | } | ||
63 | |||
64 | // --------------------------------------------------------------------------- | ||
65 | |||
66 | async function checkCanEnableOrDisableTwoFactor (userId: number | string, res: express.Response) { | ||
67 | const authUser = res.locals.oauth.token.user | ||
68 | |||
69 | if (!await checkUserIdExist(userId, res)) return | ||
70 | |||
71 | if (res.locals.user.id !== authUser.id && authUser.hasRight(UserRight.MANAGE_USERS) !== true) { | ||
72 | res.fail({ | ||
73 | status: HttpStatusCode.FORBIDDEN_403, | ||
74 | message: `User ${authUser.username} does not have right to change two factor setting of this user.` | ||
75 | }) | ||
76 | |||
77 | return false | ||
78 | } | ||
79 | |||
80 | return true | ||
81 | } | ||
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 2de5265fb..055af3b64 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts | |||
@@ -1,9 +1,8 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { body, param, query } from 'express-validator' | 2 | import { body, param, query } from 'express-validator' |
3 | import { Hooks } from '@server/lib/plugins/hooks' | 3 | import { Hooks } from '@server/lib/plugins/hooks' |
4 | import { MUserDefault } from '@server/types/models' | ||
5 | import { HttpStatusCode, UserRegister, UserRight, UserRole } from '@shared/models' | 4 | import { HttpStatusCode, UserRegister, UserRight, UserRole } from '@shared/models' |
6 | import { isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' | 5 | import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' |
7 | import { isThemeNameValid } from '../../helpers/custom-validators/plugins' | 6 | import { isThemeNameValid } from '../../helpers/custom-validators/plugins' |
8 | import { | 7 | import { |
9 | isUserAdminFlagsValid, | 8 | isUserAdminFlagsValid, |
@@ -30,8 +29,15 @@ import { isThemeRegistered } from '../../lib/plugins/theme-utils' | |||
30 | import { Redis } from '../../lib/redis' | 29 | import { Redis } from '../../lib/redis' |
31 | import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../lib/signup' | 30 | import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../lib/signup' |
32 | import { ActorModel } from '../../models/actor/actor' | 31 | import { ActorModel } from '../../models/actor/actor' |
33 | import { UserModel } from '../../models/user/user' | 32 | import { |
34 | import { areValidationErrors, doesVideoChannelIdExist, doesVideoExist, isValidVideoIdParam } from './shared' | 33 | areValidationErrors, |
34 | checkUserEmailExist, | ||
35 | checkUserIdExist, | ||
36 | checkUserNameOrEmailDoesNotAlreadyExist, | ||
37 | doesVideoChannelIdExist, | ||
38 | doesVideoExist, | ||
39 | isValidVideoIdParam | ||
40 | } from './shared' | ||
35 | 41 | ||
36 | const usersListValidator = [ | 42 | const usersListValidator = [ |
37 | query('blocked') | 43 | query('blocked') |
@@ -411,6 +417,13 @@ const usersAskResetPasswordValidator = [ | |||
411 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | 417 | return res.status(HttpStatusCode.NO_CONTENT_204).end() |
412 | } | 418 | } |
413 | 419 | ||
420 | if (res.locals.user.pluginAuth) { | ||
421 | return res.fail({ | ||
422 | status: HttpStatusCode.CONFLICT_409, | ||
423 | message: 'Cannot recover password of a user that uses a plugin authentication.' | ||
424 | }) | ||
425 | } | ||
426 | |||
414 | return next() | 427 | return next() |
415 | } | 428 | } |
416 | ] | 429 | ] |
@@ -428,7 +441,7 @@ const usersResetPasswordValidator = [ | |||
428 | if (!await checkUserIdExist(req.params.id, res)) return | 441 | if (!await checkUserIdExist(req.params.id, res)) return |
429 | 442 | ||
430 | const user = res.locals.user | 443 | const user = res.locals.user |
431 | const redisVerificationString = await Redis.Instance.getResetPasswordLink(user.id) | 444 | const redisVerificationString = await Redis.Instance.getResetPasswordVerificationString(user.id) |
432 | 445 | ||
433 | if (redisVerificationString !== req.body.verificationString) { | 446 | if (redisVerificationString !== req.body.verificationString) { |
434 | return res.fail({ | 447 | return res.fail({ |
@@ -454,6 +467,13 @@ const usersAskSendVerifyEmailValidator = [ | |||
454 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | 467 | return res.status(HttpStatusCode.NO_CONTENT_204).end() |
455 | } | 468 | } |
456 | 469 | ||
470 | if (res.locals.user.pluginAuth) { | ||
471 | return res.fail({ | ||
472 | status: HttpStatusCode.CONFLICT_409, | ||
473 | message: 'Cannot ask verification email of a user that uses a plugin authentication.' | ||
474 | }) | ||
475 | } | ||
476 | |||
457 | return next() | 477 | return next() |
458 | } | 478 | } |
459 | ] | 479 | ] |
@@ -486,6 +506,41 @@ const usersVerifyEmailValidator = [ | |||
486 | } | 506 | } |
487 | ] | 507 | ] |
488 | 508 | ||
509 | const usersCheckCurrentPasswordFactory = (targetUserIdGetter: (req: express.Request) => number | string) => { | ||
510 | return [ | ||
511 | body('currentPassword').optional().custom(exists), | ||
512 | |||
513 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
514 | if (areValidationErrors(req, res)) return | ||
515 | |||
516 | const user = res.locals.oauth.token.User | ||
517 | const isAdminOrModerator = user.role === UserRole.ADMINISTRATOR || user.role === UserRole.MODERATOR | ||
518 | const targetUserId = parseInt(targetUserIdGetter(req) + '') | ||
519 | |||
520 | // Admin/moderator action on another user, skip the password check | ||
521 | if (isAdminOrModerator && targetUserId !== user.id) { | ||
522 | return next() | ||
523 | } | ||
524 | |||
525 | if (!req.body.currentPassword) { | ||
526 | return res.fail({ | ||
527 | status: HttpStatusCode.BAD_REQUEST_400, | ||
528 | message: 'currentPassword is missing' | ||
529 | }) | ||
530 | } | ||
531 | |||
532 | if (await user.isPasswordMatch(req.body.currentPassword) !== true) { | ||
533 | return res.fail({ | ||
534 | status: HttpStatusCode.FORBIDDEN_403, | ||
535 | message: 'currentPassword is invalid.' | ||
536 | }) | ||
537 | } | ||
538 | |||
539 | return next() | ||
540 | } | ||
541 | ] | ||
542 | } | ||
543 | |||
489 | const userAutocompleteValidator = [ | 544 | const userAutocompleteValidator = [ |
490 | param('search') | 545 | param('search') |
491 | .isString() | 546 | .isString() |
@@ -553,6 +608,7 @@ export { | |||
553 | usersUpdateValidator, | 608 | usersUpdateValidator, |
554 | usersUpdateMeValidator, | 609 | usersUpdateMeValidator, |
555 | usersVideoRatingValidator, | 610 | usersVideoRatingValidator, |
611 | usersCheckCurrentPasswordFactory, | ||
556 | ensureUserRegistrationAllowed, | 612 | ensureUserRegistrationAllowed, |
557 | ensureUserRegistrationAllowedForIP, | 613 | ensureUserRegistrationAllowedForIP, |
558 | usersGetValidator, | 614 | usersGetValidator, |
@@ -566,55 +622,3 @@ export { | |||
566 | ensureCanModerateUser, | 622 | ensureCanModerateUser, |
567 | ensureCanManageChannelOrAccount | 623 | ensureCanManageChannelOrAccount |
568 | } | 624 | } |
569 | |||
570 | // --------------------------------------------------------------------------- | ||
571 | |||
572 | function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { | ||
573 | const id = parseInt(idArg + '', 10) | ||
574 | return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res) | ||
575 | } | ||
576 | |||
577 | function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { | ||
578 | return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse) | ||
579 | } | ||
580 | |||
581 | async function checkUserNameOrEmailDoesNotAlreadyExist (username: string, email: string, res: express.Response) { | ||
582 | const user = await UserModel.loadByUsernameOrEmail(username, email) | ||
583 | |||
584 | if (user) { | ||
585 | res.fail({ | ||
586 | status: HttpStatusCode.CONFLICT_409, | ||
587 | message: 'User with this username or email already exists.' | ||
588 | }) | ||
589 | return false | ||
590 | } | ||
591 | |||
592 | const actor = await ActorModel.loadLocalByName(username) | ||
593 | if (actor) { | ||
594 | res.fail({ | ||
595 | status: HttpStatusCode.CONFLICT_409, | ||
596 | message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.' | ||
597 | }) | ||
598 | return false | ||
599 | } | ||
600 | |||
601 | return true | ||
602 | } | ||
603 | |||
604 | async function checkUserExist (finder: () => Promise<MUserDefault>, res: express.Response, abortResponse = true) { | ||
605 | const user = await finder() | ||
606 | |||
607 | if (!user) { | ||
608 | if (abortResponse === true) { | ||
609 | res.fail({ | ||
610 | status: HttpStatusCode.NOT_FOUND_404, | ||
611 | message: 'User not found' | ||
612 | }) | ||
613 | } | ||
614 | |||
615 | return false | ||
616 | } | ||
617 | |||
618 | res.locals.user = user | ||
619 | return true | ||
620 | } | ||
diff --git a/server/middlewares/validators/videos/video-comments.ts b/server/middlewares/validators/videos/video-comments.ts index 69062701b..133feb7bd 100644 --- a/server/middlewares/validators/videos/video-comments.ts +++ b/server/middlewares/validators/videos/video-comments.ts | |||
@@ -208,7 +208,8 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon | |||
208 | const acceptParameters = { | 208 | const acceptParameters = { |
209 | video, | 209 | video, |
210 | commentBody: req.body, | 210 | commentBody: req.body, |
211 | user: res.locals.oauth.token.User | 211 | user: res.locals.oauth.token.User, |
212 | req | ||
212 | } | 213 | } |
213 | 214 | ||
214 | let acceptedResult: AcceptResult | 215 | let acceptedResult: AcceptResult |
@@ -234,7 +235,7 @@ async function isVideoCommentAccepted (req: express.Request, res: express.Respon | |||
234 | 235 | ||
235 | res.fail({ | 236 | res.fail({ |
236 | status: HttpStatusCode.FORBIDDEN_403, | 237 | status: HttpStatusCode.FORBIDDEN_403, |
237 | message: acceptedResult?.errorMessage || 'Refused local comment' | 238 | message: acceptedResult?.errorMessage || 'Comment has been rejected.' |
238 | }) | 239 | }) |
239 | return false | 240 | return false |
240 | } | 241 | } |
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index 7fd2b03d1..e29eb4a32 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts | |||
@@ -7,7 +7,7 @@ import { getServerActor } from '@server/models/application/application' | |||
7 | import { ExpressPromiseHandler } from '@server/types/express-handler' | 7 | import { ExpressPromiseHandler } from '@server/types/express-handler' |
8 | import { MUserAccountId, MVideoFullLight } from '@server/types/models' | 8 | import { MUserAccountId, MVideoFullLight } from '@server/types/models' |
9 | import { arrayify, getAllPrivacies } from '@shared/core-utils' | 9 | import { arrayify, getAllPrivacies } from '@shared/core-utils' |
10 | import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude } from '@shared/models' | 10 | import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models' |
11 | import { | 11 | import { |
12 | exists, | 12 | exists, |
13 | isBooleanValid, | 13 | isBooleanValid, |
@@ -48,6 +48,7 @@ import { Hooks } from '../../../lib/plugins/hooks' | |||
48 | import { VideoModel } from '../../../models/video/video' | 48 | import { VideoModel } from '../../../models/video/video' |
49 | import { | 49 | import { |
50 | areValidationErrors, | 50 | areValidationErrors, |
51 | checkCanAccessVideoStaticFiles, | ||
51 | checkCanSeeVideo, | 52 | checkCanSeeVideo, |
52 | checkUserCanManageVideo, | 53 | checkUserCanManageVideo, |
53 | checkUserQuota, | 54 | checkUserQuota, |
@@ -232,6 +233,11 @@ const videosUpdateValidator = getCommonVideoEditAttributes().concat([ | |||
232 | if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) | 233 | if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) |
233 | if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req) | 234 | if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req) |
234 | 235 | ||
236 | const video = getVideoWithAttributes(res) | ||
237 | if (req.body.privacy && video.isLive && video.state !== VideoState.WAITING_FOR_LIVE) { | ||
238 | return res.fail({ message: 'Cannot update privacy of a live that has already started' }) | ||
239 | } | ||
240 | |||
235 | // Check if the user who did the request is able to update the video | 241 | // Check if the user who did the request is able to update the video |
236 | const user = res.locals.oauth.token.User | 242 | const user = res.locals.oauth.token.User |
237 | if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) | 243 | if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) |
@@ -271,10 +277,7 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R | |||
271 | }) | 277 | }) |
272 | } | 278 | } |
273 | 279 | ||
274 | const videosCustomGetValidator = ( | 280 | const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => { |
275 | fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes', | ||
276 | authenticateInQuery = false | ||
277 | ) => { | ||
278 | return [ | 281 | return [ |
279 | isValidVideoIdParam('id'), | 282 | isValidVideoIdParam('id'), |
280 | 283 | ||
@@ -287,7 +290,7 @@ const videosCustomGetValidator = ( | |||
287 | 290 | ||
288 | const video = getVideoWithAttributes(res) as MVideoFullLight | 291 | const video = getVideoWithAttributes(res) as MVideoFullLight |
289 | 292 | ||
290 | if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id, authenticateInQuery })) return | 293 | if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return |
291 | 294 | ||
292 | return next() | 295 | return next() |
293 | } | 296 | } |
@@ -295,7 +298,6 @@ const videosCustomGetValidator = ( | |||
295 | } | 298 | } |
296 | 299 | ||
297 | const videosGetValidator = videosCustomGetValidator('all') | 300 | const videosGetValidator = videosCustomGetValidator('all') |
298 | const videosDownloadValidator = videosCustomGetValidator('all', true) | ||
299 | 301 | ||
300 | const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ | 302 | const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ |
301 | isValidVideoIdParam('id'), | 303 | isValidVideoIdParam('id'), |
@@ -311,6 +313,21 @@ const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ | |||
311 | } | 313 | } |
312 | ]) | 314 | ]) |
313 | 315 | ||
316 | const videosDownloadValidator = [ | ||
317 | isValidVideoIdParam('id'), | ||
318 | |||
319 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
320 | if (areValidationErrors(req, res)) return | ||
321 | if (!await doesVideoExist(req.params.id, res, 'all')) return | ||
322 | |||
323 | const video = getVideoWithAttributes(res) | ||
324 | |||
325 | if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return | ||
326 | |||
327 | return next() | ||
328 | } | ||
329 | ] | ||
330 | |||
314 | const videosRemoveValidator = [ | 331 | const videosRemoveValidator = [ |
315 | isValidVideoIdParam('id'), | 332 | isValidVideoIdParam('id'), |
316 | 333 | ||
@@ -372,7 +389,7 @@ function getCommonVideoEditAttributes () { | |||
372 | .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'), | 389 | .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'), |
373 | body('privacy') | 390 | body('privacy') |
374 | .optional() | 391 | .optional() |
375 | .customSanitizer(toValueOrNull) | 392 | .customSanitizer(toIntOrNull) |
376 | .custom(isVideoPrivacyValid), | 393 | .custom(isVideoPrivacyValid), |
377 | body('description') | 394 | body('description') |
378 | .optional() | 395 | .optional() |
diff --git a/server/models/user/user.ts b/server/models/user/user.ts index 1a7c84390..f70feed73 100644 --- a/server/models/user/user.ts +++ b/server/models/user/user.ts | |||
@@ -403,6 +403,11 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> { | |||
403 | @Column | 403 | @Column |
404 | lastLoginDate: Date | 404 | lastLoginDate: Date |
405 | 405 | ||
406 | @AllowNull(true) | ||
407 | @Default(null) | ||
408 | @Column | ||
409 | otpSecret: string | ||
410 | |||
406 | @CreatedAt | 411 | @CreatedAt |
407 | createdAt: Date | 412 | createdAt: Date |
408 | 413 | ||
@@ -886,8 +891,10 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> { | |||
886 | autoPlayNextVideoPlaylist: this.autoPlayNextVideoPlaylist, | 891 | autoPlayNextVideoPlaylist: this.autoPlayNextVideoPlaylist, |
887 | videoLanguages: this.videoLanguages, | 892 | videoLanguages: this.videoLanguages, |
888 | 893 | ||
889 | role: this.role, | 894 | role: { |
890 | roleLabel: USER_ROLE_LABELS[this.role], | 895 | id: this.role, |
896 | label: USER_ROLE_LABELS[this.role] | ||
897 | }, | ||
891 | 898 | ||
892 | videoQuota: this.videoQuota, | 899 | videoQuota: this.videoQuota, |
893 | videoQuotaDaily: this.videoQuotaDaily, | 900 | videoQuotaDaily: this.videoQuotaDaily, |
@@ -935,7 +942,9 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> { | |||
935 | 942 | ||
936 | pluginAuth: this.pluginAuth, | 943 | pluginAuth: this.pluginAuth, |
937 | 944 | ||
938 | lastLoginDate: this.lastLoginDate | 945 | lastLoginDate: this.lastLoginDate, |
946 | |||
947 | twoFactorEnabled: !!this.otpSecret | ||
939 | } | 948 | } |
940 | 949 | ||
941 | if (parameters.withAdminFlags) { | 950 | if (parameters.withAdminFlags) { |
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index e1b0eb610..240619f69 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts | |||
@@ -34,6 +34,7 @@ import { | |||
34 | import { | 34 | import { |
35 | MServer, | 35 | MServer, |
36 | MStreamingPlaylistRedundanciesOpt, | 36 | MStreamingPlaylistRedundanciesOpt, |
37 | MUserId, | ||
37 | MVideo, | 38 | MVideo, |
38 | MVideoAP, | 39 | MVideoAP, |
39 | MVideoFile, | 40 | MVideoFile, |
@@ -102,6 +103,7 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm | |||
102 | }, | 103 | }, |
103 | nsfw: video.nsfw, | 104 | nsfw: video.nsfw, |
104 | 105 | ||
106 | truncatedDescription: video.getTruncatedDescription(), | ||
105 | description: options && options.completeDescription === true | 107 | description: options && options.completeDescription === true |
106 | ? video.description | 108 | ? video.description |
107 | : video.getTruncatedDescription(), | 109 | : video.getTruncatedDescription(), |
@@ -180,6 +182,7 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid | |||
180 | const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON') | 182 | const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON') |
181 | 183 | ||
182 | const videoJSON = video.toFormattedJSON({ | 184 | const videoJSON = video.toFormattedJSON({ |
185 | completeDescription: true, | ||
183 | additionalAttributes: { | 186 | additionalAttributes: { |
184 | scheduledUpdate: true, | 187 | scheduledUpdate: true, |
185 | blacklistInfo: true, | 188 | blacklistInfo: true, |
@@ -245,8 +248,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) { | |||
245 | function videoFilesModelToFormattedJSON ( | 248 | function videoFilesModelToFormattedJSON ( |
246 | video: MVideoFormattable, | 249 | video: MVideoFormattable, |
247 | videoFiles: MVideoFileRedundanciesOpt[], | 250 | videoFiles: MVideoFileRedundanciesOpt[], |
248 | includeMagnet = true | 251 | options: { |
252 | includeMagnet?: boolean // default true | ||
253 | } = {} | ||
249 | ): VideoFile[] { | 254 | ): VideoFile[] { |
255 | const { includeMagnet = true } = options | ||
256 | |||
250 | const trackerUrls = includeMagnet | 257 | const trackerUrls = includeMagnet |
251 | ? video.getTrackerUrls() | 258 | ? video.getTrackerUrls() |
252 | : [] | 259 | : [] |
@@ -281,11 +288,14 @@ function videoFilesModelToFormattedJSON ( | |||
281 | }) | 288 | }) |
282 | } | 289 | } |
283 | 290 | ||
284 | function addVideoFilesInAPAcc ( | 291 | function addVideoFilesInAPAcc (options: { |
285 | acc: ActivityUrlObject[] | ActivityTagObject[], | 292 | acc: ActivityUrlObject[] | ActivityTagObject[] |
286 | video: MVideo, | 293 | video: MVideo |
287 | files: MVideoFile[] | 294 | files: MVideoFile[] |
288 | ) { | 295 | user?: MUserId |
296 | }) { | ||
297 | const { acc, video, files } = options | ||
298 | |||
289 | const trackerUrls = video.getTrackerUrls() | 299 | const trackerUrls = video.getTrackerUrls() |
290 | 300 | ||
291 | const sortedFiles = (files || []) | 301 | const sortedFiles = (files || []) |
@@ -370,7 +380,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { | |||
370 | } | 380 | } |
371 | ] | 381 | ] |
372 | 382 | ||
373 | addVideoFilesInAPAcc(url, video, video.VideoFiles || []) | 383 | addVideoFilesInAPAcc({ acc: url, video, files: video.VideoFiles || [] }) |
374 | 384 | ||
375 | for (const playlist of (video.VideoStreamingPlaylists || [])) { | 385 | for (const playlist of (video.VideoStreamingPlaylists || [])) { |
376 | const tag = playlist.p2pMediaLoaderInfohashes | 386 | const tag = playlist.p2pMediaLoaderInfohashes |
@@ -382,7 +392,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { | |||
382 | href: playlist.getSha256SegmentsUrl(video) | 392 | href: playlist.getSha256SegmentsUrl(video) |
383 | }) | 393 | }) |
384 | 394 | ||
385 | addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) | 395 | addVideoFilesInAPAcc({ acc: tag, video, files: playlist.VideoFiles || [] }) |
386 | 396 | ||
387 | url.push({ | 397 | url.push({ |
388 | type: 'Link', | 398 | type: 'Link', |
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index d4f07f85f..9c4e6d078 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -22,8 +22,14 @@ import validator from 'validator' | |||
22 | import { logger } from '@server/helpers/logger' | 22 | import { logger } from '@server/helpers/logger' |
23 | import { extractVideo } from '@server/helpers/video' | 23 | import { extractVideo } from '@server/helpers/video' |
24 | import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' | 24 | import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' |
25 | import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' | 25 | import { |
26 | getHLSPrivateFileUrl, | ||
27 | getHLSPublicFileUrl, | ||
28 | getWebTorrentPrivateFileUrl, | ||
29 | getWebTorrentPublicFileUrl | ||
30 | } from '@server/lib/object-storage' | ||
26 | import { getFSTorrentFilePath } from '@server/lib/paths' | 31 | import { getFSTorrentFilePath } from '@server/lib/paths' |
32 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' | ||
27 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' | 33 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' |
28 | import { VideoResolution, VideoStorage } from '@shared/models' | 34 | import { VideoResolution, VideoStorage } from '@shared/models' |
29 | import { AttributesOnly } from '@shared/typescript-utils' | 35 | import { AttributesOnly } from '@shared/typescript-utils' |
@@ -48,6 +54,7 @@ import { doesExist } from '../shared' | |||
48 | import { parseAggregateResult, throwIfNotValid } from '../utils' | 54 | import { parseAggregateResult, throwIfNotValid } from '../utils' |
49 | import { VideoModel } from './video' | 55 | import { VideoModel } from './video' |
50 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | 56 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' |
57 | import { CONFIG } from '@server/initializers/config' | ||
51 | 58 | ||
52 | export enum ScopeNames { | 59 | export enum ScopeNames { |
53 | WITH_VIDEO = 'WITH_VIDEO', | 60 | WITH_VIDEO = 'WITH_VIDEO', |
@@ -295,6 +302,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
295 | return VideoFileModel.findOne(query) | 302 | return VideoFileModel.findOne(query) |
296 | } | 303 | } |
297 | 304 | ||
305 | static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> { | ||
306 | const query = { | ||
307 | where: { | ||
308 | filename | ||
309 | } | ||
310 | } | ||
311 | |||
312 | return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) | ||
313 | } | ||
314 | |||
298 | static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { | 315 | static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { |
299 | const query = { | 316 | const query = { |
300 | where: { | 317 | where: { |
@@ -305,6 +322,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
305 | return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) | 322 | return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) |
306 | } | 323 | } |
307 | 324 | ||
325 | static load (id: number): Promise<MVideoFile> { | ||
326 | return VideoFileModel.findByPk(id) | ||
327 | } | ||
328 | |||
308 | static loadWithMetadata (id: number) { | 329 | static loadWithMetadata (id: number) { |
309 | return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) | 330 | return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) |
310 | } | 331 | } |
@@ -467,7 +488,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
467 | } | 488 | } |
468 | 489 | ||
469 | getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { | 490 | getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { |
470 | if (this.videoId) return (this as MVideoFileVideo).Video | 491 | if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video |
471 | 492 | ||
472 | return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist | 493 | return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist |
473 | } | 494 | } |
@@ -488,7 +509,25 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
488 | return !!this.videoStreamingPlaylistId | 509 | return !!this.videoStreamingPlaylistId |
489 | } | 510 | } |
490 | 511 | ||
491 | getObjectStorageUrl () { | 512 | // --------------------------------------------------------------------------- |
513 | |||
514 | getObjectStorageUrl (video: MVideo) { | ||
515 | if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { | ||
516 | return this.getPrivateObjectStorageUrl(video) | ||
517 | } | ||
518 | |||
519 | return this.getPublicObjectStorageUrl() | ||
520 | } | ||
521 | |||
522 | private getPrivateObjectStorageUrl (video: MVideo) { | ||
523 | if (this.isHLS()) { | ||
524 | return getHLSPrivateFileUrl(video, this.filename) | ||
525 | } | ||
526 | |||
527 | return getWebTorrentPrivateFileUrl(this.filename) | ||
528 | } | ||
529 | |||
530 | private getPublicObjectStorageUrl () { | ||
492 | if (this.isHLS()) { | 531 | if (this.isHLS()) { |
493 | return getHLSPublicFileUrl(this.fileUrl) | 532 | return getHLSPublicFileUrl(this.fileUrl) |
494 | } | 533 | } |
@@ -496,23 +535,46 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
496 | return getWebTorrentPublicFileUrl(this.fileUrl) | 535 | return getWebTorrentPublicFileUrl(this.fileUrl) |
497 | } | 536 | } |
498 | 537 | ||
538 | // --------------------------------------------------------------------------- | ||
539 | |||
499 | getFileUrl (video: MVideo) { | 540 | getFileUrl (video: MVideo) { |
500 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | 541 | if (video.isOwned()) { |
501 | return this.getObjectStorageUrl() | 542 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
502 | } | 543 | return this.getObjectStorageUrl(video) |
544 | } | ||
503 | 545 | ||
504 | if (!this.Video) this.Video = video as VideoModel | 546 | return WEBSERVER.URL + this.getFileStaticPath(video) |
505 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) | 547 | } |
506 | 548 | ||
507 | return this.fileUrl | 549 | return this.fileUrl |
508 | } | 550 | } |
509 | 551 | ||
552 | // --------------------------------------------------------------------------- | ||
553 | |||
510 | getFileStaticPath (video: MVideo) { | 554 | getFileStaticPath (video: MVideo) { |
511 | if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) | 555 | if (this.isHLS()) return this.getHLSFileStaticPath(video) |
556 | |||
557 | return this.getWebTorrentFileStaticPath(video) | ||
558 | } | ||
559 | |||
560 | private getWebTorrentFileStaticPath (video: MVideo) { | ||
561 | if (isVideoInPrivateDirectory(video.privacy)) { | ||
562 | return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename) | ||
563 | } | ||
512 | 564 | ||
513 | return join(STATIC_PATHS.WEBSEED, this.filename) | 565 | return join(STATIC_PATHS.WEBSEED, this.filename) |
514 | } | 566 | } |
515 | 567 | ||
568 | private getHLSFileStaticPath (video: MVideo) { | ||
569 | if (isVideoInPrivateDirectory(video.privacy)) { | ||
570 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename) | ||
571 | } | ||
572 | |||
573 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) | ||
574 | } | ||
575 | |||
576 | // --------------------------------------------------------------------------- | ||
577 | |||
516 | getFileDownloadUrl (video: MVideoWithHost) { | 578 | getFileDownloadUrl (video: MVideoWithHost) { |
517 | const path = this.isHLS() | 579 | const path = this.isHLS() |
518 | ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) | 580 | ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) |
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts index 7497addf1..740f6b5c6 100644 --- a/server/models/video/video-job-info.ts +++ b/server/models/video/video-job-info.ts | |||
@@ -84,7 +84,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo | |||
84 | static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { | 84 | static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { |
85 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | 85 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } |
86 | 86 | ||
87 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | 87 | const result = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` |
88 | UPDATE | 88 | UPDATE |
89 | "videoJobInfo" | 89 | "videoJobInfo" |
90 | SET | 90 | SET |
@@ -97,7 +97,9 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo | |||
97 | "${column}"; | 97 | "${column}"; |
98 | `, options) | 98 | `, options) |
99 | 99 | ||
100 | return pendingMove | 100 | if (result.length === 0) return undefined |
101 | |||
102 | return result[0].pendingMove | ||
101 | } | 103 | } |
102 | 104 | ||
103 | static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> { | 105 | static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> { |
diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 81ce3dc9e..8bbe54c49 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts | |||
@@ -49,7 +49,7 @@ import { | |||
49 | MVideoPlaylistFormattable, | 49 | MVideoPlaylistFormattable, |
50 | MVideoPlaylistFull, | 50 | MVideoPlaylistFull, |
51 | MVideoPlaylistFullSummary, | 51 | MVideoPlaylistFullSummary, |
52 | MVideoPlaylistIdWithElements | 52 | MVideoPlaylistSummaryWithElements |
53 | } from '../../types/models/video/video-playlist' | 53 | } from '../../types/models/video/video-playlist' |
54 | import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions } from '../account/account' | 54 | import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions } from '../account/account' |
55 | import { ActorModel } from '../actor/actor' | 55 | import { ActorModel } from '../actor/actor' |
@@ -470,9 +470,9 @@ export class VideoPlaylistModel extends Model<Partial<AttributesOnly<VideoPlayli | |||
470 | })) | 470 | })) |
471 | } | 471 | } |
472 | 472 | ||
473 | static listPlaylistIdsOf (accountId: number, videoIds: number[]): Promise<MVideoPlaylistIdWithElements[]> { | 473 | static listPlaylistSummariesOf (accountId: number, videoIds: number[]): Promise<MVideoPlaylistSummaryWithElements[]> { |
474 | const query = { | 474 | const query = { |
475 | attributes: [ 'id' ], | 475 | attributes: [ 'id', 'name', 'uuid' ], |
476 | where: { | 476 | where: { |
477 | ownerAccountId: accountId | 477 | ownerAccountId: accountId |
478 | }, | 478 | }, |
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index f587989dc..0386edf28 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -15,8 +15,10 @@ import { | |||
15 | Table, | 15 | Table, |
16 | UpdatedAt | 16 | UpdatedAt |
17 | } from 'sequelize-typescript' | 17 | } from 'sequelize-typescript' |
18 | import { getHLSPublicFileUrl } from '@server/lib/object-storage' | 18 | import { CONFIG } from '@server/initializers/config' |
19 | import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage' | ||
19 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' | 20 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' |
21 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' | ||
20 | import { VideoFileModel } from '@server/models/video/video-file' | 22 | import { VideoFileModel } from '@server/models/video/video-file' |
21 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' | 23 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' |
22 | import { sha1 } from '@shared/extra-utils' | 24 | import { sha1 } from '@shared/extra-utils' |
@@ -244,26 +246,52 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
244 | this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files) | 246 | this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files) |
245 | } | 247 | } |
246 | 248 | ||
249 | // --------------------------------------------------------------------------- | ||
250 | |||
247 | getMasterPlaylistUrl (video: MVideo) { | 251 | getMasterPlaylistUrl (video: MVideo) { |
248 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | 252 | if (video.isOwned()) { |
249 | return getHLSPublicFileUrl(this.playlistUrl) | 253 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
250 | } | 254 | return this.getMasterPlaylistObjectStorageUrl(video) |
255 | } | ||
251 | 256 | ||
252 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) | 257 | return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video) |
258 | } | ||
253 | 259 | ||
254 | return this.playlistUrl | 260 | return this.playlistUrl |
255 | } | 261 | } |
256 | 262 | ||
257 | getSha256SegmentsUrl (video: MVideo) { | 263 | private getMasterPlaylistObjectStorageUrl (video: MVideo) { |
258 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | 264 | if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { |
259 | return getHLSPublicFileUrl(this.segmentsSha256Url) | 265 | return getHLSPrivateFileUrl(video, this.playlistFilename) |
260 | } | 266 | } |
261 | 267 | ||
262 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) | 268 | return getHLSPublicFileUrl(this.playlistUrl) |
269 | } | ||
270 | |||
271 | // --------------------------------------------------------------------------- | ||
272 | |||
273 | getSha256SegmentsUrl (video: MVideo) { | ||
274 | if (video.isOwned()) { | ||
275 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
276 | return this.getSha256SegmentsObjectStorageUrl(video) | ||
277 | } | ||
278 | |||
279 | return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video) | ||
280 | } | ||
263 | 281 | ||
264 | return this.segmentsSha256Url | 282 | return this.segmentsSha256Url |
265 | } | 283 | } |
266 | 284 | ||
285 | private getSha256SegmentsObjectStorageUrl (video: MVideo) { | ||
286 | if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { | ||
287 | return getHLSPrivateFileUrl(video, this.segmentsSha256Filename) | ||
288 | } | ||
289 | |||
290 | return getHLSPublicFileUrl(this.segmentsSha256Url) | ||
291 | } | ||
292 | |||
293 | // --------------------------------------------------------------------------- | ||
294 | |||
267 | getStringType () { | 295 | getStringType () { |
268 | if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' | 296 | if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' |
269 | 297 | ||
@@ -283,13 +311,19 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
283 | return Object.assign(this, { Video: video }) | 311 | return Object.assign(this, { Video: video }) |
284 | } | 312 | } |
285 | 313 | ||
286 | private getMasterPlaylistStaticPath (videoUUID: string) { | 314 | private getMasterPlaylistStaticPath (video: MVideo) { |
287 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename) | 315 | if (isVideoInPrivateDirectory(video.privacy)) { |
316 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename) | ||
317 | } | ||
318 | |||
319 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename) | ||
288 | } | 320 | } |
289 | 321 | ||
290 | private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) { | 322 | private getSha256SegmentsStaticPath (video: MVideo) { |
291 | if (isLive) return join('/live', 'segments-sha256', videoUUID) | 323 | if (isVideoInPrivateDirectory(video.privacy)) { |
324 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename) | ||
325 | } | ||
292 | 326 | ||
293 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename) | 327 | return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename) |
294 | } | 328 | } |
295 | } | 329 | } |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 468117504..56cc45cfe 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -26,14 +26,15 @@ import { | |||
26 | } from 'sequelize-typescript' | 26 | } from 'sequelize-typescript' |
27 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' | 27 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' |
28 | import { LiveManager } from '@server/lib/live/live-manager' | 28 | import { LiveManager } from '@server/lib/live/live-manager' |
29 | import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' | 29 | import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' |
30 | import { tracer } from '@server/lib/opentelemetry/tracing' | 30 | import { tracer } from '@server/lib/opentelemetry/tracing' |
31 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | 31 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
32 | import { VideoPathManager } from '@server/lib/video-path-manager' | 32 | import { VideoPathManager } from '@server/lib/video-path-manager' |
33 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' | ||
33 | import { getServerActor } from '@server/models/application/application' | 34 | import { getServerActor } from '@server/models/application/application' |
34 | import { ModelCache } from '@server/models/model-cache' | 35 | import { ModelCache } from '@server/models/model-cache' |
35 | import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' | 36 | import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' |
36 | import { ffprobePromise, getAudioStream, uuidToShort } from '@shared/extra-utils' | 37 | import { ffprobePromise, getAudioStream, hasAudioStream, uuidToShort } from '@shared/extra-utils' |
37 | import { | 38 | import { |
38 | ResultList, | 39 | ResultList, |
39 | ThumbnailType, | 40 | ThumbnailType, |
@@ -52,7 +53,7 @@ import { | |||
52 | import { AttributesOnly } from '@shared/typescript-utils' | 53 | import { AttributesOnly } from '@shared/typescript-utils' |
53 | import { peertubeTruncate } from '../../helpers/core-utils' | 54 | import { peertubeTruncate } from '../../helpers/core-utils' |
54 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 55 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
55 | import { exists, isBooleanValid } from '../../helpers/custom-validators/misc' | 56 | import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc' |
56 | import { | 57 | import { |
57 | isVideoDescriptionValid, | 58 | isVideoDescriptionValid, |
58 | isVideoDurationValid, | 59 | isVideoDurationValid, |
@@ -784,9 +785,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
784 | 785 | ||
785 | // Do not wait video deletion because we could be in a transaction | 786 | // Do not wait video deletion because we could be in a transaction |
786 | Promise.all(tasks) | 787 | Promise.all(tasks) |
787 | .catch(err => { | 788 | .then(() => logger.info('Removed files of video %s.', instance.url)) |
788 | logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err }) | 789 | .catch(err => logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err })) |
789 | }) | ||
790 | 790 | ||
791 | return undefined | 791 | return undefined |
792 | } | 792 | } |
@@ -1458,6 +1458,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1458 | const query = 'SELECT 1 FROM "videoShare" ' + | 1458 | const query = 'SELECT 1 FROM "videoShare" ' + |
1459 | 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + | 1459 | 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + |
1460 | 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' + | 1460 | 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' + |
1461 | 'UNION ' + | ||
1462 | 'SELECT 1 FROM "video" ' + | ||
1463 | 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + | ||
1464 | 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + | ||
1465 | 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "account"."actorId" ' + | ||
1466 | 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "video"."id" = $videoId ' + | ||
1461 | 'LIMIT 1' | 1467 | 'LIMIT 1' |
1462 | 1468 | ||
1463 | const options = { | 1469 | const options = { |
@@ -1696,12 +1702,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1696 | let files: VideoFile[] = [] | 1702 | let files: VideoFile[] = [] |
1697 | 1703 | ||
1698 | if (Array.isArray(this.VideoFiles)) { | 1704 | if (Array.isArray(this.VideoFiles)) { |
1699 | const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, includeMagnet) | 1705 | const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet }) |
1700 | files = files.concat(result) | 1706 | files = files.concat(result) |
1701 | } | 1707 | } |
1702 | 1708 | ||
1703 | for (const p of (this.VideoStreamingPlaylists || [])) { | 1709 | for (const p of (this.VideoStreamingPlaylists || [])) { |
1704 | const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, includeMagnet) | 1710 | const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet }) |
1705 | files = files.concat(result) | 1711 | files = files.concat(result) |
1706 | } | 1712 | } |
1707 | 1713 | ||
@@ -1745,9 +1751,11 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1745 | const probe = await ffprobePromise(originalFilePath) | 1751 | const probe = await ffprobePromise(originalFilePath) |
1746 | 1752 | ||
1747 | const { audioStream } = await getAudioStream(originalFilePath, probe) | 1753 | const { audioStream } = await getAudioStream(originalFilePath, probe) |
1754 | const hasAudio = await hasAudioStream(originalFilePath, probe) | ||
1748 | 1755 | ||
1749 | return { | 1756 | return { |
1750 | audioStream, | 1757 | audioStream, |
1758 | hasAudio, | ||
1751 | 1759 | ||
1752 | ...await getVideoStreamDimensionsInfo(originalFilePath, probe) | 1760 | ...await getVideoStreamDimensionsInfo(originalFilePath, probe) |
1753 | } | 1761 | } |
@@ -1764,9 +1772,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1764 | const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) | 1772 | const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) |
1765 | if (!playlist) return undefined | 1773 | if (!playlist) return undefined |
1766 | 1774 | ||
1767 | playlist.Video = this | 1775 | return playlist.withVideo(this) |
1768 | |||
1769 | return playlist | ||
1770 | } | 1776 | } |
1771 | 1777 | ||
1772 | setHLSPlaylist (playlist: MStreamingPlaylist) { | 1778 | setHLSPlaylist (playlist: MStreamingPlaylist) { |
@@ -1832,8 +1838,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1832 | await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) | 1838 | await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) |
1833 | 1839 | ||
1834 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { | 1840 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
1835 | await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename) | 1841 | await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), videoFile.filename) |
1836 | await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), resolutionFilename) | 1842 | await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), resolutionFilename) |
1837 | } | 1843 | } |
1838 | } | 1844 | } |
1839 | 1845 | ||
@@ -1842,7 +1848,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1842 | await remove(filePath) | 1848 | await remove(filePath) |
1843 | 1849 | ||
1844 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | 1850 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { |
1845 | await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), filename) | 1851 | await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), filename) |
1846 | } | 1852 | } |
1847 | } | 1853 | } |
1848 | 1854 | ||
@@ -1868,24 +1874,39 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1868 | return setAsUpdated('video', this.id, transaction) | 1874 | return setAsUpdated('video', this.id, transaction) |
1869 | } | 1875 | } |
1870 | 1876 | ||
1871 | requiresAuth () { | 1877 | // --------------------------------------------------------------------------- |
1872 | return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist | ||
1873 | } | ||
1874 | 1878 | ||
1875 | setPrivacy (newPrivacy: VideoPrivacy) { | 1879 | requiresAuth (options: { |
1876 | if (this.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { | 1880 | urlParamId: string |
1877 | this.publishedAt = new Date() | 1881 | checkBlacklist: boolean |
1882 | }) { | ||
1883 | const { urlParamId, checkBlacklist } = options | ||
1884 | |||
1885 | if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) { | ||
1886 | return true | ||
1887 | } | ||
1888 | |||
1889 | if (this.privacy === VideoPrivacy.UNLISTED) { | ||
1890 | if (urlParamId && !isUUIDValid(urlParamId)) return true | ||
1891 | |||
1892 | return false | ||
1893 | } | ||
1894 | |||
1895 | if (checkBlacklist && this.VideoBlacklist) return true | ||
1896 | |||
1897 | if (this.privacy !== VideoPrivacy.PUBLIC) { | ||
1898 | throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`) | ||
1878 | } | 1899 | } |
1879 | 1900 | ||
1880 | this.privacy = newPrivacy | 1901 | return false |
1881 | } | 1902 | } |
1882 | 1903 | ||
1883 | isConfidential () { | 1904 | hasPrivateStaticPath () { |
1884 | return this.privacy === VideoPrivacy.PRIVATE || | 1905 | return isVideoInPrivateDirectory(this.privacy) |
1885 | this.privacy === VideoPrivacy.UNLISTED || | ||
1886 | this.privacy === VideoPrivacy.INTERNAL | ||
1887 | } | 1906 | } |
1888 | 1907 | ||
1908 | // --------------------------------------------------------------------------- | ||
1909 | |||
1889 | async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { | 1910 | async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { |
1890 | if (this.state === newState) throw new Error('Cannot use same state ' + newState) | 1911 | if (this.state === newState) throw new Error('Cannot use same state ' + newState) |
1891 | 1912 | ||
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index cd7a38459..961093bb5 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts | |||
@@ -2,6 +2,7 @@ import './abuses' | |||
2 | import './accounts' | 2 | import './accounts' |
3 | import './blocklist' | 3 | import './blocklist' |
4 | import './bulk' | 4 | import './bulk' |
5 | import './channel-import-videos' | ||
5 | import './config' | 6 | import './config' |
6 | import './contact-form' | 7 | import './contact-form' |
7 | import './custom-pages' | 8 | import './custom-pages' |
@@ -17,6 +18,7 @@ import './redundancy' | |||
17 | import './search' | 18 | import './search' |
18 | import './services' | 19 | import './services' |
19 | import './transcoding' | 20 | import './transcoding' |
21 | import './two-factor' | ||
20 | import './upload-quota' | 22 | import './upload-quota' |
21 | import './user-notifications' | 23 | import './user-notifications' |
22 | import './user-subscriptions' | 24 | import './user-subscriptions' |
@@ -24,15 +26,15 @@ import './users-admin' | |||
24 | import './users' | 26 | import './users' |
25 | import './video-blacklist' | 27 | import './video-blacklist' |
26 | import './video-captions' | 28 | import './video-captions' |
29 | import './video-channel-syncs' | ||
27 | import './video-channels' | 30 | import './video-channels' |
28 | import './video-comments' | 31 | import './video-comments' |
29 | import './video-files' | 32 | import './video-files' |
30 | import './video-imports' | 33 | import './video-imports' |
31 | import './video-channel-syncs' | ||
32 | import './channel-import-videos' | ||
33 | import './video-playlists' | 34 | import './video-playlists' |
34 | import './video-source' | 35 | import './video-source' |
35 | import './video-studio' | 36 | import './video-studio' |
37 | import './video-token' | ||
36 | import './videos-common-filters' | 38 | import './videos-common-filters' |
37 | import './videos-history' | 39 | import './videos-history' |
38 | import './videos-overviews' | 40 | import './videos-overviews' |
diff --git a/server/tests/api/check-params/live.ts b/server/tests/api/check-params/live.ts index 3f553c42b..2eff9414b 100644 --- a/server/tests/api/check-params/live.ts +++ b/server/tests/api/check-params/live.ts | |||
@@ -502,6 +502,23 @@ describe('Test video lives API validator', function () { | |||
502 | await stopFfmpeg(ffmpegCommand) | 502 | await stopFfmpeg(ffmpegCommand) |
503 | }) | 503 | }) |
504 | 504 | ||
505 | it('Should fail to change live privacy if it has already started', async function () { | ||
506 | this.timeout(40000) | ||
507 | |||
508 | const live = await command.get({ videoId: video.id }) | ||
509 | |||
510 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) | ||
511 | |||
512 | await command.waitUntilPublished({ videoId: video.id }) | ||
513 | await server.videos.update({ | ||
514 | id: video.id, | ||
515 | attributes: { privacy: VideoPrivacy.PUBLIC }, | ||
516 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
517 | }) | ||
518 | |||
519 | await stopFfmpeg(ffmpegCommand) | ||
520 | }) | ||
521 | |||
505 | it('Should fail to stream twice in the save live', async function () { | 522 | it('Should fail to stream twice in the save live', async function () { |
506 | this.timeout(40000) | 523 | this.timeout(40000) |
507 | 524 | ||
diff --git a/server/tests/api/check-params/two-factor.ts b/server/tests/api/check-params/two-factor.ts new file mode 100644 index 000000000..f8365f1b5 --- /dev/null +++ b/server/tests/api/check-params/two-factor.ts | |||
@@ -0,0 +1,288 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { HttpStatusCode } from '@shared/models' | ||
4 | import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands' | ||
5 | |||
6 | describe('Test two factor API validators', function () { | ||
7 | let server: PeerTubeServer | ||
8 | |||
9 | let rootId: number | ||
10 | let rootPassword: string | ||
11 | let rootRequestToken: string | ||
12 | let rootOTPToken: string | ||
13 | |||
14 | let userId: number | ||
15 | let userToken = '' | ||
16 | let userPassword: string | ||
17 | let userRequestToken: string | ||
18 | let userOTPToken: string | ||
19 | |||
20 | // --------------------------------------------------------------- | ||
21 | |||
22 | before(async function () { | ||
23 | this.timeout(30000) | ||
24 | |||
25 | { | ||
26 | server = await createSingleServer(1) | ||
27 | await setAccessTokensToServers([ server ]) | ||
28 | } | ||
29 | |||
30 | { | ||
31 | const result = await server.users.generate('user1') | ||
32 | userToken = result.token | ||
33 | userId = result.userId | ||
34 | userPassword = result.password | ||
35 | } | ||
36 | |||
37 | { | ||
38 | const { id } = await server.users.getMyInfo() | ||
39 | rootId = id | ||
40 | rootPassword = server.store.user.password | ||
41 | } | ||
42 | }) | ||
43 | |||
44 | describe('When requesting two factor', function () { | ||
45 | |||
46 | it('Should fail with an unknown user id', async function () { | ||
47 | await server.twoFactor.request({ userId: 42, currentPassword: rootPassword, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | ||
48 | }) | ||
49 | |||
50 | it('Should fail with an invalid user id', async function () { | ||
51 | await server.twoFactor.request({ | ||
52 | userId: 'invalid' as any, | ||
53 | currentPassword: rootPassword, | ||
54 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
55 | }) | ||
56 | }) | ||
57 | |||
58 | it('Should fail to request another user two factor without the appropriate rights', async function () { | ||
59 | await server.twoFactor.request({ | ||
60 | userId: rootId, | ||
61 | token: userToken, | ||
62 | currentPassword: userPassword, | ||
63 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
64 | }) | ||
65 | }) | ||
66 | |||
67 | it('Should succeed to request another user two factor with the appropriate rights', async function () { | ||
68 | await server.twoFactor.request({ userId, currentPassword: rootPassword }) | ||
69 | }) | ||
70 | |||
71 | it('Should fail to request two factor without a password', async function () { | ||
72 | await server.twoFactor.request({ | ||
73 | userId, | ||
74 | token: userToken, | ||
75 | currentPassword: undefined, | ||
76 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
77 | }) | ||
78 | }) | ||
79 | |||
80 | it('Should fail to request two factor with an incorrect password', async function () { | ||
81 | await server.twoFactor.request({ | ||
82 | userId, | ||
83 | token: userToken, | ||
84 | currentPassword: rootPassword, | ||
85 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
86 | }) | ||
87 | }) | ||
88 | |||
89 | it('Should succeed to request two factor without a password when targeting a remote user with an admin account', async function () { | ||
90 | await server.twoFactor.request({ userId }) | ||
91 | }) | ||
92 | |||
93 | it('Should fail to request two factor without a password when targeting myself with an admin account', async function () { | ||
94 | await server.twoFactor.request({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
95 | await server.twoFactor.request({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
96 | }) | ||
97 | |||
98 | it('Should succeed to request my two factor auth', async function () { | ||
99 | { | ||
100 | const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword }) | ||
101 | userRequestToken = otpRequest.requestToken | ||
102 | userOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() | ||
103 | } | ||
104 | |||
105 | { | ||
106 | const { otpRequest } = await server.twoFactor.request({ userId: rootId, currentPassword: rootPassword }) | ||
107 | rootRequestToken = otpRequest.requestToken | ||
108 | rootOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() | ||
109 | } | ||
110 | }) | ||
111 | }) | ||
112 | |||
113 | describe('When confirming two factor request', function () { | ||
114 | |||
115 | it('Should fail with an unknown user id', async function () { | ||
116 | await server.twoFactor.confirmRequest({ | ||
117 | userId: 42, | ||
118 | requestToken: rootRequestToken, | ||
119 | otpToken: rootOTPToken, | ||
120 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
121 | }) | ||
122 | }) | ||
123 | |||
124 | it('Should fail with an invalid user id', async function () { | ||
125 | await server.twoFactor.confirmRequest({ | ||
126 | userId: 'invalid' as any, | ||
127 | requestToken: rootRequestToken, | ||
128 | otpToken: rootOTPToken, | ||
129 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
130 | }) | ||
131 | }) | ||
132 | |||
133 | it('Should fail to confirm another user two factor request without the appropriate rights', async function () { | ||
134 | await server.twoFactor.confirmRequest({ | ||
135 | userId: rootId, | ||
136 | token: userToken, | ||
137 | requestToken: rootRequestToken, | ||
138 | otpToken: rootOTPToken, | ||
139 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
140 | }) | ||
141 | }) | ||
142 | |||
143 | it('Should fail without request token', async function () { | ||
144 | await server.twoFactor.confirmRequest({ | ||
145 | userId, | ||
146 | requestToken: undefined, | ||
147 | otpToken: userOTPToken, | ||
148 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
149 | }) | ||
150 | }) | ||
151 | |||
152 | it('Should fail with an invalid request token', async function () { | ||
153 | await server.twoFactor.confirmRequest({ | ||
154 | userId, | ||
155 | requestToken: 'toto', | ||
156 | otpToken: userOTPToken, | ||
157 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
158 | }) | ||
159 | }) | ||
160 | |||
161 | it('Should fail with request token of another user', async function () { | ||
162 | await server.twoFactor.confirmRequest({ | ||
163 | userId, | ||
164 | requestToken: rootRequestToken, | ||
165 | otpToken: userOTPToken, | ||
166 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
167 | }) | ||
168 | }) | ||
169 | |||
170 | it('Should fail without an otp token', async function () { | ||
171 | await server.twoFactor.confirmRequest({ | ||
172 | userId, | ||
173 | requestToken: userRequestToken, | ||
174 | otpToken: undefined, | ||
175 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
176 | }) | ||
177 | }) | ||
178 | |||
179 | it('Should fail with a bad otp token', async function () { | ||
180 | await server.twoFactor.confirmRequest({ | ||
181 | userId, | ||
182 | requestToken: userRequestToken, | ||
183 | otpToken: '123456', | ||
184 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
185 | }) | ||
186 | }) | ||
187 | |||
188 | it('Should succeed to confirm another user two factor request with the appropriate rights', async function () { | ||
189 | await server.twoFactor.confirmRequest({ | ||
190 | userId, | ||
191 | requestToken: userRequestToken, | ||
192 | otpToken: userOTPToken | ||
193 | }) | ||
194 | |||
195 | // Reinit | ||
196 | await server.twoFactor.disable({ userId, currentPassword: rootPassword }) | ||
197 | }) | ||
198 | |||
199 | it('Should succeed to confirm my two factor request', async function () { | ||
200 | await server.twoFactor.confirmRequest({ | ||
201 | userId, | ||
202 | token: userToken, | ||
203 | requestToken: userRequestToken, | ||
204 | otpToken: userOTPToken | ||
205 | }) | ||
206 | }) | ||
207 | |||
208 | it('Should fail to confirm again two factor request', async function () { | ||
209 | await server.twoFactor.confirmRequest({ | ||
210 | userId, | ||
211 | token: userToken, | ||
212 | requestToken: userRequestToken, | ||
213 | otpToken: userOTPToken, | ||
214 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
215 | }) | ||
216 | }) | ||
217 | }) | ||
218 | |||
219 | describe('When disabling two factor', function () { | ||
220 | |||
221 | it('Should fail with an unknown user id', async function () { | ||
222 | await server.twoFactor.disable({ | ||
223 | userId: 42, | ||
224 | currentPassword: rootPassword, | ||
225 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
226 | }) | ||
227 | }) | ||
228 | |||
229 | it('Should fail with an invalid user id', async function () { | ||
230 | await server.twoFactor.disable({ | ||
231 | userId: 'invalid' as any, | ||
232 | currentPassword: rootPassword, | ||
233 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
234 | }) | ||
235 | }) | ||
236 | |||
237 | it('Should fail to disable another user two factor without the appropriate rights', async function () { | ||
238 | await server.twoFactor.disable({ | ||
239 | userId: rootId, | ||
240 | token: userToken, | ||
241 | currentPassword: userPassword, | ||
242 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
243 | }) | ||
244 | }) | ||
245 | |||
246 | it('Should fail to disable two factor with an incorrect password', async function () { | ||
247 | await server.twoFactor.disable({ | ||
248 | userId, | ||
249 | token: userToken, | ||
250 | currentPassword: rootPassword, | ||
251 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
252 | }) | ||
253 | }) | ||
254 | |||
255 | it('Should succeed to disable two factor without a password when targeting a remote user with an admin account', async function () { | ||
256 | await server.twoFactor.disable({ userId }) | ||
257 | await server.twoFactor.requestAndConfirm({ userId }) | ||
258 | }) | ||
259 | |||
260 | it('Should fail to disable two factor without a password when targeting myself with an admin account', async function () { | ||
261 | await server.twoFactor.disable({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
262 | await server.twoFactor.disable({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
263 | }) | ||
264 | |||
265 | it('Should succeed to disable another user two factor with the appropriate rights', async function () { | ||
266 | await server.twoFactor.disable({ userId, currentPassword: rootPassword }) | ||
267 | |||
268 | await server.twoFactor.requestAndConfirm({ userId }) | ||
269 | }) | ||
270 | |||
271 | it('Should succeed to update my two factor auth', async function () { | ||
272 | await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword }) | ||
273 | }) | ||
274 | |||
275 | it('Should fail to disable again two factor', async function () { | ||
276 | await server.twoFactor.disable({ | ||
277 | userId, | ||
278 | token: userToken, | ||
279 | currentPassword: userPassword, | ||
280 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
281 | }) | ||
282 | }) | ||
283 | }) | ||
284 | |||
285 | after(async function () { | ||
286 | await cleanupTests([ server ]) | ||
287 | }) | ||
288 | }) | ||
diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts index aa4de2c83..9dc59a1b5 100644 --- a/server/tests/api/check-params/video-files.ts +++ b/server/tests/api/check-params/video-files.ts | |||
@@ -1,10 +1,12 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { HttpStatusCode, UserRole } from '@shared/models' | 3 | import { getAllFiles } from '@shared/core-utils' |
4 | import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@shared/models' | ||
4 | import { | 5 | import { |
5 | cleanupTests, | 6 | cleanupTests, |
6 | createMultipleServers, | 7 | createMultipleServers, |
7 | doubleFollow, | 8 | doubleFollow, |
9 | makeRawRequest, | ||
8 | PeerTubeServer, | 10 | PeerTubeServer, |
9 | setAccessTokensToServers, | 11 | setAccessTokensToServers, |
10 | waitJobs | 12 | waitJobs |
@@ -13,22 +15,9 @@ import { | |||
13 | describe('Test videos files', function () { | 15 | describe('Test videos files', function () { |
14 | let servers: PeerTubeServer[] | 16 | let servers: PeerTubeServer[] |
15 | 17 | ||
16 | let webtorrentId: string | ||
17 | let hlsId: string | ||
18 | let remoteId: string | ||
19 | |||
20 | let userToken: string | 18 | let userToken: string |
21 | let moderatorToken: string | 19 | let moderatorToken: string |
22 | 20 | ||
23 | let validId1: string | ||
24 | let validId2: string | ||
25 | |||
26 | let hlsFileId: number | ||
27 | let webtorrentFileId: number | ||
28 | |||
29 | let remoteHLSFileId: number | ||
30 | let remoteWebtorrentFileId: number | ||
31 | |||
32 | // --------------------------------------------------------------- | 21 | // --------------------------------------------------------------- |
33 | 22 | ||
34 | before(async function () { | 23 | before(async function () { |
@@ -41,117 +30,163 @@ describe('Test videos files', function () { | |||
41 | 30 | ||
42 | userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) | 31 | userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) |
43 | moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) | 32 | moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) |
33 | }) | ||
44 | 34 | ||
45 | { | 35 | describe('Getting metadata', function () { |
46 | const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) | 36 | let video: VideoDetails |
47 | await waitJobs(servers) | 37 | |
38 | before(async function () { | ||
39 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
40 | video = await servers[0].videos.getWithToken({ id: uuid }) | ||
41 | }) | ||
42 | |||
43 | it('Should not get metadata of private video without token', async function () { | ||
44 | for (const file of getAllFiles(video)) { | ||
45 | await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | ||
46 | } | ||
47 | }) | ||
48 | |||
49 | it('Should not get metadata of private video without the appropriate token', async function () { | ||
50 | for (const file of getAllFiles(video)) { | ||
51 | await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
52 | } | ||
53 | }) | ||
54 | |||
55 | it('Should get metadata of private video with the appropriate token', async function () { | ||
56 | for (const file of getAllFiles(video)) { | ||
57 | await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
58 | } | ||
59 | }) | ||
60 | }) | ||
61 | |||
62 | describe('Deleting files', function () { | ||
63 | let webtorrentId: string | ||
64 | let hlsId: string | ||
65 | let remoteId: string | ||
66 | |||
67 | let validId1: string | ||
68 | let validId2: string | ||
48 | 69 | ||
49 | const video = await servers[1].videos.get({ id: uuid }) | 70 | let hlsFileId: number |
50 | remoteId = video.uuid | 71 | let webtorrentFileId: number |
51 | remoteHLSFileId = video.streamingPlaylists[0].files[0].id | ||
52 | remoteWebtorrentFileId = video.files[0].id | ||
53 | } | ||
54 | 72 | ||
55 | { | 73 | let remoteHLSFileId: number |
56 | await servers[0].config.enableTranscoding(true, true) | 74 | let remoteWebtorrentFileId: number |
75 | |||
76 | before(async function () { | ||
77 | this.timeout(300_000) | ||
57 | 78 | ||
58 | { | 79 | { |
59 | const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) | 80 | const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) |
60 | await waitJobs(servers) | 81 | await waitJobs(servers) |
61 | 82 | ||
62 | const video = await servers[0].videos.get({ id: uuid }) | 83 | const video = await servers[1].videos.get({ id: uuid }) |
63 | validId1 = video.uuid | 84 | remoteId = video.uuid |
64 | hlsFileId = video.streamingPlaylists[0].files[0].id | 85 | remoteHLSFileId = video.streamingPlaylists[0].files[0].id |
65 | webtorrentFileId = video.files[0].id | 86 | remoteWebtorrentFileId = video.files[0].id |
66 | } | 87 | } |
67 | 88 | ||
68 | { | 89 | { |
69 | const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' }) | 90 | await servers[0].config.enableTranscoding(true, true) |
70 | validId2 = uuid | 91 | |
92 | { | ||
93 | const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) | ||
94 | await waitJobs(servers) | ||
95 | |||
96 | const video = await servers[0].videos.get({ id: uuid }) | ||
97 | validId1 = video.uuid | ||
98 | hlsFileId = video.streamingPlaylists[0].files[0].id | ||
99 | webtorrentFileId = video.files[0].id | ||
100 | } | ||
101 | |||
102 | { | ||
103 | const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' }) | ||
104 | validId2 = uuid | ||
105 | } | ||
71 | } | 106 | } |
72 | } | ||
73 | 107 | ||
74 | await waitJobs(servers) | 108 | await waitJobs(servers) |
75 | 109 | ||
76 | { | 110 | { |
77 | await servers[0].config.enableTranscoding(false, true) | 111 | await servers[0].config.enableTranscoding(false, true) |
78 | const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) | 112 | const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) |
79 | hlsId = uuid | 113 | hlsId = uuid |
80 | } | 114 | } |
81 | 115 | ||
82 | await waitJobs(servers) | 116 | await waitJobs(servers) |
83 | 117 | ||
84 | { | 118 | { |
85 | await servers[0].config.enableTranscoding(false, true) | 119 | await servers[0].config.enableTranscoding(false, true) |
86 | const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' }) | 120 | const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' }) |
87 | webtorrentId = uuid | 121 | webtorrentId = uuid |
88 | } | 122 | } |
89 | 123 | ||
90 | await waitJobs(servers) | 124 | await waitJobs(servers) |
91 | }) | 125 | }) |
92 | 126 | ||
93 | it('Should not delete files of a unknown video', async function () { | 127 | it('Should not delete files of a unknown video', async function () { |
94 | const expectedStatus = HttpStatusCode.NOT_FOUND_404 | 128 | const expectedStatus = HttpStatusCode.NOT_FOUND_404 |
95 | 129 | ||
96 | await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) | 130 | await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) |
97 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus }) | 131 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus }) |
98 | 132 | ||
99 | await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) | 133 | await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) |
100 | await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus }) | 134 | await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus }) |
101 | }) | 135 | }) |
102 | 136 | ||
103 | it('Should not delete unknown files', async function () { | 137 | it('Should not delete unknown files', async function () { |
104 | const expectedStatus = HttpStatusCode.NOT_FOUND_404 | 138 | const expectedStatus = HttpStatusCode.NOT_FOUND_404 |
105 | 139 | ||
106 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus }) | 140 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus }) |
107 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) | 141 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) |
108 | }) | 142 | }) |
109 | 143 | ||
110 | it('Should not delete files of a remote video', async function () { | 144 | it('Should not delete files of a remote video', async function () { |
111 | const expectedStatus = HttpStatusCode.BAD_REQUEST_400 | 145 | const expectedStatus = HttpStatusCode.BAD_REQUEST_400 |
112 | 146 | ||
113 | await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) | 147 | await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) |
114 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus }) | 148 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus }) |
115 | 149 | ||
116 | await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) | 150 | await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) |
117 | await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus }) | 151 | await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus }) |
118 | }) | 152 | }) |
119 | 153 | ||
120 | it('Should not delete files by a non admin user', async function () { | 154 | it('Should not delete files by a non admin user', async function () { |
121 | const expectedStatus = HttpStatusCode.FORBIDDEN_403 | 155 | const expectedStatus = HttpStatusCode.FORBIDDEN_403 |
122 | 156 | ||
123 | await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) | 157 | await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) |
124 | await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) | 158 | await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) |
125 | 159 | ||
126 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) | 160 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) |
127 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) | 161 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) |
128 | 162 | ||
129 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) | 163 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) |
130 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) | 164 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) |
131 | 165 | ||
132 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus }) | 166 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus }) |
133 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus }) | 167 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus }) |
134 | }) | 168 | }) |
135 | 169 | ||
136 | it('Should not delete files if the files are not available', async function () { | 170 | it('Should not delete files if the files are not available', async function () { |
137 | await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 171 | await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) |
138 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 172 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) |
139 | 173 | ||
140 | await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 174 | await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
141 | await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 175 | await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
142 | }) | 176 | }) |
143 | 177 | ||
144 | it('Should not delete files if no both versions are available', async function () { | 178 | it('Should not delete files if no both versions are available', async function () { |
145 | await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 179 | await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) |
146 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | 180 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) |
147 | }) | 181 | }) |
148 | 182 | ||
149 | it('Should delete files if both versions are available', async function () { | 183 | it('Should delete files if both versions are available', async function () { |
150 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) | 184 | await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) |
151 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId }) | 185 | await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId }) |
152 | 186 | ||
153 | await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) | 187 | await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) |
154 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 }) | 188 | await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 }) |
189 | }) | ||
155 | }) | 190 | }) |
156 | 191 | ||
157 | after(async function () { | 192 | after(async function () { |
diff --git a/server/tests/api/check-params/video-token.ts b/server/tests/api/check-params/video-token.ts new file mode 100644 index 000000000..7acb9d580 --- /dev/null +++ b/server/tests/api/check-params/video-token.ts | |||
@@ -0,0 +1,44 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | ||
4 | import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' | ||
5 | |||
6 | describe('Test video tokens', function () { | ||
7 | let server: PeerTubeServer | ||
8 | let videoId: string | ||
9 | let userToken: string | ||
10 | |||
11 | // --------------------------------------------------------------- | ||
12 | |||
13 | before(async function () { | ||
14 | this.timeout(300_000) | ||
15 | |||
16 | server = await createSingleServer(1) | ||
17 | await setAccessTokensToServers([ server ]) | ||
18 | |||
19 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
20 | videoId = uuid | ||
21 | |||
22 | userToken = await server.users.generateUserAndToken('user1') | ||
23 | }) | ||
24 | |||
25 | it('Should not generate tokens for unauthenticated user', async function () { | ||
26 | await server.videoToken.create({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | ||
27 | }) | ||
28 | |||
29 | it('Should not generate tokens of unknown video', async function () { | ||
30 | await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | ||
31 | }) | ||
32 | |||
33 | it('Should not generate tokens of a non owned video', async function () { | ||
34 | await server.videoToken.create({ videoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
35 | }) | ||
36 | |||
37 | it('Should generate token', async function () { | ||
38 | await server.videoToken.create({ videoId }) | ||
39 | }) | ||
40 | |||
41 | after(async function () { | ||
42 | await cleanupTests([ server ]) | ||
43 | }) | ||
44 | }) | ||
diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts index 64ef73a2a..c82585a9e 100644 --- a/server/tests/api/live/live-constraints.ts +++ b/server/tests/api/live/live-constraints.ts | |||
@@ -49,7 +49,7 @@ describe('Test live constraints', function () { | |||
49 | expect(video.duration).to.be.greaterThan(0) | 49 | expect(video.duration).to.be.greaterThan(0) |
50 | } | 50 | } |
51 | 51 | ||
52 | await checkLiveCleanup(servers[0], videoId, resolutions) | 52 | await checkLiveCleanup({ server: servers[0], permanent: false, videoUUID: videoId, savedResolutions: resolutions }) |
53 | } | 53 | } |
54 | 54 | ||
55 | function updateQuota (options: { total: number, daily: number }) { | 55 | function updateQuota (options: { total: number, daily: number }) { |
diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts index 502959258..971df1a61 100644 --- a/server/tests/api/live/live-fast-restream.ts +++ b/server/tests/api/live/live-fast-restream.ts | |||
@@ -43,12 +43,31 @@ describe('Fast restream in live', function () { | |||
43 | // Streaming session #1 | 43 | // Streaming session #1 |
44 | let ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) | 44 | let ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) |
45 | await server.live.waitUntilPublished({ videoId: liveVideoUUID }) | 45 | await server.live.waitUntilPublished({ videoId: liveVideoUUID }) |
46 | |||
47 | const video = await server.videos.get({ id: liveVideoUUID }) | ||
48 | const session1PlaylistId = video.streamingPlaylists[0].id | ||
49 | |||
46 | await stopFfmpeg(ffmpegCommand) | 50 | await stopFfmpeg(ffmpegCommand) |
47 | await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) | 51 | await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) |
48 | 52 | ||
49 | // Streaming session #2 | 53 | // Streaming session #2 |
50 | ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) | 54 | ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) |
51 | await server.live.waitUntilSegmentGeneration({ videoUUID: liveVideoUUID, segment: 0, playlistNumber: 0, totalSessions: 2 }) | 55 | |
56 | let hasNewPlaylist = false | ||
57 | do { | ||
58 | const video = await server.videos.get({ id: liveVideoUUID }) | ||
59 | hasNewPlaylist = video.streamingPlaylists.length === 1 && video.streamingPlaylists[0].id !== session1PlaylistId | ||
60 | |||
61 | await wait(100) | ||
62 | } while (!hasNewPlaylist) | ||
63 | |||
64 | await server.live.waitUntilSegmentGeneration({ | ||
65 | server, | ||
66 | videoUUID: liveVideoUUID, | ||
67 | segment: 1, | ||
68 | playlistNumber: 0, | ||
69 | objectStorage: false | ||
70 | }) | ||
52 | 71 | ||
53 | return { ffmpegCommand, liveVideoUUID } | 72 | return { ffmpegCommand, liveVideoUUID } |
54 | } | 73 | } |
@@ -59,9 +78,9 @@ describe('Fast restream in live', function () { | |||
59 | const video = await server.videos.get({ id: liveId }) | 78 | const video = await server.videos.get({ id: liveId }) |
60 | expect(video.streamingPlaylists).to.have.lengthOf(1) | 79 | expect(video.streamingPlaylists).to.have.lengthOf(1) |
61 | 80 | ||
62 | await server.live.getSegment({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) | 81 | await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) |
63 | await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200) | 82 | await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
64 | await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200) | 83 | await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
65 | 84 | ||
66 | await wait(100) | 85 | await wait(100) |
67 | } | 86 | } |
diff --git a/server/tests/api/live/live-permanent.ts b/server/tests/api/live/live-permanent.ts index 5d227200e..4203b1bfc 100644 --- a/server/tests/api/live/live-permanent.ts +++ b/server/tests/api/live/live-permanent.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { checkLiveCleanup } from '@server/tests/shared' | ||
4 | import { wait } from '@shared/core-utils' | 5 | import { wait } from '@shared/core-utils' |
5 | import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' | 6 | import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' |
6 | import { | 7 | import { |
@@ -129,6 +130,8 @@ describe('Permanent live', function () { | |||
129 | 130 | ||
130 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) | 131 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
131 | } | 132 | } |
133 | |||
134 | await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID }) | ||
132 | }) | 135 | }) |
133 | 136 | ||
134 | it('Should have set this live to waiting for live state', async function () { | 137 | it('Should have set this live to waiting for live state', async function () { |
@@ -186,6 +189,15 @@ describe('Permanent live', function () { | |||
186 | } | 189 | } |
187 | }) | 190 | }) |
188 | 191 | ||
192 | it('Should remove the live and have cleaned up the directory', async function () { | ||
193 | this.timeout(60000) | ||
194 | |||
195 | await servers[0].videos.remove({ id: videoUUID }) | ||
196 | await waitJobs(servers) | ||
197 | |||
198 | await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID }) | ||
199 | }) | ||
200 | |||
189 | after(async function () { | 201 | after(async function () { |
190 | await cleanupTests(servers) | 202 | await cleanupTests(servers) |
191 | }) | 203 | }) |
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index 7014292d0..8f17b4566 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts | |||
@@ -186,7 +186,7 @@ describe('Save replay setting', function () { | |||
186 | await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) | 186 | await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) |
187 | 187 | ||
188 | // No resolutions saved since we did not save replay | 188 | // No resolutions saved since we did not save replay |
189 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 189 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
190 | }) | 190 | }) |
191 | 191 | ||
192 | it('Should have appropriate ended session', async function () { | 192 | it('Should have appropriate ended session', async function () { |
@@ -220,7 +220,7 @@ describe('Save replay setting', function () { | |||
220 | 220 | ||
221 | await wait(5000) | 221 | await wait(5000) |
222 | await waitJobs(servers) | 222 | await waitJobs(servers) |
223 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 223 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
224 | }) | 224 | }) |
225 | 225 | ||
226 | it('Should have blacklisted session error', async function () { | 226 | it('Should have blacklisted session error', async function () { |
@@ -238,7 +238,7 @@ describe('Save replay setting', function () { | |||
238 | await publishLiveAndDelete({ permanent: false, replay: false }) | 238 | await publishLiveAndDelete({ permanent: false, replay: false }) |
239 | 239 | ||
240 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) | 240 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) |
241 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 241 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
242 | }) | 242 | }) |
243 | }) | 243 | }) |
244 | 244 | ||
@@ -317,7 +317,7 @@ describe('Save replay setting', function () { | |||
317 | }) | 317 | }) |
318 | 318 | ||
319 | it('Should have cleaned up the live files', async function () { | 319 | it('Should have cleaned up the live files', async function () { |
320 | await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) | 320 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] }) |
321 | }) | 321 | }) |
322 | 322 | ||
323 | it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { | 323 | it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { |
@@ -332,7 +332,7 @@ describe('Save replay setting', function () { | |||
332 | 332 | ||
333 | await wait(5000) | 333 | await wait(5000) |
334 | await waitJobs(servers) | 334 | await waitJobs(servers) |
335 | await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) | 335 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] }) |
336 | }) | 336 | }) |
337 | 337 | ||
338 | it('Should correctly terminate the stream on delete and delete the video', async function () { | 338 | it('Should correctly terminate the stream on delete and delete the video', async function () { |
@@ -341,7 +341,7 @@ describe('Save replay setting', function () { | |||
341 | await publishLiveAndDelete({ permanent: false, replay: true }) | 341 | await publishLiveAndDelete({ permanent: false, replay: true }) |
342 | 342 | ||
343 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) | 343 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) |
344 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 344 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
345 | }) | 345 | }) |
346 | }) | 346 | }) |
347 | 347 | ||
@@ -413,7 +413,7 @@ describe('Save replay setting', function () { | |||
413 | }) | 413 | }) |
414 | 414 | ||
415 | it('Should have cleaned up the live files', async function () { | 415 | it('Should have cleaned up the live files', async function () { |
416 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 416 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
417 | }) | 417 | }) |
418 | 418 | ||
419 | it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { | 419 | it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { |
@@ -432,7 +432,7 @@ describe('Save replay setting', function () { | |||
432 | await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 432 | await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
433 | } | 433 | } |
434 | 434 | ||
435 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 435 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
436 | }) | 436 | }) |
437 | 437 | ||
438 | it('Should correctly terminate the stream on delete and not save the video', async function () { | 438 | it('Should correctly terminate the stream on delete and not save the video', async function () { |
@@ -444,7 +444,7 @@ describe('Save replay setting', function () { | |||
444 | expect(replay).to.not.exist | 444 | expect(replay).to.not.exist |
445 | 445 | ||
446 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) | 446 | await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) |
447 | await checkLiveCleanup(servers[0], liveVideoUUID, []) | 447 | await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) |
448 | }) | 448 | }) |
449 | }) | 449 | }) |
450 | 450 | ||
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index c436f0f01..003cc934f 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts | |||
@@ -3,7 +3,7 @@ | |||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { basename, join } from 'path' | 4 | import { basename, join } from 'path' |
5 | import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg' | 5 | import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg' |
6 | import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared' | 6 | import { testImage, testVideoResolutions } from '@server/tests/shared' |
7 | import { getAllFiles, wait } from '@shared/core-utils' | 7 | import { getAllFiles, wait } from '@shared/core-utils' |
8 | import { | 8 | import { |
9 | HttpStatusCode, | 9 | HttpStatusCode, |
@@ -21,6 +21,7 @@ import { | |||
21 | doubleFollow, | 21 | doubleFollow, |
22 | killallServers, | 22 | killallServers, |
23 | LiveCommand, | 23 | LiveCommand, |
24 | makeGetRequest, | ||
24 | makeRawRequest, | 25 | makeRawRequest, |
25 | PeerTubeServer, | 26 | PeerTubeServer, |
26 | sendRTMPStream, | 27 | sendRTMPStream, |
@@ -157,8 +158,8 @@ describe('Test live', function () { | |||
157 | expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) | 158 | expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) |
158 | expect(video.nsfw).to.be.true | 159 | expect(video.nsfw).to.be.true |
159 | 160 | ||
160 | await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200) | 161 | await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
161 | await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200) | 162 | await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) |
162 | } | 163 | } |
163 | }) | 164 | }) |
164 | 165 | ||
@@ -372,46 +373,6 @@ describe('Test live', function () { | |||
372 | return uuid | 373 | return uuid |
373 | } | 374 | } |
374 | 375 | ||
375 | async function testVideoResolutions (liveVideoId: string, resolutions: number[]) { | ||
376 | for (const server of servers) { | ||
377 | const { data } = await server.videos.list() | ||
378 | expect(data.find(v => v.uuid === liveVideoId)).to.exist | ||
379 | |||
380 | const video = await server.videos.get({ id: liveVideoId }) | ||
381 | |||
382 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
383 | |||
384 | const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) | ||
385 | expect(hlsPlaylist).to.exist | ||
386 | |||
387 | // Only finite files are displayed | ||
388 | expect(hlsPlaylist.files).to.have.lengthOf(0) | ||
389 | |||
390 | await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) | ||
391 | |||
392 | for (let i = 0; i < resolutions.length; i++) { | ||
393 | const segmentNum = 3 | ||
394 | const segmentName = `${i}-00000${segmentNum}.ts` | ||
395 | await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum }) | ||
396 | |||
397 | const subPlaylist = await servers[0].streamingPlaylists.get({ | ||
398 | url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8` | ||
399 | }) | ||
400 | |||
401 | expect(subPlaylist).to.contain(segmentName) | ||
402 | |||
403 | const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls' | ||
404 | await checkLiveSegmentHash({ | ||
405 | server, | ||
406 | baseUrlSegment: baseUrlAndPath, | ||
407 | videoUUID: video.uuid, | ||
408 | segmentName, | ||
409 | hlsPlaylist | ||
410 | }) | ||
411 | } | ||
412 | } | ||
413 | } | ||
414 | |||
415 | function updateConf (resolutions: number[]) { | 376 | function updateConf (resolutions: number[]) { |
416 | return servers[0].config.updateCustomSubConfig({ | 377 | return servers[0].config.updateCustomSubConfig({ |
417 | newConfig: { | 378 | newConfig: { |
@@ -449,7 +410,14 @@ describe('Test live', function () { | |||
449 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 410 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
450 | await waitJobs(servers) | 411 | await waitJobs(servers) |
451 | 412 | ||
452 | await testVideoResolutions(liveVideoId, [ 720 ]) | 413 | await testVideoResolutions({ |
414 | originServer: servers[0], | ||
415 | servers, | ||
416 | liveVideoId, | ||
417 | resolutions: [ 720 ], | ||
418 | objectStorage: false, | ||
419 | transcoded: true | ||
420 | }) | ||
453 | 421 | ||
454 | await stopFfmpeg(ffmpegCommand) | 422 | await stopFfmpeg(ffmpegCommand) |
455 | }) | 423 | }) |
@@ -477,7 +445,14 @@ describe('Test live', function () { | |||
477 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 445 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
478 | await waitJobs(servers) | 446 | await waitJobs(servers) |
479 | 447 | ||
480 | await testVideoResolutions(liveVideoId, resolutions.concat([ 720 ])) | 448 | await testVideoResolutions({ |
449 | originServer: servers[0], | ||
450 | servers, | ||
451 | liveVideoId, | ||
452 | resolutions: resolutions.concat([ 720 ]), | ||
453 | objectStorage: false, | ||
454 | transcoded: true | ||
455 | }) | ||
481 | 456 | ||
482 | await stopFfmpeg(ffmpegCommand) | 457 | await stopFfmpeg(ffmpegCommand) |
483 | }) | 458 | }) |
@@ -522,7 +497,14 @@ describe('Test live', function () { | |||
522 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 497 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
523 | await waitJobs(servers) | 498 | await waitJobs(servers) |
524 | 499 | ||
525 | await testVideoResolutions(liveVideoId, resolutions) | 500 | await testVideoResolutions({ |
501 | originServer: servers[0], | ||
502 | servers, | ||
503 | liveVideoId, | ||
504 | resolutions, | ||
505 | objectStorage: false, | ||
506 | transcoded: true | ||
507 | }) | ||
526 | 508 | ||
527 | await stopFfmpeg(ffmpegCommand) | 509 | await stopFfmpeg(ffmpegCommand) |
528 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) | 510 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) |
@@ -538,7 +520,7 @@ describe('Test live', function () { | |||
538 | } | 520 | } |
539 | 521 | ||
540 | const minBitrateLimits = { | 522 | const minBitrateLimits = { |
541 | 720: 5500 * 1000, | 523 | 720: 4800 * 1000, |
542 | 360: 1000 * 1000, | 524 | 360: 1000 * 1000, |
543 | 240: 550 * 1000 | 525 | 240: 550 * 1000 |
544 | } | 526 | } |
@@ -551,8 +533,8 @@ describe('Test live', function () { | |||
551 | expect(video.files).to.have.lengthOf(0) | 533 | expect(video.files).to.have.lengthOf(0) |
552 | 534 | ||
553 | const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) | 535 | const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) |
554 | await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) | 536 | await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
555 | await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) | 537 | await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
556 | 538 | ||
557 | // We should have generated random filenames | 539 | // We should have generated random filenames |
558 | expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') | 540 | expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') |
@@ -569,7 +551,7 @@ describe('Test live', function () { | |||
569 | if (resolution >= 720) { | 551 | if (resolution >= 720) { |
570 | expect(file.fps).to.be.approximately(60, 10) | 552 | expect(file.fps).to.be.approximately(60, 10) |
571 | } else { | 553 | } else { |
572 | expect(file.fps).to.be.approximately(30, 2) | 554 | expect(file.fps).to.be.approximately(30, 3) |
573 | } | 555 | } |
574 | 556 | ||
575 | const filename = basename(file.fileUrl) | 557 | const filename = basename(file.fileUrl) |
@@ -583,8 +565,8 @@ describe('Test live', function () { | |||
583 | expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height]) | 565 | expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height]) |
584 | expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height]) | 566 | expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height]) |
585 | 567 | ||
586 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) | 568 | await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) |
587 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 569 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
588 | } | 570 | } |
589 | } | 571 | } |
590 | }) | 572 | }) |
@@ -611,7 +593,14 @@ describe('Test live', function () { | |||
611 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 593 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
612 | await waitJobs(servers) | 594 | await waitJobs(servers) |
613 | 595 | ||
614 | await testVideoResolutions(liveVideoId, resolutions) | 596 | await testVideoResolutions({ |
597 | originServer: servers[0], | ||
598 | servers, | ||
599 | liveVideoId, | ||
600 | resolutions, | ||
601 | objectStorage: false, | ||
602 | transcoded: true | ||
603 | }) | ||
615 | 604 | ||
616 | await stopFfmpeg(ffmpegCommand) | 605 | await stopFfmpeg(ffmpegCommand) |
617 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) | 606 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) |
@@ -640,7 +629,14 @@ describe('Test live', function () { | |||
640 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | 629 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) |
641 | await waitJobs(servers) | 630 | await waitJobs(servers) |
642 | 631 | ||
643 | await testVideoResolutions(liveVideoId, [ 720 ]) | 632 | await testVideoResolutions({ |
633 | originServer: servers[0], | ||
634 | servers, | ||
635 | liveVideoId, | ||
636 | resolutions: [ 720 ], | ||
637 | objectStorage: false, | ||
638 | transcoded: true | ||
639 | }) | ||
644 | 640 | ||
645 | await stopFfmpeg(ffmpegCommand) | 641 | await stopFfmpeg(ffmpegCommand) |
646 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) | 642 | await commands[0].waitUntilEnded({ videoId: liveVideoId }) |
@@ -700,9 +696,15 @@ describe('Test live', function () { | |||
700 | commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) | 696 | commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) |
701 | ]) | 697 | ]) |
702 | 698 | ||
703 | await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoId, playlistNumber: 0, segment: 2 }) | 699 | for (const videoUUID of [ liveVideoId, liveVideoReplayId, permanentLiveVideoReplayId ]) { |
704 | await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoReplayId, playlistNumber: 0, segment: 2 }) | 700 | await commands[0].waitUntilSegmentGeneration({ |
705 | await commands[0].waitUntilSegmentGeneration({ videoUUID: permanentLiveVideoReplayId, playlistNumber: 0, segment: 2 }) | 701 | server: servers[0], |
702 | videoUUID, | ||
703 | playlistNumber: 0, | ||
704 | segment: 2, | ||
705 | objectStorage: false | ||
706 | }) | ||
707 | } | ||
706 | 708 | ||
707 | { | 709 | { |
708 | const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId }) | 710 | const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId }) |
diff --git a/server/tests/api/notifications/admin-notifications.ts b/server/tests/api/notifications/admin-notifications.ts index b3bb4888e..07c981a37 100644 --- a/server/tests/api/notifications/admin-notifications.ts +++ b/server/tests/api/notifications/admin-notifications.ts | |||
@@ -37,7 +37,7 @@ describe('Test admin notifications', function () { | |||
37 | plugins: { | 37 | plugins: { |
38 | index: { | 38 | index: { |
39 | enabled: true, | 39 | enabled: true, |
40 | check_latest_versions_interval: '5 seconds' | 40 | check_latest_versions_interval: '3 seconds' |
41 | } | 41 | } |
42 | } | 42 | } |
43 | } | 43 | } |
@@ -62,7 +62,7 @@ describe('Test admin notifications', function () { | |||
62 | 62 | ||
63 | describe('Latest PeerTube version notification', function () { | 63 | describe('Latest PeerTube version notification', function () { |
64 | 64 | ||
65 | it('Should not send a notification to admins if there is not a new version', async function () { | 65 | it('Should not send a notification to admins if there is no new version', async function () { |
66 | this.timeout(30000) | 66 | this.timeout(30000) |
67 | 67 | ||
68 | joinPeerTubeServer.setLatestVersion('1.4.2') | 68 | joinPeerTubeServer.setLatestVersion('1.4.2') |
@@ -71,7 +71,7 @@ describe('Test admin notifications', function () { | |||
71 | await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' }) | 71 | await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' }) |
72 | }) | 72 | }) |
73 | 73 | ||
74 | it('Should send a notification to admins on new plugin version', async function () { | 74 | it('Should send a notification to admins on new version', async function () { |
75 | this.timeout(30000) | 75 | this.timeout(30000) |
76 | 76 | ||
77 | joinPeerTubeServer.setLatestVersion('15.4.2') | 77 | joinPeerTubeServer.setLatestVersion('15.4.2') |
diff --git a/server/tests/api/notifications/moderation-notifications.ts b/server/tests/api/notifications/moderation-notifications.ts index d8a7d576e..5a632fb22 100644 --- a/server/tests/api/notifications/moderation-notifications.ts +++ b/server/tests/api/notifications/moderation-notifications.ts | |||
@@ -382,7 +382,7 @@ describe('Test moderation notifications', function () { | |||
382 | }) | 382 | }) |
383 | 383 | ||
384 | it('Should send a notification only to admin when there is a new instance follower', async function () { | 384 | it('Should send a notification only to admin when there is a new instance follower', async function () { |
385 | this.timeout(20000) | 385 | this.timeout(60000) |
386 | 386 | ||
387 | await servers[2].follows.follow({ hosts: [ servers[0].url ] }) | 387 | await servers[2].follows.follow({ hosts: [ servers[0].url ] }) |
388 | 388 | ||
@@ -545,7 +545,7 @@ describe('Test moderation notifications', function () { | |||
545 | }) | 545 | }) |
546 | 546 | ||
547 | it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { | 547 | it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { |
548 | this.timeout(40000) | 548 | this.timeout(120000) |
549 | 549 | ||
550 | const updateAt = new Date(new Date().getTime() + 1000000) | 550 | const updateAt = new Date(new Date().getTime() + 1000000) |
551 | 551 | ||
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts index f319d6ef5..1f4489fa3 100644 --- a/server/tests/api/object-storage/index.ts +++ b/server/tests/api/object-storage/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | export * from './live' | 1 | export * from './live' |
2 | export * from './video-imports' | 2 | export * from './video-imports' |
3 | export * from './video-static-file-privacy' | ||
3 | export * from './videos' | 4 | export * from './videos' |
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts index 0958ffe0f..ad2b554b7 100644 --- a/server/tests/api/object-storage/live.ts +++ b/server/tests/api/object-storage/live.ts | |||
@@ -1,9 +1,9 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { expectStartWith } from '@server/tests/shared' | 4 | import { expectStartWith, testVideoResolutions } from '@server/tests/shared' |
5 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
6 | import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' | 6 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' |
7 | import { | 7 | import { |
8 | createMultipleServers, | 8 | createMultipleServers, |
9 | doubleFollow, | 9 | doubleFollow, |
@@ -35,54 +35,56 @@ async function createLive (server: PeerTubeServer, permanent: boolean) { | |||
35 | return uuid | 35 | return uuid |
36 | } | 36 | } |
37 | 37 | ||
38 | async function checkFiles (files: VideoFile[]) { | 38 | async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) { |
39 | for (const file of files) { | 39 | for (const server of servers) { |
40 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | 40 | const video = await server.videos.get({ id: videoUUID }) |
41 | 41 | ||
42 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 42 | expect(video.files).to.have.lengthOf(0) |
43 | } | 43 | expect(video.streamingPlaylists).to.have.lengthOf(1) |
44 | } | ||
45 | 44 | ||
46 | async function getFiles (server: PeerTubeServer, videoUUID: string) { | 45 | const files = video.streamingPlaylists[0].files |
47 | const video = await server.videos.get({ id: videoUUID }) | 46 | expect(files).to.have.lengthOf(numberOfFiles) |
48 | 47 | ||
49 | expect(video.files).to.have.lengthOf(0) | 48 | for (const file of files) { |
50 | expect(video.streamingPlaylists).to.have.lengthOf(1) | 49 | expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
51 | 50 | ||
52 | return video.streamingPlaylists[0].files | 51 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
52 | } | ||
53 | } | ||
53 | } | 54 | } |
54 | 55 | ||
55 | async function streamAndEnd (servers: PeerTubeServer[], liveUUID: string) { | 56 | async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) { |
56 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveUUID }) | 57 | const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`) |
57 | await waitUntilLivePublishedOnAllServers(servers, liveUUID) | ||
58 | |||
59 | const videoLiveDetails = await servers[0].videos.get({ id: liveUUID }) | ||
60 | const liveDetails = await servers[0].live.get({ videoId: liveUUID }) | ||
61 | 58 | ||
62 | await stopFfmpeg(ffmpegCommand) | 59 | for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) { |
63 | 60 | await server.live.getPlaylistFile({ | |
64 | if (liveDetails.permanentLive) { | 61 | videoUUID, |
65 | await waitUntilLiveWaitingOnAllServers(servers, liveUUID) | 62 | playlistName, |
66 | } else { | 63 | expectedStatus: HttpStatusCode.NOT_FOUND_404, |
67 | await waitUntilLiveReplacedByReplayOnAllServers(servers, liveUUID) | 64 | objectStorage: true |
65 | }) | ||
68 | } | 66 | } |
69 | 67 | ||
70 | await waitJobs(servers) | 68 | await server.live.getSegmentFile({ |
71 | 69 | videoUUID, | |
72 | return { videoLiveDetails, liveDetails } | 70 | playlistNumber: 0, |
71 | segment: 0, | ||
72 | objectStorage: true, | ||
73 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
74 | }) | ||
73 | } | 75 | } |
74 | 76 | ||
75 | describe('Object storage for lives', function () { | 77 | describe('Object storage for lives', function () { |
76 | if (areObjectStorageTestsDisabled()) return | 78 | if (areMockObjectStorageTestsDisabled()) return |
77 | 79 | ||
78 | let servers: PeerTubeServer[] | 80 | let servers: PeerTubeServer[] |
79 | 81 | ||
80 | before(async function () { | 82 | before(async function () { |
81 | this.timeout(120000) | 83 | this.timeout(120000) |
82 | 84 | ||
83 | await ObjectStorageCommand.prepareDefaultBuckets() | 85 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
84 | 86 | ||
85 | servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) | 87 | servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig()) |
86 | 88 | ||
87 | await setAccessTokensToServers(servers) | 89 | await setAccessTokensToServers(servers) |
88 | await setDefaultVideoChannel(servers) | 90 | await setDefaultVideoChannel(servers) |
@@ -100,57 +102,124 @@ describe('Object storage for lives', function () { | |||
100 | videoUUID = await createLive(servers[0], false) | 102 | videoUUID = await createLive(servers[0], false) |
101 | }) | 103 | }) |
102 | 104 | ||
103 | it('Should create a live and save the replay on object storage', async function () { | 105 | it('Should create a live and publish it on object storage', async function () { |
106 | this.timeout(220000) | ||
107 | |||
108 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
109 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
110 | |||
111 | await testVideoResolutions({ | ||
112 | originServer: servers[0], | ||
113 | servers, | ||
114 | liveVideoId: videoUUID, | ||
115 | resolutions: [ 720 ], | ||
116 | transcoded: false, | ||
117 | objectStorage: true | ||
118 | }) | ||
119 | |||
120 | await stopFfmpeg(ffmpegCommand) | ||
121 | }) | ||
122 | |||
123 | it('Should have saved the replay on object storage', async function () { | ||
104 | this.timeout(220000) | 124 | this.timeout(220000) |
105 | 125 | ||
106 | await streamAndEnd(servers, videoUUID) | 126 | await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID) |
127 | await waitJobs(servers) | ||
107 | 128 | ||
108 | for (const server of servers) { | 129 | await checkFilesExist(servers, videoUUID, 1) |
109 | const files = await getFiles(server, videoUUID) | 130 | }) |
110 | expect(files).to.have.lengthOf(1) | ||
111 | 131 | ||
112 | await checkFiles(files) | 132 | it('Should have cleaned up live files from object storage', async function () { |
113 | } | 133 | await checkFilesCleanup(servers[0], videoUUID, [ 720 ]) |
114 | }) | 134 | }) |
115 | }) | 135 | }) |
116 | 136 | ||
117 | describe('With live transcoding', async function () { | 137 | describe('With live transcoding', async function () { |
118 | let videoUUIDPermanent: string | 138 | const resolutions = [ 720, 480, 360, 240, 144 ] |
119 | let videoUUIDNonPermanent: string | ||
120 | 139 | ||
121 | before(async function () { | 140 | before(async function () { |
122 | await servers[0].config.enableLive({ transcoding: true }) | 141 | await servers[0].config.enableLive({ transcoding: true }) |
123 | |||
124 | videoUUIDPermanent = await createLive(servers[0], true) | ||
125 | videoUUIDNonPermanent = await createLive(servers[0], false) | ||
126 | }) | 142 | }) |
127 | 143 | ||
128 | it('Should create a live and save the replay on object storage', async function () { | 144 | describe('Normal replay', function () { |
129 | this.timeout(240000) | 145 | let videoUUIDNonPermanent: string |
146 | |||
147 | before(async function () { | ||
148 | videoUUIDNonPermanent = await createLive(servers[0], false) | ||
149 | }) | ||
150 | |||
151 | it('Should create a live and publish it on object storage', async function () { | ||
152 | this.timeout(240000) | ||
153 | |||
154 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent }) | ||
155 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent) | ||
156 | |||
157 | await testVideoResolutions({ | ||
158 | originServer: servers[0], | ||
159 | servers, | ||
160 | liveVideoId: videoUUIDNonPermanent, | ||
161 | resolutions, | ||
162 | transcoded: true, | ||
163 | objectStorage: true | ||
164 | }) | ||
165 | |||
166 | await stopFfmpeg(ffmpegCommand) | ||
167 | }) | ||
130 | 168 | ||
131 | await streamAndEnd(servers, videoUUIDNonPermanent) | 169 | it('Should have saved the replay on object storage', async function () { |
170 | this.timeout(220000) | ||
132 | 171 | ||
133 | for (const server of servers) { | 172 | await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent) |
134 | const files = await getFiles(server, videoUUIDNonPermanent) | 173 | await waitJobs(servers) |
135 | expect(files).to.have.lengthOf(5) | ||
136 | 174 | ||
137 | await checkFiles(files) | 175 | await checkFilesExist(servers, videoUUIDNonPermanent, 5) |
138 | } | 176 | }) |
177 | |||
178 | it('Should have cleaned up live files from object storage', async function () { | ||
179 | await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions) | ||
180 | }) | ||
139 | }) | 181 | }) |
140 | 182 | ||
141 | it('Should create a live and save the replay of permanent live on object storage', async function () { | 183 | describe('Permanent replay', function () { |
142 | this.timeout(240000) | 184 | let videoUUIDPermanent: string |
185 | |||
186 | before(async function () { | ||
187 | videoUUIDPermanent = await createLive(servers[0], true) | ||
188 | }) | ||
189 | |||
190 | it('Should create a live and publish it on object storage', async function () { | ||
191 | this.timeout(240000) | ||
192 | |||
193 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) | ||
194 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) | ||
195 | |||
196 | await testVideoResolutions({ | ||
197 | originServer: servers[0], | ||
198 | servers, | ||
199 | liveVideoId: videoUUIDPermanent, | ||
200 | resolutions, | ||
201 | transcoded: true, | ||
202 | objectStorage: true | ||
203 | }) | ||
204 | |||
205 | await stopFfmpeg(ffmpegCommand) | ||
206 | }) | ||
207 | |||
208 | it('Should have saved the replay on object storage', async function () { | ||
209 | this.timeout(220000) | ||
143 | 210 | ||
144 | const { videoLiveDetails } = await streamAndEnd(servers, videoUUIDPermanent) | 211 | await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent) |
212 | await waitJobs(servers) | ||
145 | 213 | ||
146 | const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) | 214 | const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent }) |
215 | const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) | ||
147 | 216 | ||
148 | for (const server of servers) { | 217 | await checkFilesExist(servers, replay.uuid, 5) |
149 | const files = await getFiles(server, replay.uuid) | 218 | }) |
150 | expect(files).to.have.lengthOf(5) | ||
151 | 219 | ||
152 | await checkFiles(files) | 220 | it('Should have cleaned up live files from object storage', async function () { |
153 | } | 221 | await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions) |
222 | }) | ||
154 | }) | 223 | }) |
155 | }) | 224 | }) |
156 | 225 | ||
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts index f688c7018..11c866411 100644 --- a/server/tests/api/object-storage/video-imports.ts +++ b/server/tests/api/object-storage/video-imports.ts | |||
@@ -2,7 +2,7 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' | 4 | import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' |
5 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
6 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | 6 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' |
7 | import { | 7 | import { |
8 | createSingleServer, | 8 | createSingleServer, |
@@ -29,16 +29,16 @@ async function importVideo (server: PeerTubeServer) { | |||
29 | } | 29 | } |
30 | 30 | ||
31 | describe('Object storage for video import', function () { | 31 | describe('Object storage for video import', function () { |
32 | if (areObjectStorageTestsDisabled()) return | 32 | if (areMockObjectStorageTestsDisabled()) return |
33 | 33 | ||
34 | let server: PeerTubeServer | 34 | let server: PeerTubeServer |
35 | 35 | ||
36 | before(async function () { | 36 | before(async function () { |
37 | this.timeout(120000) | 37 | this.timeout(120000) |
38 | 38 | ||
39 | await ObjectStorageCommand.prepareDefaultBuckets() | 39 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
40 | 40 | ||
41 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) | 41 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig()) |
42 | 42 | ||
43 | await setAccessTokensToServers([ server ]) | 43 | await setAccessTokensToServers([ server ]) |
44 | await setDefaultVideoChannel([ server ]) | 44 | await setDefaultVideoChannel([ server ]) |
@@ -64,9 +64,9 @@ describe('Object storage for video import', function () { | |||
64 | expect(video.streamingPlaylists).to.have.lengthOf(0) | 64 | expect(video.streamingPlaylists).to.have.lengthOf(0) |
65 | 65 | ||
66 | const fileUrl = video.files[0].fileUrl | 66 | const fileUrl = video.files[0].fileUrl |
67 | expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 67 | expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
68 | 68 | ||
69 | await makeRawRequest(fileUrl, HttpStatusCode.OK_200) | 69 | await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
70 | }) | 70 | }) |
71 | }) | 71 | }) |
72 | 72 | ||
@@ -89,15 +89,15 @@ describe('Object storage for video import', function () { | |||
89 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) | 89 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) |
90 | 90 | ||
91 | for (const file of video.files) { | 91 | for (const file of video.files) { |
92 | expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 92 | expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
93 | 93 | ||
94 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 94 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
95 | } | 95 | } |
96 | 96 | ||
97 | for (const file of video.streamingPlaylists[0].files) { | 97 | for (const file of video.streamingPlaylists[0].files) { |
98 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | 98 | expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
99 | 99 | ||
100 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 100 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
101 | } | 101 | } |
102 | }) | 102 | }) |
103 | }) | 103 | }) |
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts new file mode 100644 index 000000000..62edd10ba --- /dev/null +++ b/server/tests/api/object-storage/video-static-file-privacy.ts | |||
@@ -0,0 +1,402 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { basename } from 'path' | ||
5 | import { expectStartWith } from '@server/tests/shared' | ||
6 | import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' | ||
7 | import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' | ||
8 | import { | ||
9 | cleanupTests, | ||
10 | createSingleServer, | ||
11 | findExternalSavedVideo, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
14 | PeerTubeServer, | ||
15 | sendRTMPStream, | ||
16 | setAccessTokensToServers, | ||
17 | setDefaultVideoChannel, | ||
18 | stopFfmpeg, | ||
19 | waitJobs | ||
20 | } from '@shared/server-commands' | ||
21 | |||
22 | function extractFilenameFromUrl (url: string) { | ||
23 | const parts = basename(url).split(':') | ||
24 | |||
25 | return parts[parts.length - 1] | ||
26 | } | ||
27 | |||
28 | describe('Object storage for video static file privacy', function () { | ||
29 | // We need real world object storage to check ACL | ||
30 | if (areScalewayObjectStorageTestsDisabled()) return | ||
31 | |||
32 | let server: PeerTubeServer | ||
33 | let userToken: string | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | async function checkPrivateVODFiles (uuid: string) { | ||
38 | const video = await server.videos.getWithToken({ id: uuid }) | ||
39 | |||
40 | for (const file of video.files) { | ||
41 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/') | ||
42 | |||
43 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
44 | } | ||
45 | |||
46 | for (const file of getAllFiles(video)) { | ||
47 | const internalFileUrl = await server.sql.getInternalFileUrl(file.id) | ||
48 | expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
49 | await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
50 | } | ||
51 | |||
52 | const hls = getHLS(video) | ||
53 | |||
54 | if (hls) { | ||
55 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
56 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
57 | } | ||
58 | |||
59 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
60 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
61 | |||
62 | for (const file of hls.files) { | ||
63 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
64 | |||
65 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
66 | } | ||
67 | } | ||
68 | } | ||
69 | |||
70 | async function checkPublicVODFiles (uuid: string) { | ||
71 | const video = await server.videos.getWithToken({ id: uuid }) | ||
72 | |||
73 | for (const file of getAllFiles(video)) { | ||
74 | expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
75 | |||
76 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
77 | } | ||
78 | |||
79 | const hls = getHLS(video) | ||
80 | |||
81 | if (hls) { | ||
82 | expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
83 | expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) | ||
84 | |||
85 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
86 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | ||
87 | } | ||
88 | } | ||
89 | |||
90 | // --------------------------------------------------------------------------- | ||
91 | |||
92 | before(async function () { | ||
93 | this.timeout(120000) | ||
94 | |||
95 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 })) | ||
96 | await setAccessTokensToServers([ server ]) | ||
97 | await setDefaultVideoChannel([ server ]) | ||
98 | |||
99 | await server.config.enableMinimumTranscoding() | ||
100 | |||
101 | userToken = await server.users.generateUserAndToken('user1') | ||
102 | }) | ||
103 | |||
104 | describe('VOD', function () { | ||
105 | let privateVideoUUID: string | ||
106 | let publicVideoUUID: string | ||
107 | let userPrivateVideoUUID: string | ||
108 | |||
109 | // --------------------------------------------------------------------------- | ||
110 | |||
111 | async function getSampleFileUrls (videoId: string) { | ||
112 | const video = await server.videos.getWithToken({ id: videoId }) | ||
113 | |||
114 | return { | ||
115 | webTorrentFile: video.files[0].fileUrl, | ||
116 | hlsFile: getHLS(video).files[0].fileUrl | ||
117 | } | ||
118 | } | ||
119 | |||
120 | // --------------------------------------------------------------------------- | ||
121 | |||
122 | it('Should upload a private video and have appropriate object storage ACL', async function () { | ||
123 | this.timeout(60000) | ||
124 | |||
125 | { | ||
126 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
127 | privateVideoUUID = uuid | ||
128 | } | ||
129 | |||
130 | { | ||
131 | const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE }) | ||
132 | userPrivateVideoUUID = uuid | ||
133 | } | ||
134 | |||
135 | await waitJobs([ server ]) | ||
136 | |||
137 | await checkPrivateVODFiles(privateVideoUUID) | ||
138 | }) | ||
139 | |||
140 | it('Should upload a public video and have appropriate object storage ACL', async function () { | ||
141 | this.timeout(60000) | ||
142 | |||
143 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED }) | ||
144 | await waitJobs([ server ]) | ||
145 | |||
146 | publicVideoUUID = uuid | ||
147 | |||
148 | await checkPublicVODFiles(publicVideoUUID) | ||
149 | }) | ||
150 | |||
151 | it('Should not get files without appropriate OAuth token', async function () { | ||
152 | this.timeout(60000) | ||
153 | |||
154 | const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) | ||
155 | |||
156 | await makeRawRequest({ url: webTorrentFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
157 | await makeRawRequest({ url: webTorrentFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
158 | |||
159 | await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
160 | await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
161 | }) | ||
162 | |||
163 | it('Should not get HLS file of another video', async function () { | ||
164 | this.timeout(60000) | ||
165 | |||
166 | const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID }) | ||
167 | const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl) | ||
168 | |||
169 | const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename | ||
170 | const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename | ||
171 | |||
172 | await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | ||
173 | await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
174 | }) | ||
175 | |||
176 | it('Should correctly check OAuth or video file token', async function () { | ||
177 | this.timeout(60000) | ||
178 | |||
179 | const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) | ||
180 | const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) | ||
181 | |||
182 | const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) | ||
183 | |||
184 | for (const url of [ webTorrentFile, hlsFile ]) { | ||
185 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
186 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
187 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
188 | |||
189 | await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
190 | await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
191 | } | ||
192 | }) | ||
193 | |||
194 | it('Should update public video to private', async function () { | ||
195 | this.timeout(60000) | ||
196 | |||
197 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) | ||
198 | |||
199 | await checkPrivateVODFiles(publicVideoUUID) | ||
200 | }) | ||
201 | |||
202 | it('Should update private video to public', async function () { | ||
203 | this.timeout(60000) | ||
204 | |||
205 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) | ||
206 | |||
207 | await checkPublicVODFiles(publicVideoUUID) | ||
208 | }) | ||
209 | }) | ||
210 | |||
211 | describe('Live', function () { | ||
212 | let normalLiveId: string | ||
213 | let normalLive: LiveVideo | ||
214 | |||
215 | let permanentLiveId: string | ||
216 | let permanentLive: LiveVideo | ||
217 | |||
218 | let unrelatedFileToken: string | ||
219 | |||
220 | // --------------------------------------------------------------------------- | ||
221 | |||
222 | async function checkLiveFiles (live: LiveVideo, liveId: string) { | ||
223 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) | ||
224 | await server.live.waitUntilPublished({ videoId: liveId }) | ||
225 | |||
226 | const video = await server.videos.getWithToken({ id: liveId }) | ||
227 | const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) | ||
228 | |||
229 | const hls = video.streamingPlaylists[0] | ||
230 | |||
231 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
232 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
233 | |||
234 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
235 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
236 | |||
237 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
238 | await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
239 | |||
240 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
241 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
242 | await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
243 | } | ||
244 | |||
245 | await stopFfmpeg(ffmpegCommand) | ||
246 | } | ||
247 | |||
248 | async function checkReplay (replay: VideoDetails) { | ||
249 | const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) | ||
250 | |||
251 | const hls = replay.streamingPlaylists[0] | ||
252 | expect(hls.files).to.not.have.lengthOf(0) | ||
253 | |||
254 | for (const file of hls.files) { | ||
255 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
256 | await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
257 | |||
258 | await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
259 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
260 | await makeRawRequest({ | ||
261 | url: file.fileUrl, | ||
262 | query: { videoFileToken: unrelatedFileToken }, | ||
263 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
264 | }) | ||
265 | } | ||
266 | |||
267 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
268 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
269 | |||
270 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
271 | await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
272 | |||
273 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
274 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
275 | await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
276 | } | ||
277 | } | ||
278 | |||
279 | // --------------------------------------------------------------------------- | ||
280 | |||
281 | before(async function () { | ||
282 | await server.config.enableMinimumTranscoding() | ||
283 | |||
284 | const { uuid } = await server.videos.quickUpload({ name: 'another video' }) | ||
285 | unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) | ||
286 | |||
287 | await server.config.enableLive({ | ||
288 | allowReplay: true, | ||
289 | transcoding: true, | ||
290 | resolutions: 'min' | ||
291 | }) | ||
292 | |||
293 | { | ||
294 | const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE }) | ||
295 | normalLiveId = video.uuid | ||
296 | normalLive = live | ||
297 | } | ||
298 | |||
299 | { | ||
300 | const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE }) | ||
301 | permanentLiveId = video.uuid | ||
302 | permanentLive = live | ||
303 | } | ||
304 | }) | ||
305 | |||
306 | it('Should create a private normal live and have a private static path', async function () { | ||
307 | this.timeout(240000) | ||
308 | |||
309 | await checkLiveFiles(normalLive, normalLiveId) | ||
310 | }) | ||
311 | |||
312 | it('Should create a private permanent live and have a private static path', async function () { | ||
313 | this.timeout(240000) | ||
314 | |||
315 | await checkLiveFiles(permanentLive, permanentLiveId) | ||
316 | }) | ||
317 | |||
318 | it('Should have created a replay of the normal live with a private static path', async function () { | ||
319 | this.timeout(240000) | ||
320 | |||
321 | await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) | ||
322 | |||
323 | const replay = await server.videos.getWithToken({ id: normalLiveId }) | ||
324 | await checkReplay(replay) | ||
325 | }) | ||
326 | |||
327 | it('Should have created a replay of the permanent live with a private static path', async function () { | ||
328 | this.timeout(240000) | ||
329 | |||
330 | await server.live.waitUntilWaiting({ videoId: permanentLiveId }) | ||
331 | await waitJobs([ server ]) | ||
332 | |||
333 | const live = await server.videos.getWithToken({ id: permanentLiveId }) | ||
334 | const replayFromList = await findExternalSavedVideo(server, live) | ||
335 | const replay = await server.videos.getWithToken({ id: replayFromList.id }) | ||
336 | |||
337 | await checkReplay(replay) | ||
338 | }) | ||
339 | }) | ||
340 | |||
341 | describe('With private files proxy disabled and public ACL for private files', function () { | ||
342 | let videoUUID: string | ||
343 | |||
344 | before(async function () { | ||
345 | this.timeout(240000) | ||
346 | |||
347 | await server.kill() | ||
348 | |||
349 | const config = ObjectStorageCommand.getDefaultScalewayConfig({ | ||
350 | serverNumber: 1, | ||
351 | enablePrivateProxy: false, | ||
352 | privateACL: 'public-read' | ||
353 | }) | ||
354 | await server.run(config) | ||
355 | |||
356 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
357 | videoUUID = uuid | ||
358 | |||
359 | await waitJobs([ server ]) | ||
360 | }) | ||
361 | |||
362 | it('Should display object storage path for a private video and be able to access them', async function () { | ||
363 | this.timeout(60000) | ||
364 | |||
365 | await checkPublicVODFiles(videoUUID) | ||
366 | }) | ||
367 | |||
368 | it('Should not be able to access object storage proxy', async function () { | ||
369 | const privateVideo = await server.videos.getWithToken({ id: videoUUID }) | ||
370 | const webtorrentFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl) | ||
371 | const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl) | ||
372 | |||
373 | await makeRawRequest({ | ||
374 | url: server.url + '/object-storage-proxy/webseed/private/' + webtorrentFilename, | ||
375 | token: server.accessToken, | ||
376 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
377 | }) | ||
378 | |||
379 | await makeRawRequest({ | ||
380 | url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename, | ||
381 | token: server.accessToken, | ||
382 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
383 | }) | ||
384 | }) | ||
385 | }) | ||
386 | |||
387 | after(async function () { | ||
388 | this.timeout(240000) | ||
389 | |||
390 | const { data } = await server.videos.listAllForAdmin() | ||
391 | |||
392 | for (const v of data) { | ||
393 | await server.videos.remove({ id: v.uuid }) | ||
394 | } | ||
395 | |||
396 | for (const v of data) { | ||
397 | await server.servers.waitUntilLog('Removed files of video ' + v.url) | ||
398 | } | ||
399 | |||
400 | await cleanupTests([ server ]) | ||
401 | }) | ||
402 | }) | ||
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts index 3e65e1093..d1875febb 100644 --- a/server/tests/api/object-storage/videos.ts +++ b/server/tests/api/object-storage/videos.ts | |||
@@ -11,7 +11,7 @@ import { | |||
11 | generateHighBitrateVideo, | 11 | generateHighBitrateVideo, |
12 | MockObjectStorage | 12 | MockObjectStorage |
13 | } from '@server/tests/shared' | 13 | } from '@server/tests/shared' |
14 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 14 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
15 | import { HttpStatusCode, VideoDetails } from '@shared/models' | 15 | import { HttpStatusCode, VideoDetails } from '@shared/models' |
16 | import { | 16 | import { |
17 | cleanupTests, | 17 | cleanupTests, |
@@ -52,18 +52,18 @@ async function checkFiles (options: { | |||
52 | for (const file of video.files) { | 52 | for (const file of video.files) { |
53 | const baseUrl = baseMockUrl | 53 | const baseUrl = baseMockUrl |
54 | ? `${baseMockUrl}/${webtorrentBucket}/` | 54 | ? `${baseMockUrl}/${webtorrentBucket}/` |
55 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | 55 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` |
56 | 56 | ||
57 | const prefix = webtorrentPrefix || '' | 57 | const prefix = webtorrentPrefix || '' |
58 | const start = baseUrl + prefix | 58 | const start = baseUrl + prefix |
59 | 59 | ||
60 | expectStartWith(file.fileUrl, start) | 60 | expectStartWith(file.fileUrl, start) |
61 | 61 | ||
62 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | 62 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) |
63 | const location = res.headers['location'] | 63 | const location = res.headers['location'] |
64 | expectStartWith(location, start) | 64 | expectStartWith(location, start) |
65 | 65 | ||
66 | await makeRawRequest(location, HttpStatusCode.OK_200) | 66 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) |
67 | } | 67 | } |
68 | 68 | ||
69 | const hls = video.streamingPlaylists[0] | 69 | const hls = video.streamingPlaylists[0] |
@@ -73,7 +73,7 @@ async function checkFiles (options: { | |||
73 | 73 | ||
74 | const baseUrl = baseMockUrl | 74 | const baseUrl = baseMockUrl |
75 | ? `${baseMockUrl}/${playlistBucket}/` | 75 | ? `${baseMockUrl}/${playlistBucket}/` |
76 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | 76 | : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` |
77 | 77 | ||
78 | const prefix = playlistPrefix || '' | 78 | const prefix = playlistPrefix || '' |
79 | const start = baseUrl + prefix | 79 | const start = baseUrl + prefix |
@@ -81,19 +81,19 @@ async function checkFiles (options: { | |||
81 | expectStartWith(hls.playlistUrl, start) | 81 | expectStartWith(hls.playlistUrl, start) |
82 | expectStartWith(hls.segmentsSha256Url, start) | 82 | expectStartWith(hls.segmentsSha256Url, start) |
83 | 83 | ||
84 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | 84 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
85 | 85 | ||
86 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | 86 | const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
87 | expect(JSON.stringify(resSha.body)).to.not.throw | 87 | expect(JSON.stringify(resSha.body)).to.not.throw |
88 | 88 | ||
89 | for (const file of hls.files) { | 89 | for (const file of hls.files) { |
90 | expectStartWith(file.fileUrl, start) | 90 | expectStartWith(file.fileUrl, start) |
91 | 91 | ||
92 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | 92 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) |
93 | const location = res.headers['location'] | 93 | const location = res.headers['location'] |
94 | expectStartWith(location, start) | 94 | expectStartWith(location, start) |
95 | 95 | ||
96 | await makeRawRequest(location, HttpStatusCode.OK_200) | 96 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) |
97 | } | 97 | } |
98 | } | 98 | } |
99 | 99 | ||
@@ -104,7 +104,7 @@ async function checkFiles (options: { | |||
104 | expect(torrent.files.length).to.equal(1) | 104 | expect(torrent.files.length).to.equal(1) |
105 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | 105 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') |
106 | 106 | ||
107 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 107 | const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
108 | expect(res.body).to.have.length.above(100) | 108 | expect(res.body).to.have.length.above(100) |
109 | } | 109 | } |
110 | 110 | ||
@@ -141,16 +141,16 @@ function runTestSuite (options: { | |||
141 | const port = await mockObjectStorage.initialize() | 141 | const port = await mockObjectStorage.initialize() |
142 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | 142 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined |
143 | 143 | ||
144 | await ObjectStorageCommand.createBucket(options.playlistBucket) | 144 | await ObjectStorageCommand.createMockBucket(options.playlistBucket) |
145 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | 145 | await ObjectStorageCommand.createMockBucket(options.webtorrentBucket) |
146 | 146 | ||
147 | const config = { | 147 | const config = { |
148 | object_storage: { | 148 | object_storage: { |
149 | enabled: true, | 149 | enabled: true, |
150 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | 150 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), |
151 | region: ObjectStorageCommand.getRegion(), | 151 | region: ObjectStorageCommand.getMockRegion(), |
152 | 152 | ||
153 | credentials: ObjectStorageCommand.getCredentialsConfig(), | 153 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), |
154 | 154 | ||
155 | max_upload_part: options.maxUploadPart || '5MB', | 155 | max_upload_part: options.maxUploadPart || '5MB', |
156 | 156 | ||
@@ -220,7 +220,7 @@ function runTestSuite (options: { | |||
220 | 220 | ||
221 | it('Should fetch correctly all the files', async function () { | 221 | it('Should fetch correctly all the files', async function () { |
222 | for (const url of deletedUrls.concat(keptUrls)) { | 222 | for (const url of deletedUrls.concat(keptUrls)) { |
223 | await makeRawRequest(url, HttpStatusCode.OK_200) | 223 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
224 | } | 224 | } |
225 | }) | 225 | }) |
226 | 226 | ||
@@ -231,13 +231,13 @@ function runTestSuite (options: { | |||
231 | await waitJobs(servers) | 231 | await waitJobs(servers) |
232 | 232 | ||
233 | for (const url of deletedUrls) { | 233 | for (const url of deletedUrls) { |
234 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | 234 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
235 | } | 235 | } |
236 | }) | 236 | }) |
237 | 237 | ||
238 | it('Should have kept other files', async function () { | 238 | it('Should have kept other files', async function () { |
239 | for (const url of keptUrls) { | 239 | for (const url of keptUrls) { |
240 | await makeRawRequest(url, HttpStatusCode.OK_200) | 240 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
241 | } | 241 | } |
242 | }) | 242 | }) |
243 | 243 | ||
@@ -261,7 +261,7 @@ function runTestSuite (options: { | |||
261 | } | 261 | } |
262 | 262 | ||
263 | describe('Object storage for videos', function () { | 263 | describe('Object storage for videos', function () { |
264 | if (areObjectStorageTestsDisabled()) return | 264 | if (areMockObjectStorageTestsDisabled()) return |
265 | 265 | ||
266 | describe('Test config', function () { | 266 | describe('Test config', function () { |
267 | let server: PeerTubeServer | 267 | let server: PeerTubeServer |
@@ -269,17 +269,17 @@ describe('Object storage for videos', function () { | |||
269 | const baseConfig = { | 269 | const baseConfig = { |
270 | object_storage: { | 270 | object_storage: { |
271 | enabled: true, | 271 | enabled: true, |
272 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | 272 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), |
273 | region: ObjectStorageCommand.getRegion(), | 273 | region: ObjectStorageCommand.getMockRegion(), |
274 | 274 | ||
275 | credentials: ObjectStorageCommand.getCredentialsConfig(), | 275 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), |
276 | 276 | ||
277 | streaming_playlists: { | 277 | streaming_playlists: { |
278 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | 278 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET |
279 | }, | 279 | }, |
280 | 280 | ||
281 | videos: { | 281 | videos: { |
282 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | 282 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET |
283 | } | 283 | } |
284 | } | 284 | } |
285 | } | 285 | } |
@@ -310,7 +310,7 @@ describe('Object storage for videos', function () { | |||
310 | it('Should fail with bad credentials', async function () { | 310 | it('Should fail with bad credentials', async function () { |
311 | this.timeout(60000) | 311 | this.timeout(60000) |
312 | 312 | ||
313 | await ObjectStorageCommand.prepareDefaultBuckets() | 313 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
314 | 314 | ||
315 | const config = merge({}, baseConfig, { | 315 | const config = merge({}, baseConfig, { |
316 | object_storage: { | 316 | object_storage: { |
@@ -323,7 +323,7 @@ describe('Object storage for videos', function () { | |||
323 | 323 | ||
324 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | 324 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) |
325 | 325 | ||
326 | await waitJobs([ server ], true) | 326 | await waitJobs([ server ], { skipDelayed: true }) |
327 | const video = await server.videos.get({ id: uuid }) | 327 | const video = await server.videos.get({ id: uuid }) |
328 | 328 | ||
329 | expectStartWith(video.files[0].fileUrl, server.url) | 329 | expectStartWith(video.files[0].fileUrl, server.url) |
@@ -334,7 +334,7 @@ describe('Object storage for videos', function () { | |||
334 | it('Should succeed with credentials from env', async function () { | 334 | it('Should succeed with credentials from env', async function () { |
335 | this.timeout(60000) | 335 | this.timeout(60000) |
336 | 336 | ||
337 | await ObjectStorageCommand.prepareDefaultBuckets() | 337 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
338 | 338 | ||
339 | const config = merge({}, baseConfig, { | 339 | const config = merge({}, baseConfig, { |
340 | object_storage: { | 340 | object_storage: { |
@@ -345,7 +345,7 @@ describe('Object storage for videos', function () { | |||
345 | } | 345 | } |
346 | }) | 346 | }) |
347 | 347 | ||
348 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | 348 | const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() |
349 | 349 | ||
350 | server = await createSingleServer(1, config, { | 350 | server = await createSingleServer(1, config, { |
351 | env: { | 351 | env: { |
@@ -358,10 +358,10 @@ describe('Object storage for videos', function () { | |||
358 | 358 | ||
359 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | 359 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) |
360 | 360 | ||
361 | await waitJobs([ server ], true) | 361 | await waitJobs([ server ], { skipDelayed: true }) |
362 | const video = await server.videos.get({ id: uuid }) | 362 | const video = await server.videos.get({ id: uuid }) |
363 | 363 | ||
364 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 364 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
365 | }) | 365 | }) |
366 | 366 | ||
367 | after(async function () { | 367 | after(async function () { |
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 5abed358f..fb2e6e91c 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -5,7 +5,7 @@ import { readdir } from 'fs-extra' | |||
5 | import magnetUtil from 'magnet-uri' | 5 | import magnetUtil from 'magnet-uri' |
6 | import { basename, join } from 'path' | 6 | import { basename, join } from 'path' |
7 | import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared' | 7 | import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared' |
8 | import { root, wait } from '@shared/core-utils' | 8 | import { wait } from '@shared/core-utils' |
9 | import { | 9 | import { |
10 | HttpStatusCode, | 10 | HttpStatusCode, |
11 | VideoDetails, | 11 | VideoDetails, |
@@ -39,7 +39,7 @@ async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], ser | |||
39 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) | 39 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) |
40 | 40 | ||
41 | for (const url of parsed.urlList) { | 41 | for (const url of parsed.urlList) { |
42 | await makeRawRequest(url, HttpStatusCode.OK_200) | 42 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
43 | } | 43 | } |
44 | } | 44 | } |
45 | 45 | ||
@@ -125,7 +125,7 @@ async function check1WebSeed (videoUUID?: string) { | |||
125 | if (!videoUUID) videoUUID = video1Server2.uuid | 125 | if (!videoUUID) videoUUID = video1Server2.uuid |
126 | 126 | ||
127 | const webseeds = [ | 127 | const webseeds = [ |
128 | `http://localhost:${servers[1].port}/static/webseed/` | 128 | `${servers[1].url}/static/webseed/` |
129 | ] | 129 | ] |
130 | 130 | ||
131 | for (const server of servers) { | 131 | for (const server of servers) { |
@@ -144,8 +144,8 @@ async function check2Webseeds (videoUUID?: string) { | |||
144 | if (!videoUUID) videoUUID = video1Server2.uuid | 144 | if (!videoUUID) videoUUID = video1Server2.uuid |
145 | 145 | ||
146 | const webseeds = [ | 146 | const webseeds = [ |
147 | `http://localhost:${servers[0].port}/static/redundancy/`, | 147 | `${servers[0].url}/static/redundancy/`, |
148 | `http://localhost:${servers[1].port}/static/webseed/` | 148 | `${servers[1].url}/static/webseed/` |
149 | ] | 149 | ] |
150 | 150 | ||
151 | for (const server of servers) { | 151 | for (const server of servers) { |
@@ -159,12 +159,12 @@ async function check2Webseeds (videoUUID?: string) { | |||
159 | const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) | 159 | const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) |
160 | 160 | ||
161 | const directories = [ | 161 | const directories = [ |
162 | 'test' + servers[0].internalServerNumber + '/redundancy', | 162 | servers[0].getDirectoryPath('redundancy'), |
163 | 'test' + servers[1].internalServerNumber + '/videos' | 163 | servers[1].getDirectoryPath('videos') |
164 | ] | 164 | ] |
165 | 165 | ||
166 | for (const directory of directories) { | 166 | for (const directory of directories) { |
167 | const files = await readdir(join(root(), directory)) | 167 | const files = await readdir(directory) |
168 | expect(files).to.have.length.at.least(4) | 168 | expect(files).to.have.length.at.least(4) |
169 | 169 | ||
170 | // Ensure we files exist on disk | 170 | // Ensure we files exist on disk |
@@ -214,12 +214,12 @@ async function check1PlaylistRedundancies (videoUUID?: string) { | |||
214 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) | 214 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) |
215 | 215 | ||
216 | const directories = [ | 216 | const directories = [ |
217 | 'test' + servers[0].internalServerNumber + '/redundancy/hls', | 217 | servers[0].getDirectoryPath('redundancy/hls'), |
218 | 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' | 218 | servers[1].getDirectoryPath('streaming-playlists/hls') |
219 | ] | 219 | ] |
220 | 220 | ||
221 | for (const directory of directories) { | 221 | for (const directory of directories) { |
222 | const files = await readdir(join(root(), directory, videoUUID)) | 222 | const files = await readdir(join(directory, videoUUID)) |
223 | expect(files).to.have.length.at.least(4) | 223 | expect(files).to.have.length.at.least(4) |
224 | 224 | ||
225 | // Ensure we files exist on disk | 225 | // Ensure we files exist on disk |
diff --git a/server/tests/api/server/follow-constraints.ts b/server/tests/api/server/follow-constraints.ts index 5998f58cc..e1ec2b069 100644 --- a/server/tests/api/server/follow-constraints.ts +++ b/server/tests/api/server/follow-constraints.ts | |||
@@ -1,8 +1,15 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' | ||
5 | import { HttpStatusCode, PeerTubeProblemDocument, ServerErrorCode } from '@shared/models' | 4 | import { HttpStatusCode, PeerTubeProblemDocument, ServerErrorCode } from '@shared/models' |
5 | import { | ||
6 | cleanupTests, | ||
7 | createMultipleServers, | ||
8 | doubleFollow, | ||
9 | PeerTubeServer, | ||
10 | setAccessTokensToServers, | ||
11 | waitJobs | ||
12 | } from '@shared/server-commands' | ||
6 | 13 | ||
7 | describe('Test follow constraints', function () { | 14 | describe('Test follow constraints', function () { |
8 | let servers: PeerTubeServer[] = [] | 15 | let servers: PeerTubeServer[] = [] |
@@ -189,6 +196,7 @@ describe('Test follow constraints', function () { | |||
189 | }) | 196 | }) |
190 | 197 | ||
191 | describe('With a logged user', function () { | 198 | describe('With a logged user', function () { |
199 | |||
192 | it('Should get the local video', async function () { | 200 | it('Should get the local video', async function () { |
193 | await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) | 201 | await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) |
194 | }) | 202 | }) |
@@ -229,6 +237,84 @@ describe('Test follow constraints', function () { | |||
229 | }) | 237 | }) |
230 | }) | 238 | }) |
231 | 239 | ||
240 | describe('When following a remote account', function () { | ||
241 | |||
242 | before(async function () { | ||
243 | this.timeout(60000) | ||
244 | |||
245 | await servers[0].follows.follow({ handles: [ 'root@' + servers[1].host ] }) | ||
246 | await waitJobs(servers) | ||
247 | }) | ||
248 | |||
249 | it('Should get the remote video with an unlogged user', async function () { | ||
250 | await servers[0].videos.get({ id: video2UUID }) | ||
251 | }) | ||
252 | |||
253 | it('Should get the remote video with a logged in user', async function () { | ||
254 | await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) | ||
255 | }) | ||
256 | }) | ||
257 | |||
258 | describe('When unfollowing a remote account', function () { | ||
259 | |||
260 | before(async function () { | ||
261 | this.timeout(60000) | ||
262 | |||
263 | await servers[0].follows.unfollow({ target: 'root@' + servers[1].host }) | ||
264 | await waitJobs(servers) | ||
265 | }) | ||
266 | |||
267 | it('Should not get the remote video with an unlogged user', async function () { | ||
268 | const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
269 | |||
270 | const error = body as unknown as PeerTubeProblemDocument | ||
271 | expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) | ||
272 | }) | ||
273 | |||
274 | it('Should get the remote video with a logged in user', async function () { | ||
275 | await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) | ||
276 | }) | ||
277 | }) | ||
278 | |||
279 | describe('When following a remote channel', function () { | ||
280 | |||
281 | before(async function () { | ||
282 | this.timeout(60000) | ||
283 | |||
284 | await servers[0].follows.follow({ handles: [ 'root_channel@' + servers[1].host ] }) | ||
285 | await waitJobs(servers) | ||
286 | }) | ||
287 | |||
288 | it('Should get the remote video with an unlogged user', async function () { | ||
289 | await servers[0].videos.get({ id: video2UUID }) | ||
290 | }) | ||
291 | |||
292 | it('Should get the remote video with a logged in user', async function () { | ||
293 | await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) | ||
294 | }) | ||
295 | }) | ||
296 | |||
297 | describe('When unfollowing a remote channel', function () { | ||
298 | |||
299 | before(async function () { | ||
300 | this.timeout(60000) | ||
301 | |||
302 | await servers[0].follows.unfollow({ target: 'root_channel@' + servers[1].host }) | ||
303 | await waitJobs(servers) | ||
304 | }) | ||
305 | |||
306 | it('Should not get the remote video with an unlogged user', async function () { | ||
307 | const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
308 | |||
309 | const error = body as unknown as PeerTubeProblemDocument | ||
310 | expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) | ||
311 | }) | ||
312 | |||
313 | it('Should get the remote video with a logged in user', async function () { | ||
314 | await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) | ||
315 | }) | ||
316 | }) | ||
317 | |||
232 | after(async function () { | 318 | after(async function () { |
233 | await cleanupTests(servers) | 319 | await cleanupTests(servers) |
234 | }) | 320 | }) |
diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts index 43a27cc32..7a294be82 100644 --- a/server/tests/api/server/open-telemetry.ts +++ b/server/tests/api/server/open-telemetry.ts | |||
@@ -18,7 +18,7 @@ describe('Open Telemetry', function () { | |||
18 | 18 | ||
19 | let hasError = false | 19 | let hasError = false |
20 | try { | 20 | try { |
21 | await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404) | 21 | await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
22 | } catch (err) { | 22 | } catch (err) { |
23 | hasError = err.message.includes('ECONNREFUSED') | 23 | hasError = err.message.includes('ECONNREFUSED') |
24 | } | 24 | } |
@@ -37,7 +37,7 @@ describe('Open Telemetry', function () { | |||
37 | } | 37 | } |
38 | }) | 38 | }) |
39 | 39 | ||
40 | const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) | 40 | const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) |
41 | expect(res.text).to.contain('peertube_job_queue_total{') | 41 | expect(res.text).to.contain('peertube_job_queue_total{') |
42 | }) | 42 | }) |
43 | 43 | ||
@@ -60,7 +60,7 @@ describe('Open Telemetry', function () { | |||
60 | } | 60 | } |
61 | }) | 61 | }) |
62 | 62 | ||
63 | const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200) | 63 | const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) |
64 | expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') | 64 | expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') |
65 | }) | 65 | }) |
66 | 66 | ||
diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts index a4151ebdd..71c444efd 100644 --- a/server/tests/api/server/proxy.ts +++ b/server/tests/api/server/proxy.ts | |||
@@ -2,7 +2,7 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' | 4 | import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' |
5 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
6 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | 6 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' |
7 | import { | 7 | import { |
8 | cleanupTests, | 8 | cleanupTests, |
@@ -120,40 +120,40 @@ describe('Test proxy', function () { | |||
120 | }) | 120 | }) |
121 | 121 | ||
122 | describe('Object storage', function () { | 122 | describe('Object storage', function () { |
123 | if (areObjectStorageTestsDisabled()) return | 123 | if (areMockObjectStorageTestsDisabled()) return |
124 | 124 | ||
125 | before(async function () { | 125 | before(async function () { |
126 | this.timeout(30000) | 126 | this.timeout(30000) |
127 | 127 | ||
128 | await ObjectStorageCommand.prepareDefaultBuckets() | 128 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
129 | }) | 129 | }) |
130 | 130 | ||
131 | it('Should succeed to upload to object storage with the appropriate proxy config', async function () { | 131 | it('Should succeed to upload to object storage with the appropriate proxy config', async function () { |
132 | this.timeout(120000) | 132 | this.timeout(120000) |
133 | 133 | ||
134 | await servers[0].kill() | 134 | await servers[0].kill() |
135 | await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv }) | 135 | await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv }) |
136 | 136 | ||
137 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) | 137 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) |
138 | await waitJobs(servers) | 138 | await waitJobs(servers) |
139 | 139 | ||
140 | const video = await servers[0].videos.get({ id: uuid }) | 140 | const video = await servers[0].videos.get({ id: uuid }) |
141 | 141 | ||
142 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 142 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
143 | }) | 143 | }) |
144 | 144 | ||
145 | it('Should fail to upload to object storage with a wrong proxy config', async function () { | 145 | it('Should fail to upload to object storage with a wrong proxy config', async function () { |
146 | this.timeout(120000) | 146 | this.timeout(120000) |
147 | 147 | ||
148 | await servers[0].kill() | 148 | await servers[0].kill() |
149 | await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv }) | 149 | await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv }) |
150 | 150 | ||
151 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) | 151 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) |
152 | await waitJobs(servers) | 152 | await waitJobs(servers, { skipDelayed: true }) |
153 | 153 | ||
154 | const video = await servers[0].videos.get({ id: uuid }) | 154 | const video = await servers[0].videos.get({ id: uuid }) |
155 | 155 | ||
156 | expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 156 | expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
157 | }) | 157 | }) |
158 | }) | 158 | }) |
159 | 159 | ||
diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts index a50bf7654..85389a949 100644 --- a/server/tests/api/transcoding/create-transcoding.ts +++ b/server/tests/api/transcoding/create-transcoding.ts | |||
@@ -2,7 +2,7 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared' | 4 | import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared' |
5 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
6 | import { HttpStatusCode, VideoDetails } from '@shared/models' | 6 | import { HttpStatusCode, VideoDetails } from '@shared/models' |
7 | import { | 7 | import { |
8 | cleanupTests, | 8 | cleanupTests, |
@@ -19,23 +19,23 @@ import { | |||
19 | 19 | ||
20 | async function checkFilesInObjectStorage (video: VideoDetails) { | 20 | async function checkFilesInObjectStorage (video: VideoDetails) { |
21 | for (const file of video.files) { | 21 | for (const file of video.files) { |
22 | expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 22 | expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
23 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 23 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
24 | } | 24 | } |
25 | 25 | ||
26 | if (video.streamingPlaylists.length === 0) return | 26 | if (video.streamingPlaylists.length === 0) return |
27 | 27 | ||
28 | const hlsPlaylist = video.streamingPlaylists[0] | 28 | const hlsPlaylist = video.streamingPlaylists[0] |
29 | for (const file of hlsPlaylist.files) { | 29 | for (const file of hlsPlaylist.files) { |
30 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | 30 | expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
31 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 31 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
32 | } | 32 | } |
33 | 33 | ||
34 | expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | 34 | expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
35 | await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) | 35 | await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
36 | 36 | ||
37 | expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl()) | 37 | expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
38 | await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) | 38 | await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
39 | } | 39 | } |
40 | 40 | ||
41 | function runTests (objectStorage: boolean) { | 41 | function runTests (objectStorage: boolean) { |
@@ -49,7 +49,7 @@ function runTests (objectStorage: boolean) { | |||
49 | this.timeout(120000) | 49 | this.timeout(120000) |
50 | 50 | ||
51 | const config = objectStorage | 51 | const config = objectStorage |
52 | ? ObjectStorageCommand.getDefaultConfig() | 52 | ? ObjectStorageCommand.getDefaultMockConfig() |
53 | : {} | 53 | : {} |
54 | 54 | ||
55 | // Run server 2 to have transcoding enabled | 55 | // Run server 2 to have transcoding enabled |
@@ -60,7 +60,7 @@ function runTests (objectStorage: boolean) { | |||
60 | 60 | ||
61 | await doubleFollow(servers[0], servers[1]) | 61 | await doubleFollow(servers[0], servers[1]) |
62 | 62 | ||
63 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | 63 | if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets() |
64 | 64 | ||
65 | const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) | 65 | const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) |
66 | videoUUID = shortUUID | 66 | videoUUID = shortUUID |
@@ -234,7 +234,7 @@ function runTests (objectStorage: boolean) { | |||
234 | 234 | ||
235 | it('Should have correctly deleted previous files', async function () { | 235 | it('Should have correctly deleted previous files', async function () { |
236 | for (const fileUrl of shouldBeDeleted) { | 236 | for (const fileUrl of shouldBeDeleted) { |
237 | await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404) | 237 | await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
238 | } | 238 | } |
239 | }) | 239 | }) |
240 | 240 | ||
@@ -256,7 +256,7 @@ describe('Test create transcoding jobs from API', function () { | |||
256 | }) | 256 | }) |
257 | 257 | ||
258 | describe('On object storage', function () { | 258 | describe('On object storage', function () { |
259 | if (areObjectStorageTestsDisabled()) return | 259 | if (areMockObjectStorageTestsDisabled()) return |
260 | 260 | ||
261 | runTests(true) | 261 | runTests(true) |
262 | }) | 262 | }) |
diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts index 252422e5d..84a53c0bd 100644 --- a/server/tests/api/transcoding/hls.ts +++ b/server/tests/api/transcoding/hls.ts | |||
@@ -1,168 +1,48 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { join } from 'path' |
4 | import { basename, join } from 'path' | 4 | import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared' |
5 | import { | 5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
6 | checkDirectoryIsEmpty, | 6 | import { HttpStatusCode } from '@shared/models' |
7 | checkResolutionsInMasterPlaylist, | ||
8 | checkSegmentHash, | ||
9 | checkTmpIsEmpty, | ||
10 | expectStartWith, | ||
11 | hlsInfohashExist | ||
12 | } from '@server/tests/shared' | ||
13 | import { areObjectStorageTestsDisabled, removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' | ||
14 | import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' | ||
15 | import { | 7 | import { |
16 | cleanupTests, | 8 | cleanupTests, |
17 | createMultipleServers, | 9 | createMultipleServers, |
18 | doubleFollow, | 10 | doubleFollow, |
19 | makeRawRequest, | ||
20 | ObjectStorageCommand, | 11 | ObjectStorageCommand, |
21 | PeerTubeServer, | 12 | PeerTubeServer, |
22 | setAccessTokensToServers, | 13 | setAccessTokensToServers, |
23 | waitJobs, | 14 | waitJobs |
24 | webtorrentAdd | ||
25 | } from '@shared/server-commands' | 15 | } from '@shared/server-commands' |
26 | import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' | 16 | import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' |
27 | 17 | ||
28 | async function checkHlsPlaylist (options: { | ||
29 | servers: PeerTubeServer[] | ||
30 | videoUUID: string | ||
31 | hlsOnly: boolean | ||
32 | |||
33 | resolutions?: number[] | ||
34 | objectStorageBaseUrl: string | ||
35 | }) { | ||
36 | const { videoUUID, hlsOnly, objectStorageBaseUrl } = options | ||
37 | |||
38 | const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] | ||
39 | |||
40 | for (const server of options.servers) { | ||
41 | const videoDetails = await server.videos.get({ id: videoUUID }) | ||
42 | const baseUrl = `http://${videoDetails.account.host}` | ||
43 | |||
44 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | ||
45 | |||
46 | const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) | ||
47 | expect(hlsPlaylist).to.not.be.undefined | ||
48 | |||
49 | const hlsFiles = hlsPlaylist.files | ||
50 | expect(hlsFiles).to.have.lengthOf(resolutions.length) | ||
51 | |||
52 | if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0) | ||
53 | else expect(videoDetails.files).to.have.lengthOf(resolutions.length) | ||
54 | |||
55 | // Check JSON files | ||
56 | for (const resolution of resolutions) { | ||
57 | const file = hlsFiles.find(f => f.resolution.id === resolution) | ||
58 | expect(file).to.not.be.undefined | ||
59 | |||
60 | expect(file.magnetUri).to.have.lengthOf.above(2) | ||
61 | expect(file.torrentUrl).to.match( | ||
62 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) | ||
63 | ) | ||
64 | |||
65 | if (objectStorageBaseUrl) { | ||
66 | expectStartWith(file.fileUrl, objectStorageBaseUrl) | ||
67 | } else { | ||
68 | expect(file.fileUrl).to.match( | ||
69 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | ||
70 | ) | ||
71 | } | ||
72 | |||
73 | expect(file.resolution.label).to.equal(resolution + 'p') | ||
74 | |||
75 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) | ||
76 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
77 | |||
78 | const torrent = await webtorrentAdd(file.magnetUri, true) | ||
79 | expect(torrent.files).to.be.an('array') | ||
80 | expect(torrent.files.length).to.equal(1) | ||
81 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | ||
82 | } | ||
83 | |||
84 | // Check master playlist | ||
85 | { | ||
86 | await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) | ||
87 | |||
88 | const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl }) | ||
89 | |||
90 | let i = 0 | ||
91 | for (const resolution of resolutions) { | ||
92 | expect(masterPlaylist).to.contain(`${resolution}.m3u8`) | ||
93 | expect(masterPlaylist).to.contain(`${resolution}.m3u8`) | ||
94 | |||
95 | const url = 'http://' + videoDetails.account.host | ||
96 | await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i) | ||
97 | |||
98 | i++ | ||
99 | } | ||
100 | } | ||
101 | |||
102 | // Check resolution playlists | ||
103 | { | ||
104 | for (const resolution of resolutions) { | ||
105 | const file = hlsFiles.find(f => f.resolution.id === resolution) | ||
106 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' | ||
107 | |||
108 | const url = objectStorageBaseUrl | ||
109 | ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}` | ||
110 | : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` | ||
111 | |||
112 | const subPlaylist = await server.streamingPlaylists.get({ url }) | ||
113 | |||
114 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) | ||
115 | expect(subPlaylist).to.contain(basename(file.fileUrl)) | ||
116 | } | ||
117 | } | ||
118 | |||
119 | { | ||
120 | const baseUrlAndPath = objectStorageBaseUrl | ||
121 | ? objectStorageBaseUrl + 'hls/' + videoUUID | ||
122 | : baseUrl + '/static/streaming-playlists/hls/' + videoUUID | ||
123 | |||
124 | for (const resolution of resolutions) { | ||
125 | await checkSegmentHash({ | ||
126 | server, | ||
127 | baseUrlPlaylist: baseUrlAndPath, | ||
128 | baseUrlSegment: baseUrlAndPath, | ||
129 | resolution, | ||
130 | hlsPlaylist | ||
131 | }) | ||
132 | } | ||
133 | } | ||
134 | } | ||
135 | } | ||
136 | |||
137 | describe('Test HLS videos', function () { | 18 | describe('Test HLS videos', function () { |
138 | let servers: PeerTubeServer[] = [] | 19 | let servers: PeerTubeServer[] = [] |
139 | let videoUUID = '' | ||
140 | let videoAudioUUID = '' | ||
141 | 20 | ||
142 | function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { | 21 | function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { |
22 | const videoUUIDs: string[] = [] | ||
143 | 23 | ||
144 | it('Should upload a video and transcode it to HLS', async function () { | 24 | it('Should upload a video and transcode it to HLS', async function () { |
145 | this.timeout(120000) | 25 | this.timeout(120000) |
146 | 26 | ||
147 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } }) | 27 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } }) |
148 | videoUUID = uuid | 28 | videoUUIDs.push(uuid) |
149 | 29 | ||
150 | await waitJobs(servers) | 30 | await waitJobs(servers) |
151 | 31 | ||
152 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) | 32 | await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) |
153 | }) | 33 | }) |
154 | 34 | ||
155 | it('Should upload an audio file and transcode it to HLS', async function () { | 35 | it('Should upload an audio file and transcode it to HLS', async function () { |
156 | this.timeout(120000) | 36 | this.timeout(120000) |
157 | 37 | ||
158 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } }) | 38 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } }) |
159 | videoAudioUUID = uuid | 39 | videoUUIDs.push(uuid) |
160 | 40 | ||
161 | await waitJobs(servers) | 41 | await waitJobs(servers) |
162 | 42 | ||
163 | await checkHlsPlaylist({ | 43 | await completeCheckHlsPlaylist({ |
164 | servers, | 44 | servers, |
165 | videoUUID: videoAudioUUID, | 45 | videoUUID: uuid, |
166 | hlsOnly, | 46 | hlsOnly, |
167 | resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], | 47 | resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], |
168 | objectStorageBaseUrl | 48 | objectStorageBaseUrl |
@@ -172,31 +52,36 @@ describe('Test HLS videos', function () { | |||
172 | it('Should update the video', async function () { | 52 | it('Should update the video', async function () { |
173 | this.timeout(30000) | 53 | this.timeout(30000) |
174 | 54 | ||
175 | await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video 1 updated' } }) | 55 | await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } }) |
176 | 56 | ||
177 | await waitJobs(servers) | 57 | await waitJobs(servers) |
178 | 58 | ||
179 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) | 59 | await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl }) |
180 | }) | 60 | }) |
181 | 61 | ||
182 | it('Should delete videos', async function () { | 62 | it('Should delete videos', async function () { |
183 | this.timeout(10000) | 63 | this.timeout(10000) |
184 | 64 | ||
185 | await servers[0].videos.remove({ id: videoUUID }) | 65 | for (const uuid of videoUUIDs) { |
186 | await servers[0].videos.remove({ id: videoAudioUUID }) | 66 | await servers[0].videos.remove({ id: uuid }) |
67 | } | ||
187 | 68 | ||
188 | await waitJobs(servers) | 69 | await waitJobs(servers) |
189 | 70 | ||
190 | for (const server of servers) { | 71 | for (const server of servers) { |
191 | await server.videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 72 | for (const uuid of videoUUIDs) { |
192 | await server.videos.get({ id: videoAudioUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | 73 | await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
74 | } | ||
193 | } | 75 | } |
194 | }) | 76 | }) |
195 | 77 | ||
196 | it('Should have the playlists/segment deleted from the disk', async function () { | 78 | it('Should have the playlists/segment deleted from the disk', async function () { |
197 | for (const server of servers) { | 79 | for (const server of servers) { |
198 | await checkDirectoryIsEmpty(server, 'videos') | 80 | await checkDirectoryIsEmpty(server, 'videos', [ 'private' ]) |
199 | await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls')) | 81 | await checkDirectoryIsEmpty(server, join('videos', 'private')) |
82 | |||
83 | await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ]) | ||
84 | await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private')) | ||
200 | } | 85 | } |
201 | }) | 86 | }) |
202 | 87 | ||
@@ -265,19 +150,19 @@ describe('Test HLS videos', function () { | |||
265 | }) | 150 | }) |
266 | 151 | ||
267 | describe('With object storage enabled', function () { | 152 | describe('With object storage enabled', function () { |
268 | if (areObjectStorageTestsDisabled()) return | 153 | if (areMockObjectStorageTestsDisabled()) return |
269 | 154 | ||
270 | before(async function () { | 155 | before(async function () { |
271 | this.timeout(120000) | 156 | this.timeout(120000) |
272 | 157 | ||
273 | const configOverride = ObjectStorageCommand.getDefaultConfig() | 158 | const configOverride = ObjectStorageCommand.getDefaultMockConfig() |
274 | await ObjectStorageCommand.prepareDefaultBuckets() | 159 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
275 | 160 | ||
276 | await servers[0].kill() | 161 | await servers[0].kill() |
277 | await servers[0].run(configOverride) | 162 | await servers[0].run(configOverride) |
278 | }) | 163 | }) |
279 | 164 | ||
280 | runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) | 165 | runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
281 | }) | 166 | }) |
282 | 167 | ||
283 | after(async function () { | 168 | after(async function () { |
diff --git a/server/tests/api/transcoding/index.ts b/server/tests/api/transcoding/index.ts index 0cc28b4a4..9866418d6 100644 --- a/server/tests/api/transcoding/index.ts +++ b/server/tests/api/transcoding/index.ts | |||
@@ -2,4 +2,5 @@ export * from './audio-only' | |||
2 | export * from './create-transcoding' | 2 | export * from './create-transcoding' |
3 | export * from './hls' | 3 | export * from './hls' |
4 | export * from './transcoder' | 4 | export * from './transcoder' |
5 | export * from './update-while-transcoding' | ||
5 | export * from './video-studio' | 6 | export * from './video-studio' |
diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts new file mode 100644 index 000000000..8e32ea069 --- /dev/null +++ b/server/tests/api/transcoding/update-while-transcoding.ts | |||
@@ -0,0 +1,151 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { completeCheckHlsPlaylist } from '@server/tests/shared' | ||
4 | import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' | ||
5 | import { VideoPrivacy } from '@shared/models' | ||
6 | import { | ||
7 | cleanupTests, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | ObjectStorageCommand, | ||
11 | PeerTubeServer, | ||
12 | setAccessTokensToServers, | ||
13 | waitJobs | ||
14 | } from '@shared/server-commands' | ||
15 | |||
16 | describe('Test update video privacy while transcoding', function () { | ||
17 | let servers: PeerTubeServer[] = [] | ||
18 | |||
19 | const videoUUIDs: string[] = [] | ||
20 | |||
21 | function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { | ||
22 | |||
23 | it('Should not have an error while quickly updating a private video to public after upload #1', async function () { | ||
24 | this.timeout(360_000) | ||
25 | |||
26 | const attributes = { | ||
27 | name: 'quick update', | ||
28 | privacy: VideoPrivacy.PRIVATE | ||
29 | } | ||
30 | |||
31 | const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false }) | ||
32 | await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) | ||
33 | videoUUIDs.push(uuid) | ||
34 | |||
35 | await waitJobs(servers) | ||
36 | |||
37 | await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) | ||
38 | }) | ||
39 | |||
40 | it('Should not have an error while quickly updating a private video to public after upload #2', async function () { | ||
41 | |||
42 | { | ||
43 | const attributes = { | ||
44 | name: 'quick update 2', | ||
45 | privacy: VideoPrivacy.PRIVATE | ||
46 | } | ||
47 | |||
48 | const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true }) | ||
49 | await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) | ||
50 | videoUUIDs.push(uuid) | ||
51 | |||
52 | await waitJobs(servers) | ||
53 | |||
54 | await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) | ||
55 | } | ||
56 | }) | ||
57 | |||
58 | it('Should not have an error while quickly updating a private video to public after upload #3', async function () { | ||
59 | const attributes = { | ||
60 | name: 'quick update 3', | ||
61 | privacy: VideoPrivacy.PRIVATE | ||
62 | } | ||
63 | |||
64 | const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true }) | ||
65 | await wait(1000) | ||
66 | await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) | ||
67 | videoUUIDs.push(uuid) | ||
68 | |||
69 | await waitJobs(servers) | ||
70 | |||
71 | await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) | ||
72 | }) | ||
73 | } | ||
74 | |||
75 | before(async function () { | ||
76 | this.timeout(120000) | ||
77 | |||
78 | const configOverride = { | ||
79 | transcoding: { | ||
80 | enabled: true, | ||
81 | allow_audio_files: true, | ||
82 | hls: { | ||
83 | enabled: true | ||
84 | } | ||
85 | } | ||
86 | } | ||
87 | servers = await createMultipleServers(2, configOverride) | ||
88 | |||
89 | // Get the access tokens | ||
90 | await setAccessTokensToServers(servers) | ||
91 | |||
92 | // Server 1 and server 2 follow each other | ||
93 | await doubleFollow(servers[0], servers[1]) | ||
94 | }) | ||
95 | |||
96 | describe('With WebTorrent & HLS enabled', function () { | ||
97 | runTestSuite(false) | ||
98 | }) | ||
99 | |||
100 | describe('With only HLS enabled', function () { | ||
101 | |||
102 | before(async function () { | ||
103 | await servers[0].config.updateCustomSubConfig({ | ||
104 | newConfig: { | ||
105 | transcoding: { | ||
106 | enabled: true, | ||
107 | allowAudioFiles: true, | ||
108 | resolutions: { | ||
109 | '144p': false, | ||
110 | '240p': true, | ||
111 | '360p': true, | ||
112 | '480p': true, | ||
113 | '720p': true, | ||
114 | '1080p': true, | ||
115 | '1440p': true, | ||
116 | '2160p': true | ||
117 | }, | ||
118 | hls: { | ||
119 | enabled: true | ||
120 | }, | ||
121 | webtorrent: { | ||
122 | enabled: false | ||
123 | } | ||
124 | } | ||
125 | } | ||
126 | }) | ||
127 | }) | ||
128 | |||
129 | runTestSuite(true) | ||
130 | }) | ||
131 | |||
132 | describe('With object storage enabled', function () { | ||
133 | if (areMockObjectStorageTestsDisabled()) return | ||
134 | |||
135 | before(async function () { | ||
136 | this.timeout(120000) | ||
137 | |||
138 | const configOverride = ObjectStorageCommand.getDefaultMockConfig() | ||
139 | await ObjectStorageCommand.prepareDefaultMockBuckets() | ||
140 | |||
141 | await servers[0].kill() | ||
142 | await servers[0].run(configOverride) | ||
143 | }) | ||
144 | |||
145 | runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl()) | ||
146 | }) | ||
147 | |||
148 | after(async function () { | ||
149 | await cleanupTests(servers) | ||
150 | }) | ||
151 | }) | ||
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index 9613111b5..ab08e8fb6 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { expect } from 'chai' | 1 | import { expect } from 'chai' |
2 | import { expectStartWith } from '@server/tests/shared' | 2 | import { expectStartWith } from '@server/tests/shared' |
3 | import { areObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' | 3 | import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' |
4 | import { VideoStudioTask } from '@shared/models' | 4 | import { VideoStudioTask } from '@shared/models' |
5 | import { | 5 | import { |
6 | cleanupTests, | 6 | cleanupTests, |
@@ -315,13 +315,13 @@ describe('Test video studio', function () { | |||
315 | }) | 315 | }) |
316 | 316 | ||
317 | describe('Object storage video edition', function () { | 317 | describe('Object storage video edition', function () { |
318 | if (areObjectStorageTestsDisabled()) return | 318 | if (areMockObjectStorageTestsDisabled()) return |
319 | 319 | ||
320 | before(async function () { | 320 | before(async function () { |
321 | await ObjectStorageCommand.prepareDefaultBuckets() | 321 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
322 | 322 | ||
323 | await servers[0].kill() | 323 | await servers[0].kill() |
324 | await servers[0].run(ObjectStorageCommand.getDefaultConfig()) | 324 | await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) |
325 | 325 | ||
326 | await servers[0].config.enableMinimumTranscoding() | 326 | await servers[0].config.enableMinimumTranscoding() |
327 | }) | 327 | }) |
@@ -344,11 +344,11 @@ describe('Test video studio', function () { | |||
344 | } | 344 | } |
345 | 345 | ||
346 | for (const webtorrentFile of video.files) { | 346 | for (const webtorrentFile of video.files) { |
347 | expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 347 | expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
348 | } | 348 | } |
349 | 349 | ||
350 | for (const hlsFile of video.streamingPlaylists[0].files) { | 350 | for (const hlsFile of video.streamingPlaylists[0].files) { |
351 | expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | 351 | expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) |
352 | } | 352 | } |
353 | 353 | ||
354 | await checkDuration(server, 9) | 354 | await checkDuration(server, 9) |
diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts index c65152c6f..643f1a531 100644 --- a/server/tests/api/users/index.ts +++ b/server/tests/api/users/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import './two-factor' | ||
1 | import './user-subscriptions' | 2 | import './user-subscriptions' |
2 | import './user-videos' | 3 | import './user-videos' |
3 | import './users' | 4 | import './users' |
diff --git a/server/tests/api/users/two-factor.ts b/server/tests/api/users/two-factor.ts new file mode 100644 index 000000000..0dcab9e17 --- /dev/null +++ b/server/tests/api/users/two-factor.ts | |||
@@ -0,0 +1,200 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { expectStartWith } from '@server/tests/shared' | ||
5 | import { HttpStatusCode } from '@shared/models' | ||
6 | import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands' | ||
7 | |||
8 | async function login (options: { | ||
9 | server: PeerTubeServer | ||
10 | username: string | ||
11 | password: string | ||
12 | otpToken?: string | ||
13 | expectedStatus?: HttpStatusCode | ||
14 | }) { | ||
15 | const { server, username, password, otpToken, expectedStatus } = options | ||
16 | |||
17 | const user = { username, password } | ||
18 | const { res, body: { access_token: token } } = await server.login.loginAndGetResponse({ user, otpToken, expectedStatus }) | ||
19 | |||
20 | return { res, token } | ||
21 | } | ||
22 | |||
23 | describe('Test users', function () { | ||
24 | let server: PeerTubeServer | ||
25 | let otpSecret: string | ||
26 | let requestToken: string | ||
27 | |||
28 | const userUsername = 'user1' | ||
29 | let userId: number | ||
30 | let userPassword: string | ||
31 | let userToken: string | ||
32 | |||
33 | before(async function () { | ||
34 | this.timeout(30000) | ||
35 | |||
36 | server = await createSingleServer(1) | ||
37 | |||
38 | await setAccessTokensToServers([ server ]) | ||
39 | const res = await server.users.generate(userUsername) | ||
40 | userId = res.userId | ||
41 | userPassword = res.password | ||
42 | userToken = res.token | ||
43 | }) | ||
44 | |||
45 | it('Should not add the header on login if two factor is not enabled', async function () { | ||
46 | const { res, token } = await login({ server, username: userUsername, password: userPassword }) | ||
47 | |||
48 | expect(res.header['x-peertube-otp']).to.not.exist | ||
49 | |||
50 | await server.users.getMyInfo({ token }) | ||
51 | }) | ||
52 | |||
53 | it('Should request two factor and get the secret and uri', async function () { | ||
54 | const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword }) | ||
55 | |||
56 | expect(otpRequest.requestToken).to.exist | ||
57 | |||
58 | expect(otpRequest.secret).to.exist | ||
59 | expect(otpRequest.secret).to.have.lengthOf(32) | ||
60 | |||
61 | expect(otpRequest.uri).to.exist | ||
62 | expectStartWith(otpRequest.uri, 'otpauth://') | ||
63 | expect(otpRequest.uri).to.include(otpRequest.secret) | ||
64 | |||
65 | requestToken = otpRequest.requestToken | ||
66 | otpSecret = otpRequest.secret | ||
67 | }) | ||
68 | |||
69 | it('Should not have two factor confirmed yet', async function () { | ||
70 | const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) | ||
71 | expect(twoFactorEnabled).to.be.false | ||
72 | }) | ||
73 | |||
74 | it('Should confirm two factor', async function () { | ||
75 | await server.twoFactor.confirmRequest({ | ||
76 | userId, | ||
77 | token: userToken, | ||
78 | otpToken: TwoFactorCommand.buildOTP({ secret: otpSecret }).generate(), | ||
79 | requestToken | ||
80 | }) | ||
81 | }) | ||
82 | |||
83 | it('Should not add the header on login if two factor is enabled and password is incorrect', async function () { | ||
84 | const { res, token } = await login({ server, username: userUsername, password: 'fake', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
85 | |||
86 | expect(res.header['x-peertube-otp']).to.not.exist | ||
87 | expect(token).to.not.exist | ||
88 | }) | ||
89 | |||
90 | it('Should add the header on login if two factor is enabled and password is correct', async function () { | ||
91 | const { res, token } = await login({ | ||
92 | server, | ||
93 | username: userUsername, | ||
94 | password: userPassword, | ||
95 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
96 | }) | ||
97 | |||
98 | expect(res.header['x-peertube-otp']).to.exist | ||
99 | expect(token).to.not.exist | ||
100 | |||
101 | await server.users.getMyInfo({ token }) | ||
102 | }) | ||
103 | |||
104 | it('Should not login with correct password and incorrect otp secret', async function () { | ||
105 | const otp = TwoFactorCommand.buildOTP({ secret: 'a'.repeat(32) }) | ||
106 | |||
107 | const { res, token } = await login({ | ||
108 | server, | ||
109 | username: userUsername, | ||
110 | password: userPassword, | ||
111 | otpToken: otp.generate(), | ||
112 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
113 | }) | ||
114 | |||
115 | expect(res.header['x-peertube-otp']).to.not.exist | ||
116 | expect(token).to.not.exist | ||
117 | }) | ||
118 | |||
119 | it('Should not login with correct password and incorrect otp code', async function () { | ||
120 | const { res, token } = await login({ | ||
121 | server, | ||
122 | username: userUsername, | ||
123 | password: userPassword, | ||
124 | otpToken: '123456', | ||
125 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
126 | }) | ||
127 | |||
128 | expect(res.header['x-peertube-otp']).to.not.exist | ||
129 | expect(token).to.not.exist | ||
130 | }) | ||
131 | |||
132 | it('Should not login with incorrect password and correct otp code', async function () { | ||
133 | const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate() | ||
134 | |||
135 | const { res, token } = await login({ | ||
136 | server, | ||
137 | username: userUsername, | ||
138 | password: 'fake', | ||
139 | otpToken, | ||
140 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
141 | }) | ||
142 | |||
143 | expect(res.header['x-peertube-otp']).to.not.exist | ||
144 | expect(token).to.not.exist | ||
145 | }) | ||
146 | |||
147 | it('Should correctly login with correct password and otp code', async function () { | ||
148 | const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate() | ||
149 | |||
150 | const { res, token } = await login({ server, username: userUsername, password: userPassword, otpToken }) | ||
151 | |||
152 | expect(res.header['x-peertube-otp']).to.not.exist | ||
153 | expect(token).to.exist | ||
154 | |||
155 | await server.users.getMyInfo({ token }) | ||
156 | }) | ||
157 | |||
158 | it('Should have two factor enabled when getting my info', async function () { | ||
159 | const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) | ||
160 | expect(twoFactorEnabled).to.be.true | ||
161 | }) | ||
162 | |||
163 | it('Should disable two factor and be able to login without otp token', async function () { | ||
164 | await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword }) | ||
165 | |||
166 | const { res, token } = await login({ server, username: userUsername, password: userPassword }) | ||
167 | expect(res.header['x-peertube-otp']).to.not.exist | ||
168 | |||
169 | await server.users.getMyInfo({ token }) | ||
170 | }) | ||
171 | |||
172 | it('Should have two factor disabled when getting my info', async function () { | ||
173 | const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) | ||
174 | expect(twoFactorEnabled).to.be.false | ||
175 | }) | ||
176 | |||
177 | it('Should enable two factor auth without password from an admin', async function () { | ||
178 | const { otpRequest } = await server.twoFactor.request({ userId }) | ||
179 | |||
180 | await server.twoFactor.confirmRequest({ | ||
181 | userId, | ||
182 | otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate(), | ||
183 | requestToken: otpRequest.requestToken | ||
184 | }) | ||
185 | |||
186 | const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) | ||
187 | expect(twoFactorEnabled).to.be.true | ||
188 | }) | ||
189 | |||
190 | it('Should disable two factor auth without password from an admin', async function () { | ||
191 | await server.twoFactor.disable({ userId }) | ||
192 | |||
193 | const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) | ||
194 | expect(twoFactorEnabled).to.be.false | ||
195 | }) | ||
196 | |||
197 | after(async function () { | ||
198 | await cleanupTests([ server ]) | ||
199 | }) | ||
200 | }) | ||
diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts index 62d668d1e..188e6f137 100644 --- a/server/tests/api/users/users-multiple-servers.ts +++ b/server/tests/api/users/users-multiple-servers.ts | |||
@@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () { | |||
197 | it('Should not have actor files', async () => { | 197 | it('Should not have actor files', async () => { |
198 | for (const server of servers) { | 198 | for (const server of servers) { |
199 | for (const userAvatarFilename of userAvatarFilenames) { | 199 | for (const userAvatarFilename of userAvatarFilenames) { |
200 | await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber) | 200 | await checkActorFilesWereRemoved(userAvatarFilename, server) |
201 | } | 201 | } |
202 | } | 202 | } |
203 | }) | 203 | }) |
diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts index 9e657b387..421b3ce16 100644 --- a/server/tests/api/users/users.ts +++ b/server/tests/api/users/users.ts | |||
@@ -181,7 +181,7 @@ describe('Test users', function () { | |||
181 | }) | 181 | }) |
182 | 182 | ||
183 | it('Should refresh the token', async function () { | 183 | it('Should refresh the token', async function () { |
184 | this.timeout(15000) | 184 | this.timeout(50000) |
185 | 185 | ||
186 | const futureDate = new Date(new Date().getTime() + 1000 * 60).toISOString() | 186 | const futureDate = new Date(new Date().getTime() + 1000 * 60).toISOString() |
187 | await server.sql.setTokenField(server.accessToken, 'refreshTokenExpiresAt', futureDate) | 187 | await server.sql.setTokenField(server.accessToken, 'refreshTokenExpiresAt', futureDate) |
@@ -219,7 +219,7 @@ describe('Test users', function () { | |||
219 | expect(user.email).to.equal('user_1@example.com') | 219 | expect(user.email).to.equal('user_1@example.com') |
220 | expect(user.nsfwPolicy).to.equal('display') | 220 | expect(user.nsfwPolicy).to.equal('display') |
221 | expect(user.videoQuota).to.equal(2 * 1024 * 1024) | 221 | expect(user.videoQuota).to.equal(2 * 1024 * 1024) |
222 | expect(user.roleLabel).to.equal('User') | 222 | expect(user.role.label).to.equal('User') |
223 | expect(user.id).to.be.a('number') | 223 | expect(user.id).to.be.a('number') |
224 | expect(user.account.displayName).to.equal('user_1') | 224 | expect(user.account.displayName).to.equal('user_1') |
225 | expect(user.account.description).to.be.null | 225 | expect(user.account.description).to.be.null |
@@ -277,7 +277,7 @@ describe('Test users', function () { | |||
277 | const user = data[0] | 277 | const user = data[0] |
278 | expect(user.username).to.equal('root') | 278 | expect(user.username).to.equal('root') |
279 | expect(user.email).to.equal('admin' + server.internalServerNumber + '@example.com') | 279 | expect(user.email).to.equal('admin' + server.internalServerNumber + '@example.com') |
280 | expect(user.roleLabel).to.equal('Administrator') | 280 | expect(user.role.label).to.equal('Administrator') |
281 | expect(user.nsfwPolicy).to.equal('display') | 281 | expect(user.nsfwPolicy).to.equal('display') |
282 | }) | 282 | }) |
283 | 283 | ||
@@ -531,7 +531,7 @@ describe('Test users', function () { | |||
531 | expect(user.emailVerified).to.be.true | 531 | expect(user.emailVerified).to.be.true |
532 | expect(user.nsfwPolicy).to.equal('do_not_list') | 532 | expect(user.nsfwPolicy).to.equal('do_not_list') |
533 | expect(user.videoQuota).to.equal(42) | 533 | expect(user.videoQuota).to.equal(42) |
534 | expect(user.roleLabel).to.equal('Moderator') | 534 | expect(user.role.label).to.equal('Moderator') |
535 | expect(user.id).to.be.a('number') | 535 | expect(user.id).to.be.a('number') |
536 | expect(user.adminFlags).to.equal(UserAdminFlag.NONE) | 536 | expect(user.adminFlags).to.equal(UserAdminFlag.NONE) |
537 | expect(user.pluginAuth).to.equal('toto') | 537 | expect(user.pluginAuth).to.equal('toto') |
diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts index 7cfd02fbb..a66f88a0e 100644 --- a/server/tests/api/videos/channel-import-videos.ts +++ b/server/tests/api/videos/channel-import-videos.ts | |||
@@ -109,6 +109,45 @@ describe('Test videos import in a channel', function () { | |||
109 | } | 109 | } |
110 | }) | 110 | }) |
111 | 111 | ||
112 | it('Should limit max amount of videos synced on full sync', async function () { | ||
113 | this.timeout(240_000) | ||
114 | |||
115 | await server.kill() | ||
116 | await server.run({ | ||
117 | import: { | ||
118 | video_channel_synchronization: { | ||
119 | full_sync_videos_limit: 1 | ||
120 | } | ||
121 | } | ||
122 | }) | ||
123 | |||
124 | const { id } = await server.channels.create({ attributes: { name: 'channel3' } }) | ||
125 | const channel3Id = id | ||
126 | |||
127 | const { videoChannelSync } = await server.channelSyncs.create({ | ||
128 | attributes: { | ||
129 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
130 | videoChannelId: channel3Id | ||
131 | } | ||
132 | }) | ||
133 | const syncId = videoChannelSync.id | ||
134 | |||
135 | await waitJobs(server) | ||
136 | |||
137 | await server.channels.importVideos({ | ||
138 | channelName: 'channel3', | ||
139 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
140 | videoChannelSyncId: syncId | ||
141 | }) | ||
142 | |||
143 | await waitJobs(server) | ||
144 | |||
145 | const { total, data } = await server.videos.listByChannel({ handle: 'channel3' }) | ||
146 | |||
147 | expect(total).to.equal(1) | ||
148 | expect(data).to.have.lengthOf(1) | ||
149 | }) | ||
150 | |||
112 | after(async function () { | 151 | after(async function () { |
113 | await server?.kill() | 152 | await server?.kill() |
114 | }) | 153 | }) |
@@ -116,5 +155,7 @@ describe('Test videos import in a channel', function () { | |||
116 | } | 155 | } |
117 | 156 | ||
118 | runSuite('yt-dlp') | 157 | runSuite('yt-dlp') |
119 | runSuite('youtube-dl') | 158 | |
159 | // FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails | ||
160 | // runSuite('youtube-dl') | ||
120 | }) | 161 | }) |
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 266155297..357c08199 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts | |||
@@ -19,3 +19,4 @@ import './videos-common-filters' | |||
19 | import './videos-history' | 19 | import './videos-history' |
20 | import './videos-overview' | 20 | import './videos-overview' |
21 | import './video-source' | 21 | import './video-source' |
22 | import './video-static-file-privacy' | ||
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index d47807a79..2ad749fd4 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts | |||
@@ -156,7 +156,7 @@ describe('Test multiple servers', function () { | |||
156 | }) | 156 | }) |
157 | 157 | ||
158 | it('Should upload the video on server 2 and propagate on each server', async function () { | 158 | it('Should upload the video on server 2 and propagate on each server', async function () { |
159 | this.timeout(100000) | 159 | this.timeout(240000) |
160 | 160 | ||
161 | const user = { | 161 | const user = { |
162 | username: 'user1', | 162 | username: 'user1', |
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts index 865b25f04..91291524d 100644 --- a/server/tests/api/videos/video-channel-syncs.ts +++ b/server/tests/api/videos/video-channel-syncs.ts | |||
@@ -220,7 +220,7 @@ describe('Test channel synchronizations', function () { | |||
220 | expect(total).to.equal(0) | 220 | expect(total).to.equal(0) |
221 | }) | 221 | }) |
222 | 222 | ||
223 | // FIXME: youtube-dl doesn't work when speicifying a port after the hostname | 223 | // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname |
224 | // it('Should import a remote PeerTube channel', async function () { | 224 | // it('Should import a remote PeerTube channel', async function () { |
225 | // this.timeout(240_000) | 225 | // this.timeout(240_000) |
226 | 226 | ||
diff --git a/server/tests/api/videos/video-description.ts b/server/tests/api/videos/video-description.ts index a4b3ff6e7..c4185882a 100644 --- a/server/tests/api/videos/video-description.ts +++ b/server/tests/api/videos/video-description.ts | |||
@@ -14,8 +14,12 @@ describe('Test video description', function () { | |||
14 | let servers: PeerTubeServer[] = [] | 14 | let servers: PeerTubeServer[] = [] |
15 | let videoUUID = '' | 15 | let videoUUID = '' |
16 | let videoId: number | 16 | let videoId: number |
17 | |||
17 | const longDescription = 'my super description for server 1'.repeat(50) | 18 | const longDescription = 'my super description for server 1'.repeat(50) |
18 | 19 | ||
20 | // 30 characters * 6 -> 240 characters | ||
21 | const truncatedDescription = 'my super description for server 1'.repeat(7) + 'my super descrip...' | ||
22 | |||
19 | before(async function () { | 23 | before(async function () { |
20 | this.timeout(40000) | 24 | this.timeout(40000) |
21 | 25 | ||
@@ -45,15 +49,22 @@ describe('Test video description', function () { | |||
45 | videoUUID = data[0].uuid | 49 | videoUUID = data[0].uuid |
46 | }) | 50 | }) |
47 | 51 | ||
48 | it('Should have a truncated description on each server', async function () { | 52 | it('Should have a truncated description on each server when listing videos', async function () { |
49 | for (const server of servers) { | 53 | for (const server of servers) { |
50 | const video = await server.videos.get({ id: videoUUID }) | 54 | const { data } = await server.videos.list() |
51 | 55 | const video = data.find(v => v.uuid === videoUUID) | |
52 | // 30 characters * 6 -> 240 characters | ||
53 | const truncatedDescription = 'my super description for server 1'.repeat(7) + | ||
54 | 'my super descrip...' | ||
55 | 56 | ||
56 | expect(video.description).to.equal(truncatedDescription) | 57 | expect(video.description).to.equal(truncatedDescription) |
58 | expect(video.truncatedDescription).to.equal(truncatedDescription) | ||
59 | } | ||
60 | }) | ||
61 | |||
62 | it('Should not have a truncated description on each server when getting videos', async function () { | ||
63 | for (const server of servers) { | ||
64 | const video = await server.videos.get({ id: videoUUID }) | ||
65 | |||
66 | expect(video.description).to.equal(longDescription) | ||
67 | expect(video.truncatedDescription).to.equal(truncatedDescription) | ||
57 | } | 68 | } |
58 | }) | 69 | }) |
59 | 70 | ||
diff --git a/server/tests/api/videos/video-files.ts b/server/tests/api/videos/video-files.ts index 10277b9cf..8c913bf31 100644 --- a/server/tests/api/videos/video-files.ts +++ b/server/tests/api/videos/video-files.ts | |||
@@ -33,7 +33,7 @@ describe('Test videos files', function () { | |||
33 | let validId2: string | 33 | let validId2: string |
34 | 34 | ||
35 | before(async function () { | 35 | before(async function () { |
36 | this.timeout(120_000) | 36 | this.timeout(360_000) |
37 | 37 | ||
38 | { | 38 | { |
39 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | 39 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) |
@@ -153,7 +153,7 @@ describe('Test videos files', function () { | |||
153 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) | 153 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) |
154 | expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist | 154 | expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist |
155 | 155 | ||
156 | const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl) | 156 | const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
157 | 157 | ||
158 | expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false | 158 | expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false |
159 | expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true | 159 | expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true |
diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts index 47b8c7b1e..a3de73ba5 100644 --- a/server/tests/api/videos/video-playlists.ts +++ b/server/tests/api/videos/video-playlists.ts | |||
@@ -23,6 +23,7 @@ import { | |||
23 | setDefaultVideoChannel, | 23 | setDefaultVideoChannel, |
24 | waitJobs | 24 | waitJobs |
25 | } from '@shared/server-commands' | 25 | } from '@shared/server-commands' |
26 | import { uuidToShort } from '@shared/extra-utils' | ||
26 | 27 | ||
27 | async function checkPlaylistElementType ( | 28 | async function checkPlaylistElementType ( |
28 | servers: PeerTubeServer[], | 29 | servers: PeerTubeServer[], |
@@ -56,6 +57,7 @@ describe('Test video playlists', function () { | |||
56 | let playlistServer2UUID2: string | 57 | let playlistServer2UUID2: string |
57 | 58 | ||
58 | let playlistServer1Id: number | 59 | let playlistServer1Id: number |
60 | let playlistServer1DisplayName: string | ||
59 | let playlistServer1UUID: string | 61 | let playlistServer1UUID: string |
60 | let playlistServer1UUID2: string | 62 | let playlistServer1UUID2: string |
61 | 63 | ||
@@ -70,7 +72,7 @@ describe('Test video playlists', function () { | |||
70 | let commands: PlaylistsCommand[] | 72 | let commands: PlaylistsCommand[] |
71 | 73 | ||
72 | before(async function () { | 74 | before(async function () { |
73 | this.timeout(120000) | 75 | this.timeout(240000) |
74 | 76 | ||
75 | servers = await createMultipleServers(3) | 77 | servers = await createMultipleServers(3) |
76 | 78 | ||
@@ -489,15 +491,17 @@ describe('Test video playlists', function () { | |||
489 | return commands[0].addElement({ playlistId: playlistServer1Id, attributes }) | 491 | return commands[0].addElement({ playlistId: playlistServer1Id, attributes }) |
490 | } | 492 | } |
491 | 493 | ||
494 | const playlistDisplayName = 'playlist 4' | ||
492 | const playlist = await commands[0].create({ | 495 | const playlist = await commands[0].create({ |
493 | attributes: { | 496 | attributes: { |
494 | displayName: 'playlist 4', | 497 | displayName: playlistDisplayName, |
495 | privacy: VideoPlaylistPrivacy.PUBLIC, | 498 | privacy: VideoPlaylistPrivacy.PUBLIC, |
496 | videoChannelId: servers[0].store.channel.id | 499 | videoChannelId: servers[0].store.channel.id |
497 | } | 500 | } |
498 | }) | 501 | }) |
499 | 502 | ||
500 | playlistServer1Id = playlist.id | 503 | playlistServer1Id = playlist.id |
504 | playlistServer1DisplayName = playlistDisplayName | ||
501 | playlistServer1UUID = playlist.uuid | 505 | playlistServer1UUID = playlist.uuid |
502 | 506 | ||
503 | await addVideo({ videoId: servers[0].store.videos[0].uuid, startTimestamp: 15, stopTimestamp: 28 }) | 507 | await addVideo({ videoId: servers[0].store.videos[0].uuid, startTimestamp: 15, stopTimestamp: 28 }) |
@@ -908,6 +912,8 @@ describe('Test video playlists', function () { | |||
908 | const elem = obj[servers[0].store.videos[0].id] | 912 | const elem = obj[servers[0].store.videos[0].id] |
909 | expect(elem).to.have.lengthOf(1) | 913 | expect(elem).to.have.lengthOf(1) |
910 | expect(elem[0].playlistElementId).to.exist | 914 | expect(elem[0].playlistElementId).to.exist |
915 | expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) | ||
916 | expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) | ||
911 | expect(elem[0].playlistId).to.equal(playlistServer1Id) | 917 | expect(elem[0].playlistId).to.equal(playlistServer1Id) |
912 | expect(elem[0].startTimestamp).to.equal(15) | 918 | expect(elem[0].startTimestamp).to.equal(15) |
913 | expect(elem[0].stopTimestamp).to.equal(28) | 919 | expect(elem[0].stopTimestamp).to.equal(28) |
@@ -917,6 +923,8 @@ describe('Test video playlists', function () { | |||
917 | const elem = obj[servers[0].store.videos[3].id] | 923 | const elem = obj[servers[0].store.videos[3].id] |
918 | expect(elem).to.have.lengthOf(1) | 924 | expect(elem).to.have.lengthOf(1) |
919 | expect(elem[0].playlistElementId).to.equal(playlistElementServer1Video4) | 925 | expect(elem[0].playlistElementId).to.equal(playlistElementServer1Video4) |
926 | expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) | ||
927 | expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) | ||
920 | expect(elem[0].playlistId).to.equal(playlistServer1Id) | 928 | expect(elem[0].playlistId).to.equal(playlistServer1Id) |
921 | expect(elem[0].startTimestamp).to.equal(1) | 929 | expect(elem[0].startTimestamp).to.equal(1) |
922 | expect(elem[0].stopTimestamp).to.equal(35) | 930 | expect(elem[0].stopTimestamp).to.equal(35) |
@@ -926,6 +934,8 @@ describe('Test video playlists', function () { | |||
926 | const elem = obj[servers[0].store.videos[4].id] | 934 | const elem = obj[servers[0].store.videos[4].id] |
927 | expect(elem).to.have.lengthOf(1) | 935 | expect(elem).to.have.lengthOf(1) |
928 | expect(elem[0].playlistId).to.equal(playlistServer1Id) | 936 | expect(elem[0].playlistId).to.equal(playlistServer1Id) |
937 | expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) | ||
938 | expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) | ||
929 | expect(elem[0].startTimestamp).to.equal(45) | 939 | expect(elem[0].startTimestamp).to.equal(45) |
930 | expect(elem[0].stopTimestamp).to.equal(null) | 940 | expect(elem[0].stopTimestamp).to.equal(null) |
931 | } | 941 | } |
@@ -1049,7 +1059,7 @@ describe('Test video playlists', function () { | |||
1049 | this.timeout(30000) | 1059 | this.timeout(30000) |
1050 | 1060 | ||
1051 | for (const server of servers) { | 1061 | for (const server of servers) { |
1052 | await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber) | 1062 | await checkPlaylistFilesWereRemoved(playlistServer1UUID, server) |
1053 | } | 1063 | } |
1054 | }) | 1064 | }) |
1055 | 1065 | ||
diff --git a/server/tests/api/videos/video-privacy.ts b/server/tests/api/videos/video-privacy.ts index b18c71c94..264a05d3f 100644 --- a/server/tests/api/videos/video-privacy.ts +++ b/server/tests/api/videos/video-privacy.ts | |||
@@ -45,7 +45,7 @@ describe('Test video privacy', function () { | |||
45 | describe('Private and internal videos', function () { | 45 | describe('Private and internal videos', function () { |
46 | 46 | ||
47 | it('Should upload a private and internal videos on server 1', async function () { | 47 | it('Should upload a private and internal videos on server 1', async function () { |
48 | this.timeout(10000) | 48 | this.timeout(50000) |
49 | 49 | ||
50 | for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { | 50 | for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { |
51 | const attributes = { privacy } | 51 | const attributes = { privacy } |
@@ -128,7 +128,7 @@ describe('Test video privacy', function () { | |||
128 | describe('Unlisted videos', function () { | 128 | describe('Unlisted videos', function () { |
129 | 129 | ||
130 | it('Should upload an unlisted video on server 2', async function () { | 130 | it('Should upload an unlisted video on server 2', async function () { |
131 | this.timeout(60000) | 131 | this.timeout(120000) |
132 | 132 | ||
133 | const attributes = { | 133 | const attributes = { |
134 | name: 'unlisted video', | 134 | name: 'unlisted video', |
diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts new file mode 100644 index 000000000..eaaed5aad --- /dev/null +++ b/server/tests/api/videos/video-static-file-privacy.ts | |||
@@ -0,0 +1,422 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { decode } from 'magnet-uri' | ||
5 | import { expectStartWith } from '@server/tests/shared' | ||
6 | import { getAllFiles, wait } from '@shared/core-utils' | ||
7 | import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' | ||
8 | import { | ||
9 | cleanupTests, | ||
10 | createSingleServer, | ||
11 | findExternalSavedVideo, | ||
12 | makeRawRequest, | ||
13 | parseTorrentVideo, | ||
14 | PeerTubeServer, | ||
15 | sendRTMPStream, | ||
16 | setAccessTokensToServers, | ||
17 | setDefaultVideoChannel, | ||
18 | stopFfmpeg, | ||
19 | waitJobs | ||
20 | } from '@shared/server-commands' | ||
21 | |||
22 | describe('Test video static file privacy', function () { | ||
23 | let server: PeerTubeServer | ||
24 | let userToken: string | ||
25 | |||
26 | before(async function () { | ||
27 | this.timeout(50000) | ||
28 | |||
29 | server = await createSingleServer(1) | ||
30 | await setAccessTokensToServers([ server ]) | ||
31 | await setDefaultVideoChannel([ server ]) | ||
32 | |||
33 | userToken = await server.users.generateUserAndToken('user1') | ||
34 | }) | ||
35 | |||
36 | describe('VOD static file path', function () { | ||
37 | |||
38 | function runSuite () { | ||
39 | |||
40 | async function checkPrivateFiles (uuid: string) { | ||
41 | const video = await server.videos.getWithToken({ id: uuid }) | ||
42 | |||
43 | for (const file of video.files) { | ||
44 | expect(file.fileDownloadUrl).to.not.include('/private/') | ||
45 | expectStartWith(file.fileUrl, server.url + '/static/webseed/private/') | ||
46 | |||
47 | const torrent = await parseTorrentVideo(server, file) | ||
48 | expect(torrent.urlList).to.have.lengthOf(0) | ||
49 | |||
50 | const magnet = decode(file.magnetUri) | ||
51 | expect(magnet.urlList).to.have.lengthOf(0) | ||
52 | |||
53 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
54 | } | ||
55 | |||
56 | const hls = video.streamingPlaylists[0] | ||
57 | if (hls) { | ||
58 | expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/') | ||
59 | expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/') | ||
60 | |||
61 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
62 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
63 | } | ||
64 | } | ||
65 | |||
66 | async function checkPublicFiles (uuid: string) { | ||
67 | const video = await server.videos.get({ id: uuid }) | ||
68 | |||
69 | for (const file of getAllFiles(video)) { | ||
70 | expect(file.fileDownloadUrl).to.not.include('/private/') | ||
71 | expect(file.fileUrl).to.not.include('/private/') | ||
72 | |||
73 | const torrent = await parseTorrentVideo(server, file) | ||
74 | expect(torrent.urlList[0]).to.not.include('private') | ||
75 | |||
76 | const magnet = decode(file.magnetUri) | ||
77 | expect(magnet.urlList[0]).to.not.include('private') | ||
78 | |||
79 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
80 | await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 }) | ||
81 | await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 }) | ||
82 | } | ||
83 | |||
84 | const hls = video.streamingPlaylists[0] | ||
85 | if (hls) { | ||
86 | expect(hls.playlistUrl).to.not.include('private') | ||
87 | expect(hls.segmentsSha256Url).to.not.include('private') | ||
88 | |||
89 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
90 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | ||
91 | } | ||
92 | } | ||
93 | |||
94 | it('Should upload a private/internal video and have a private static path', async function () { | ||
95 | this.timeout(120000) | ||
96 | |||
97 | for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { | ||
98 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy }) | ||
99 | await waitJobs([ server ]) | ||
100 | |||
101 | await checkPrivateFiles(uuid) | ||
102 | } | ||
103 | }) | ||
104 | |||
105 | it('Should upload a public video and update it as private/internal to have a private static path', async function () { | ||
106 | this.timeout(120000) | ||
107 | |||
108 | for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { | ||
109 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC }) | ||
110 | await waitJobs([ server ]) | ||
111 | |||
112 | await server.videos.update({ id: uuid, attributes: { privacy } }) | ||
113 | await waitJobs([ server ]) | ||
114 | |||
115 | await checkPrivateFiles(uuid) | ||
116 | } | ||
117 | }) | ||
118 | |||
119 | it('Should upload a private video and update it to unlisted to have a public static path', async function () { | ||
120 | this.timeout(120000) | ||
121 | |||
122 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
123 | await waitJobs([ server ]) | ||
124 | |||
125 | await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) | ||
126 | await waitJobs([ server ]) | ||
127 | |||
128 | await checkPublicFiles(uuid) | ||
129 | }) | ||
130 | |||
131 | it('Should upload an internal video and update it to public to have a public static path', async function () { | ||
132 | this.timeout(120000) | ||
133 | |||
134 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) | ||
135 | await waitJobs([ server ]) | ||
136 | |||
137 | await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) | ||
138 | await waitJobs([ server ]) | ||
139 | |||
140 | await checkPublicFiles(uuid) | ||
141 | }) | ||
142 | |||
143 | it('Should upload an internal video and schedule a public publish', async function () { | ||
144 | this.timeout(120000) | ||
145 | |||
146 | const attributes = { | ||
147 | name: 'video', | ||
148 | privacy: VideoPrivacy.PRIVATE, | ||
149 | scheduleUpdate: { | ||
150 | updateAt: new Date(Date.now() + 1000).toISOString(), | ||
151 | privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC | ||
152 | } | ||
153 | } | ||
154 | |||
155 | const { uuid } = await server.videos.upload({ attributes }) | ||
156 | |||
157 | await waitJobs([ server ]) | ||
158 | await wait(1000) | ||
159 | await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } }) | ||
160 | |||
161 | await waitJobs([ server ]) | ||
162 | |||
163 | await checkPublicFiles(uuid) | ||
164 | }) | ||
165 | } | ||
166 | |||
167 | describe('Without transcoding', function () { | ||
168 | runSuite() | ||
169 | }) | ||
170 | |||
171 | describe('With transcoding', function () { | ||
172 | |||
173 | before(async function () { | ||
174 | await server.config.enableMinimumTranscoding() | ||
175 | }) | ||
176 | |||
177 | runSuite() | ||
178 | }) | ||
179 | }) | ||
180 | |||
181 | describe('VOD static file right check', function () { | ||
182 | let unrelatedFileToken: string | ||
183 | |||
184 | async function checkVideoFiles (options: { | ||
185 | id: string | ||
186 | expectedStatus: HttpStatusCode | ||
187 | token: string | ||
188 | videoFileToken: string | ||
189 | }) { | ||
190 | const { id, expectedStatus, token, videoFileToken } = options | ||
191 | |||
192 | const video = await server.videos.getWithToken({ id }) | ||
193 | |||
194 | for (const file of getAllFiles(video)) { | ||
195 | await makeRawRequest({ url: file.fileUrl, token, expectedStatus }) | ||
196 | await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus }) | ||
197 | |||
198 | await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus }) | ||
199 | await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus }) | ||
200 | } | ||
201 | |||
202 | const hls = video.streamingPlaylists[0] | ||
203 | await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus }) | ||
204 | await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus }) | ||
205 | |||
206 | await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus }) | ||
207 | await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus }) | ||
208 | } | ||
209 | |||
210 | before(async function () { | ||
211 | await server.config.enableMinimumTranscoding() | ||
212 | |||
213 | const { uuid } = await server.videos.quickUpload({ name: 'another video' }) | ||
214 | unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) | ||
215 | }) | ||
216 | |||
217 | it('Should not be able to access a private video files without OAuth token and file token', async function () { | ||
218 | this.timeout(120000) | ||
219 | |||
220 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) | ||
221 | await waitJobs([ server ]) | ||
222 | |||
223 | await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null }) | ||
224 | }) | ||
225 | |||
226 | it('Should not be able to access an internal video files without appropriate OAuth token and file token', async function () { | ||
227 | this.timeout(120000) | ||
228 | |||
229 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
230 | await waitJobs([ server ]) | ||
231 | |||
232 | await checkVideoFiles({ | ||
233 | id: uuid, | ||
234 | expectedStatus: HttpStatusCode.FORBIDDEN_403, | ||
235 | token: userToken, | ||
236 | videoFileToken: unrelatedFileToken | ||
237 | }) | ||
238 | }) | ||
239 | |||
240 | it('Should be able to access a private video files with appropriate OAuth token or file token', async function () { | ||
241 | this.timeout(120000) | ||
242 | |||
243 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
244 | const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) | ||
245 | |||
246 | await waitJobs([ server ]) | ||
247 | |||
248 | await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) | ||
249 | }) | ||
250 | |||
251 | it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () { | ||
252 | this.timeout(120000) | ||
253 | |||
254 | const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE }) | ||
255 | const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) | ||
256 | |||
257 | await waitJobs([ server ]) | ||
258 | |||
259 | await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) | ||
260 | }) | ||
261 | }) | ||
262 | |||
263 | describe('Live static file path and check', function () { | ||
264 | let normalLiveId: string | ||
265 | let normalLive: LiveVideo | ||
266 | |||
267 | let permanentLiveId: string | ||
268 | let permanentLive: LiveVideo | ||
269 | |||
270 | let unrelatedFileToken: string | ||
271 | |||
272 | async function checkLiveFiles (live: LiveVideo, liveId: string) { | ||
273 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) | ||
274 | await server.live.waitUntilPublished({ videoId: liveId }) | ||
275 | |||
276 | const video = await server.videos.getWithToken({ id: liveId }) | ||
277 | const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) | ||
278 | |||
279 | const hls = video.streamingPlaylists[0] | ||
280 | |||
281 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
282 | expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/') | ||
283 | |||
284 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
285 | await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
286 | |||
287 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
288 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
289 | await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
290 | } | ||
291 | |||
292 | await stopFfmpeg(ffmpegCommand) | ||
293 | } | ||
294 | |||
295 | async function checkReplay (replay: VideoDetails) { | ||
296 | const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) | ||
297 | |||
298 | const hls = replay.streamingPlaylists[0] | ||
299 | expect(hls.files).to.not.have.lengthOf(0) | ||
300 | |||
301 | for (const file of hls.files) { | ||
302 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
303 | await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
304 | |||
305 | await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
306 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
307 | await makeRawRequest({ | ||
308 | url: file.fileUrl, | ||
309 | query: { videoFileToken: unrelatedFileToken }, | ||
310 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
311 | }) | ||
312 | } | ||
313 | |||
314 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
315 | expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/') | ||
316 | |||
317 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
318 | await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
319 | |||
320 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
321 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
322 | await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
323 | } | ||
324 | } | ||
325 | |||
326 | before(async function () { | ||
327 | await server.config.enableMinimumTranscoding() | ||
328 | |||
329 | const { uuid } = await server.videos.quickUpload({ name: 'another video' }) | ||
330 | unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) | ||
331 | |||
332 | await server.config.enableLive({ | ||
333 | allowReplay: true, | ||
334 | transcoding: true, | ||
335 | resolutions: 'min' | ||
336 | }) | ||
337 | |||
338 | { | ||
339 | const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE }) | ||
340 | normalLiveId = video.uuid | ||
341 | normalLive = live | ||
342 | } | ||
343 | |||
344 | { | ||
345 | const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE }) | ||
346 | permanentLiveId = video.uuid | ||
347 | permanentLive = live | ||
348 | } | ||
349 | }) | ||
350 | |||
351 | it('Should create a private normal live and have a private static path', async function () { | ||
352 | this.timeout(240000) | ||
353 | |||
354 | await checkLiveFiles(normalLive, normalLiveId) | ||
355 | }) | ||
356 | |||
357 | it('Should create a private permanent live and have a private static path', async function () { | ||
358 | this.timeout(240000) | ||
359 | |||
360 | await checkLiveFiles(permanentLive, permanentLiveId) | ||
361 | }) | ||
362 | |||
363 | it('Should have created a replay of the normal live with a private static path', async function () { | ||
364 | this.timeout(240000) | ||
365 | |||
366 | await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) | ||
367 | |||
368 | const replay = await server.videos.getWithToken({ id: normalLiveId }) | ||
369 | await checkReplay(replay) | ||
370 | }) | ||
371 | |||
372 | it('Should have created a replay of the permanent live with a private static path', async function () { | ||
373 | this.timeout(240000) | ||
374 | |||
375 | await server.live.waitUntilWaiting({ videoId: permanentLiveId }) | ||
376 | await waitJobs([ server ]) | ||
377 | |||
378 | const live = await server.videos.getWithToken({ id: permanentLiveId }) | ||
379 | const replayFromList = await findExternalSavedVideo(server, live) | ||
380 | const replay = await server.videos.getWithToken({ id: replayFromList.id }) | ||
381 | |||
382 | await checkReplay(replay) | ||
383 | }) | ||
384 | }) | ||
385 | |||
386 | describe('With static file right check disabled', function () { | ||
387 | let videoUUID: string | ||
388 | |||
389 | before(async function () { | ||
390 | this.timeout(240000) | ||
391 | |||
392 | await server.kill() | ||
393 | |||
394 | await server.run({ | ||
395 | static_files: { | ||
396 | private_files_require_auth: false | ||
397 | } | ||
398 | }) | ||
399 | |||
400 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) | ||
401 | videoUUID = uuid | ||
402 | |||
403 | await waitJobs([ server ]) | ||
404 | }) | ||
405 | |||
406 | it('Should not check auth for private static files', async function () { | ||
407 | const video = await server.videos.getWithToken({ id: videoUUID }) | ||
408 | |||
409 | for (const file of getAllFiles(video)) { | ||
410 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
411 | } | ||
412 | |||
413 | const hls = video.streamingPlaylists[0] | ||
414 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
415 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | ||
416 | }) | ||
417 | }) | ||
418 | |||
419 | after(async function () { | ||
420 | await cleanupTests([ server ]) | ||
421 | }) | ||
422 | }) | ||
diff --git a/server/tests/api/videos/videos-common-filters.ts b/server/tests/api/videos/videos-common-filters.ts index e7fc15e42..b176d90ab 100644 --- a/server/tests/api/videos/videos-common-filters.ts +++ b/server/tests/api/videos/videos-common-filters.ts | |||
@@ -232,7 +232,7 @@ describe('Test videos filter', function () { | |||
232 | }) | 232 | }) |
233 | 233 | ||
234 | it('Should display only remote videos', async function () { | 234 | it('Should display only remote videos', async function () { |
235 | this.timeout(40000) | 235 | this.timeout(120000) |
236 | 236 | ||
237 | await servers[1].videos.upload({ attributes: { name: 'remote video' } }) | 237 | await servers[1].videos.upload({ attributes: { name: 'remote video' } }) |
238 | 238 | ||
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index 2cf2dd8f8..43f53035b 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 4 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
5 | import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models' | 5 | import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models' |
6 | import { | 6 | import { |
7 | cleanupTests, | 7 | cleanupTests, |
@@ -27,9 +27,9 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s | |||
27 | 27 | ||
28 | async function checkFiles (video: VideoDetails, objectStorage: boolean) { | 28 | async function checkFiles (video: VideoDetails, objectStorage: boolean) { |
29 | for (const file of video.files) { | 29 | for (const file of video.files) { |
30 | if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | 30 | if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
31 | 31 | ||
32 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 32 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
33 | } | 33 | } |
34 | } | 34 | } |
35 | 35 | ||
@@ -43,7 +43,7 @@ function runTests (objectStorage: boolean) { | |||
43 | this.timeout(90000) | 43 | this.timeout(90000) |
44 | 44 | ||
45 | const config = objectStorage | 45 | const config = objectStorage |
46 | ? ObjectStorageCommand.getDefaultConfig() | 46 | ? ObjectStorageCommand.getDefaultMockConfig() |
47 | : {} | 47 | : {} |
48 | 48 | ||
49 | // Run server 2 to have transcoding enabled | 49 | // Run server 2 to have transcoding enabled |
@@ -52,7 +52,7 @@ function runTests (objectStorage: boolean) { | |||
52 | 52 | ||
53 | await doubleFollow(servers[0], servers[1]) | 53 | await doubleFollow(servers[0], servers[1]) |
54 | 54 | ||
55 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | 55 | if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets() |
56 | 56 | ||
57 | // Upload two videos for our needs | 57 | // Upload two videos for our needs |
58 | { | 58 | { |
@@ -157,7 +157,7 @@ describe('Test create import video jobs', function () { | |||
157 | }) | 157 | }) |
158 | 158 | ||
159 | describe('On object storage', function () { | 159 | describe('On object storage', function () { |
160 | if (areObjectStorageTestsDisabled()) return | 160 | if (areMockObjectStorageTestsDisabled()) return |
161 | 161 | ||
162 | runTests(true) | 162 | runTests(true) |
163 | }) | 163 | }) |
diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts index 6a12a2c6c..c357f501b 100644 --- a/server/tests/cli/create-move-video-storage-job.ts +++ b/server/tests/cli/create-move-video-storage-job.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 3 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
4 | import { HttpStatusCode, VideoDetails } from '@shared/models' | 4 | import { HttpStatusCode, VideoDetails } from '@shared/models' |
5 | import { | 5 | import { |
6 | cleanupTests, | 6 | cleanupTests, |
@@ -17,16 +17,16 @@ import { expectStartWith } from '../shared' | |||
17 | async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) { | 17 | async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) { |
18 | for (const file of video.files) { | 18 | for (const file of video.files) { |
19 | const start = inObjectStorage | 19 | const start = inObjectStorage |
20 | ? ObjectStorageCommand.getWebTorrentBaseUrl() | 20 | ? ObjectStorageCommand.getMockWebTorrentBaseUrl() |
21 | : origin.url | 21 | : origin.url |
22 | 22 | ||
23 | expectStartWith(file.fileUrl, start) | 23 | expectStartWith(file.fileUrl, start) |
24 | 24 | ||
25 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 25 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
26 | } | 26 | } |
27 | 27 | ||
28 | const start = inObjectStorage | 28 | const start = inObjectStorage |
29 | ? ObjectStorageCommand.getPlaylistBaseUrl() | 29 | ? ObjectStorageCommand.getMockPlaylistBaseUrl() |
30 | : origin.url | 30 | : origin.url |
31 | 31 | ||
32 | const hls = video.streamingPlaylists[0] | 32 | const hls = video.streamingPlaylists[0] |
@@ -36,12 +36,12 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject | |||
36 | for (const file of hls.files) { | 36 | for (const file of hls.files) { |
37 | expectStartWith(file.fileUrl, start) | 37 | expectStartWith(file.fileUrl, start) |
38 | 38 | ||
39 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 39 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
40 | } | 40 | } |
41 | } | 41 | } |
42 | 42 | ||
43 | describe('Test create move video storage job', function () { | 43 | describe('Test create move video storage job', function () { |
44 | if (areObjectStorageTestsDisabled()) return | 44 | if (areMockObjectStorageTestsDisabled()) return |
45 | 45 | ||
46 | let servers: PeerTubeServer[] = [] | 46 | let servers: PeerTubeServer[] = [] |
47 | const uuids: string[] = [] | 47 | const uuids: string[] = [] |
@@ -55,7 +55,7 @@ describe('Test create move video storage job', function () { | |||
55 | 55 | ||
56 | await doubleFollow(servers[0], servers[1]) | 56 | await doubleFollow(servers[0], servers[1]) |
57 | 57 | ||
58 | await ObjectStorageCommand.prepareDefaultBuckets() | 58 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
59 | 59 | ||
60 | await servers[0].config.enableTranscoding() | 60 | await servers[0].config.enableTranscoding() |
61 | 61 | ||
@@ -67,14 +67,14 @@ describe('Test create move video storage job', function () { | |||
67 | await waitJobs(servers) | 67 | await waitJobs(servers) |
68 | 68 | ||
69 | await servers[0].kill() | 69 | await servers[0].kill() |
70 | await servers[0].run(ObjectStorageCommand.getDefaultConfig()) | 70 | await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) |
71 | }) | 71 | }) |
72 | 72 | ||
73 | it('Should move only one file', async function () { | 73 | it('Should move only one file', async function () { |
74 | this.timeout(120000) | 74 | this.timeout(120000) |
75 | 75 | ||
76 | const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` | 76 | const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` |
77 | await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) | 77 | await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig()) |
78 | await waitJobs(servers) | 78 | await waitJobs(servers) |
79 | 79 | ||
80 | for (const server of servers) { | 80 | for (const server of servers) { |
@@ -94,7 +94,7 @@ describe('Test create move video storage job', function () { | |||
94 | this.timeout(120000) | 94 | this.timeout(120000) |
95 | 95 | ||
96 | const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` | 96 | const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` |
97 | await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) | 97 | await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig()) |
98 | await waitJobs(servers) | 98 | await waitJobs(servers) |
99 | 99 | ||
100 | for (const server of servers) { | 100 | for (const server of servers) { |
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts index 8897d8c23..38b737829 100644 --- a/server/tests/cli/create-transcoding-job.ts +++ b/server/tests/cli/create-transcoding-job.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | 4 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
5 | import { HttpStatusCode, VideoFile } from '@shared/models' | 5 | import { HttpStatusCode, VideoFile } from '@shared/models' |
6 | import { | 6 | import { |
7 | cleanupTests, | 7 | cleanupTests, |
@@ -18,12 +18,12 @@ import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared' | |||
18 | async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { | 18 | async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { |
19 | for (const file of files) { | 19 | for (const file of files) { |
20 | const shouldStartWith = type === 'webtorrent' | 20 | const shouldStartWith = type === 'webtorrent' |
21 | ? ObjectStorageCommand.getWebTorrentBaseUrl() | 21 | ? ObjectStorageCommand.getMockWebTorrentBaseUrl() |
22 | : ObjectStorageCommand.getPlaylistBaseUrl() | 22 | : ObjectStorageCommand.getMockPlaylistBaseUrl() |
23 | 23 | ||
24 | expectStartWith(file.fileUrl, shouldStartWith) | 24 | expectStartWith(file.fileUrl, shouldStartWith) |
25 | 25 | ||
26 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | 26 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
27 | } | 27 | } |
28 | } | 28 | } |
29 | 29 | ||
@@ -36,7 +36,7 @@ function runTests (objectStorage: boolean) { | |||
36 | this.timeout(120000) | 36 | this.timeout(120000) |
37 | 37 | ||
38 | const config = objectStorage | 38 | const config = objectStorage |
39 | ? ObjectStorageCommand.getDefaultConfig() | 39 | ? ObjectStorageCommand.getDefaultMockConfig() |
40 | : {} | 40 | : {} |
41 | 41 | ||
42 | // Run server 2 to have transcoding enabled | 42 | // Run server 2 to have transcoding enabled |
@@ -47,7 +47,7 @@ function runTests (objectStorage: boolean) { | |||
47 | 47 | ||
48 | await doubleFollow(servers[0], servers[1]) | 48 | await doubleFollow(servers[0], servers[1]) |
49 | 49 | ||
50 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | 50 | if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets() |
51 | 51 | ||
52 | for (let i = 1; i <= 5; i++) { | 52 | for (let i = 1; i <= 5; i++) { |
53 | const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) | 53 | const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) |
@@ -255,7 +255,7 @@ describe('Test create transcoding jobs', function () { | |||
255 | }) | 255 | }) |
256 | 256 | ||
257 | describe('On object storage', function () { | 257 | describe('On object storage', function () { |
258 | if (areObjectStorageTestsDisabled()) return | 258 | if (areMockObjectStorageTestsDisabled()) return |
259 | 259 | ||
260 | runTests(true) | 260 | runTests(true) |
261 | }) | 261 | }) |
diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index a89e17e3c..ba0fa1f86 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts | |||
@@ -5,7 +5,7 @@ import { createFile, readdir } from 'fs-extra' | |||
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { wait } from '@shared/core-utils' | 6 | import { wait } from '@shared/core-utils' |
7 | import { buildUUID } from '@shared/extra-utils' | 7 | import { buildUUID } from '@shared/extra-utils' |
8 | import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' | 8 | import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' |
9 | import { | 9 | import { |
10 | cleanupTests, | 10 | cleanupTests, |
11 | CLICommand, | 11 | CLICommand, |
@@ -36,22 +36,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst | |||
36 | async function assertCountAreOkay (servers: PeerTubeServer[]) { | 36 | async function assertCountAreOkay (servers: PeerTubeServer[]) { |
37 | for (const server of servers) { | 37 | for (const server of servers) { |
38 | const videosCount = await countFiles(server, 'videos') | 38 | const videosCount = await countFiles(server, 'videos') |
39 | expect(videosCount).to.equal(8) | 39 | expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory |
40 | |||
41 | const privateVideosCount = await countFiles(server, 'videos/private') | ||
42 | expect(privateVideosCount).to.equal(4) | ||
40 | 43 | ||
41 | const torrentsCount = await countFiles(server, 'torrents') | 44 | const torrentsCount = await countFiles(server, 'torrents') |
42 | expect(torrentsCount).to.equal(16) | 45 | expect(torrentsCount).to.equal(24) |
43 | 46 | ||
44 | const previewsCount = await countFiles(server, 'previews') | 47 | const previewsCount = await countFiles(server, 'previews') |
45 | expect(previewsCount).to.equal(2) | 48 | expect(previewsCount).to.equal(3) |
46 | 49 | ||
47 | const thumbnailsCount = await countFiles(server, 'thumbnails') | 50 | const thumbnailsCount = await countFiles(server, 'thumbnails') |
48 | expect(thumbnailsCount).to.equal(6) | 51 | expect(thumbnailsCount).to.equal(7) // 3 local videos, 1 local playlist, 2 remotes videos and 1 remote playlist |
49 | 52 | ||
50 | const avatarsCount = await countFiles(server, 'avatars') | 53 | const avatarsCount = await countFiles(server, 'avatars') |
51 | expect(avatarsCount).to.equal(4) | 54 | expect(avatarsCount).to.equal(4) |
52 | 55 | ||
53 | const hlsRootCount = await countFiles(server, 'streaming-playlists/hls') | 56 | const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls')) |
54 | expect(hlsRootCount).to.equal(2) | 57 | expect(hlsRootCount).to.equal(3) // 2 videos + private directory |
58 | |||
59 | const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private')) | ||
60 | expect(hlsPrivateRootCount).to.equal(1) | ||
55 | } | 61 | } |
56 | } | 62 | } |
57 | 63 | ||
@@ -67,8 +73,10 @@ describe('Test prune storage scripts', function () { | |||
67 | await setDefaultVideoChannel(servers) | 73 | await setDefaultVideoChannel(servers) |
68 | 74 | ||
69 | for (const server of servers) { | 75 | for (const server of servers) { |
70 | await server.videos.upload({ attributes: { name: 'video 1' } }) | 76 | await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } }) |
71 | await server.videos.upload({ attributes: { name: 'video 2' } }) | 77 | await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } }) |
78 | |||
79 | await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) | ||
72 | 80 | ||
73 | await server.users.updateMyAvatar({ fixture: 'avatar.png' }) | 81 | await server.users.updateMyAvatar({ fixture: 'avatar.png' }) |
74 | 82 | ||
@@ -123,13 +131,16 @@ describe('Test prune storage scripts', function () { | |||
123 | it('Should create some dirty files', async function () { | 131 | it('Should create some dirty files', async function () { |
124 | for (let i = 0; i < 2; i++) { | 132 | for (let i = 0; i < 2; i++) { |
125 | { | 133 | { |
126 | const base = servers[0].servers.buildDirectory('videos') | 134 | const basePublic = servers[0].servers.buildDirectory('videos') |
135 | const basePrivate = servers[0].servers.buildDirectory(join('videos', 'private')) | ||
127 | 136 | ||
128 | const n1 = buildUUID() + '.mp4' | 137 | const n1 = buildUUID() + '.mp4' |
129 | const n2 = buildUUID() + '.webm' | 138 | const n2 = buildUUID() + '.webm' |
130 | 139 | ||
131 | await createFile(join(base, n1)) | 140 | await createFile(join(basePublic, n1)) |
132 | await createFile(join(base, n2)) | 141 | await createFile(join(basePublic, n2)) |
142 | await createFile(join(basePrivate, n1)) | ||
143 | await createFile(join(basePrivate, n2)) | ||
133 | 144 | ||
134 | badNames['videos'] = [ n1, n2 ] | 145 | badNames['videos'] = [ n1, n2 ] |
135 | } | 146 | } |
@@ -184,10 +195,12 @@ describe('Test prune storage scripts', function () { | |||
184 | 195 | ||
185 | { | 196 | { |
186 | const directory = join('streaming-playlists', 'hls') | 197 | const directory = join('streaming-playlists', 'hls') |
187 | const base = servers[0].servers.buildDirectory(directory) | 198 | const basePublic = servers[0].servers.buildDirectory(directory) |
199 | const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private')) | ||
188 | 200 | ||
189 | const n1 = buildUUID() | 201 | const n1 = buildUUID() |
190 | await createFile(join(base, n1)) | 202 | await createFile(join(basePublic, n1)) |
203 | await createFile(join(basePrivate, n1)) | ||
191 | badNames[directory] = [ n1 ] | 204 | badNames[directory] = [ n1 ] |
192 | } | 205 | } |
193 | } | 206 | } |
diff --git a/server/tests/cli/regenerate-thumbnails.ts b/server/tests/cli/regenerate-thumbnails.ts index f459b11b8..16a8adcda 100644 --- a/server/tests/cli/regenerate-thumbnails.ts +++ b/server/tests/cli/regenerate-thumbnails.ts | |||
@@ -6,7 +6,7 @@ import { | |||
6 | cleanupTests, | 6 | cleanupTests, |
7 | createMultipleServers, | 7 | createMultipleServers, |
8 | doubleFollow, | 8 | doubleFollow, |
9 | makeRawRequest, | 9 | makeGetRequest, |
10 | PeerTubeServer, | 10 | PeerTubeServer, |
11 | setAccessTokensToServers, | 11 | setAccessTokensToServers, |
12 | waitJobs | 12 | waitJobs |
@@ -16,8 +16,8 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string) | |||
16 | const video = await server.videos.get({ id: videoId }) | 16 | const video = await server.videos.get({ id: videoId }) |
17 | 17 | ||
18 | const requests = [ | 18 | const requests = [ |
19 | makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200), | 19 | makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }), |
20 | makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200) | 20 | makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
21 | ] | 21 | ] |
22 | 22 | ||
23 | for (const req of requests) { | 23 | for (const req of requests) { |
@@ -69,17 +69,17 @@ describe('Test regenerate thumbnails script', function () { | |||
69 | 69 | ||
70 | it('Should have empty thumbnails', async function () { | 70 | it('Should have empty thumbnails', async function () { |
71 | { | 71 | { |
72 | const res = await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.OK_200) | 72 | const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
73 | expect(res.body).to.have.lengthOf(0) | 73 | expect(res.body).to.have.lengthOf(0) |
74 | } | 74 | } |
75 | 75 | ||
76 | { | 76 | { |
77 | const res = await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.OK_200) | 77 | const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
78 | expect(res.body).to.not.have.lengthOf(0) | 78 | expect(res.body).to.not.have.lengthOf(0) |
79 | } | 79 | } |
80 | 80 | ||
81 | { | 81 | { |
82 | const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) | 82 | const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
83 | expect(res.body).to.have.lengthOf(0) | 83 | expect(res.body).to.have.lengthOf(0) |
84 | } | 84 | } |
85 | }) | 85 | }) |
@@ -94,21 +94,21 @@ describe('Test regenerate thumbnails script', function () { | |||
94 | await testThumbnail(servers[0], video1.uuid) | 94 | await testThumbnail(servers[0], video1.uuid) |
95 | await testThumbnail(servers[0], video2.uuid) | 95 | await testThumbnail(servers[0], video2.uuid) |
96 | 96 | ||
97 | const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) | 97 | const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
98 | expect(res.body).to.have.lengthOf(0) | 98 | expect(res.body).to.have.lengthOf(0) |
99 | }) | 99 | }) |
100 | 100 | ||
101 | it('Should have deleted old thumbnail files', async function () { | 101 | it('Should have deleted old thumbnail files', async function () { |
102 | { | 102 | { |
103 | await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.NOT_FOUND_404) | 103 | await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
104 | } | 104 | } |
105 | 105 | ||
106 | { | 106 | { |
107 | await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.NOT_FOUND_404) | 107 | await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
108 | } | 108 | } |
109 | 109 | ||
110 | { | 110 | { |
111 | const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200) | 111 | const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) |
112 | expect(res.body).to.have.lengthOf(0) | 112 | expect(res.body).to.have.lengthOf(0) |
113 | } | 113 | } |
114 | }) | 114 | }) |
diff --git a/server/tests/external-plugins/akismet.ts b/server/tests/external-plugins/akismet.ts new file mode 100644 index 000000000..974bf0011 --- /dev/null +++ b/server/tests/external-plugins/akismet.ts | |||
@@ -0,0 +1,160 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { HttpStatusCode } from '@shared/models' | ||
5 | import { | ||
6 | cleanupTests, | ||
7 | createMultipleServers, | ||
8 | doubleFollow, | ||
9 | PeerTubeServer, | ||
10 | setAccessTokensToServers, | ||
11 | waitJobs | ||
12 | } from '@shared/server-commands' | ||
13 | |||
14 | describe('Official plugin Akismet', function () { | ||
15 | let servers: PeerTubeServer[] | ||
16 | let videoUUID: string | ||
17 | |||
18 | before(async function () { | ||
19 | this.timeout(30000) | ||
20 | |||
21 | servers = await createMultipleServers(2) | ||
22 | await setAccessTokensToServers(servers) | ||
23 | |||
24 | await servers[0].plugins.install({ | ||
25 | npmName: 'peertube-plugin-akismet' | ||
26 | }) | ||
27 | |||
28 | if (!process.env.AKISMET_KEY) throw new Error('Missing AKISMET_KEY from env') | ||
29 | |||
30 | await servers[0].plugins.updateSettings({ | ||
31 | npmName: 'peertube-plugin-akismet', | ||
32 | settings: { | ||
33 | 'akismet-api-key': process.env.AKISMET_KEY | ||
34 | } | ||
35 | }) | ||
36 | |||
37 | await doubleFollow(servers[0], servers[1]) | ||
38 | }) | ||
39 | |||
40 | describe('Local threads/replies', function () { | ||
41 | |||
42 | before(async function () { | ||
43 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | ||
44 | videoUUID = uuid | ||
45 | }) | ||
46 | |||
47 | it('Should not detect a thread as spam', async function () { | ||
48 | await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' }) | ||
49 | }) | ||
50 | |||
51 | it('Should not detect a reply as spam', async function () { | ||
52 | await servers[0].comments.addReplyToLastThread({ text: 'reply' }) | ||
53 | }) | ||
54 | |||
55 | it('Should detect a thread as spam', async function () { | ||
56 | await servers[0].comments.createThread({ | ||
57 | videoId: videoUUID, | ||
58 | text: 'akismet-guaranteed-spam', | ||
59 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
60 | }) | ||
61 | }) | ||
62 | |||
63 | it('Should detect a thread as spam', async function () { | ||
64 | await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' }) | ||
65 | await servers[0].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
66 | }) | ||
67 | }) | ||
68 | |||
69 | describe('Remote threads/replies', function () { | ||
70 | |||
71 | before(async function () { | ||
72 | this.timeout(60000) | ||
73 | |||
74 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | ||
75 | videoUUID = uuid | ||
76 | |||
77 | await waitJobs(servers) | ||
78 | }) | ||
79 | |||
80 | it('Should not detect a thread as spam', async function () { | ||
81 | this.timeout(30000) | ||
82 | |||
83 | await servers[1].comments.createThread({ videoId: videoUUID, text: 'remote comment 1' }) | ||
84 | await waitJobs(servers) | ||
85 | |||
86 | const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) | ||
87 | expect(data).to.have.lengthOf(1) | ||
88 | }) | ||
89 | |||
90 | it('Should not detect a reply as spam', async function () { | ||
91 | this.timeout(30000) | ||
92 | |||
93 | await servers[1].comments.addReplyToLastThread({ text: 'I agree with you' }) | ||
94 | await waitJobs(servers) | ||
95 | |||
96 | const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) | ||
97 | expect(data).to.have.lengthOf(1) | ||
98 | |||
99 | const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: data[0].id }) | ||
100 | expect(tree.children).to.have.lengthOf(1) | ||
101 | }) | ||
102 | |||
103 | it('Should detect a thread as spam', async function () { | ||
104 | this.timeout(30000) | ||
105 | |||
106 | await servers[1].comments.createThread({ videoId: videoUUID, text: 'akismet-guaranteed-spam' }) | ||
107 | await waitJobs(servers) | ||
108 | |||
109 | const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) | ||
110 | expect(data).to.have.lengthOf(1) | ||
111 | }) | ||
112 | |||
113 | it('Should detect a thread as spam', async function () { | ||
114 | this.timeout(30000) | ||
115 | |||
116 | await servers[1].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam' }) | ||
117 | await waitJobs(servers) | ||
118 | |||
119 | const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) | ||
120 | expect(data).to.have.lengthOf(1) | ||
121 | |||
122 | const thread = data[0] | ||
123 | const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: thread.id }) | ||
124 | expect(tree.children).to.have.lengthOf(1) | ||
125 | }) | ||
126 | }) | ||
127 | |||
128 | describe('Signup', function () { | ||
129 | |||
130 | before(async function () { | ||
131 | await servers[0].config.updateExistingSubConfig({ | ||
132 | newConfig: { | ||
133 | signup: { | ||
134 | enabled: true | ||
135 | } | ||
136 | } | ||
137 | }) | ||
138 | }) | ||
139 | |||
140 | it('Should allow signup', async function () { | ||
141 | await servers[0].users.register({ | ||
142 | username: 'user1', | ||
143 | displayName: 'user 1' | ||
144 | }) | ||
145 | }) | ||
146 | |||
147 | it('Should detect a signup as SPAM', async function () { | ||
148 | await servers[0].users.register({ | ||
149 | username: 'user2', | ||
150 | displayName: 'user 2', | ||
151 | email: 'akismet-guaranteed-spam@example.com', | ||
152 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
153 | }) | ||
154 | }) | ||
155 | }) | ||
156 | |||
157 | after(async function () { | ||
158 | await cleanupTests(servers) | ||
159 | }) | ||
160 | }) | ||
diff --git a/server/tests/external-plugins/auth-ldap.ts b/server/tests/external-plugins/auth-ldap.ts index d7f155d2a..6f6a574a0 100644 --- a/server/tests/external-plugins/auth-ldap.ts +++ b/server/tests/external-plugins/auth-ldap.ts | |||
@@ -94,6 +94,14 @@ describe('Official plugin auth-ldap', function () { | |||
94 | await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } }) | 94 | await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } }) |
95 | }) | 95 | }) |
96 | 96 | ||
97 | it('Should not be able to ask password reset', async function () { | ||
98 | await server.users.askResetPassword({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 }) | ||
99 | }) | ||
100 | |||
101 | it('Should not be able to ask email verification', async function () { | ||
102 | await server.users.askSendVerifyEmail({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 }) | ||
103 | }) | ||
104 | |||
97 | it('Should not login if the plugin is uninstalled', async function () { | 105 | it('Should not login if the plugin is uninstalled', async function () { |
98 | await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' }) | 106 | await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' }) |
99 | 107 | ||
diff --git a/server/tests/external-plugins/index.ts b/server/tests/external-plugins/index.ts index 31d818b51..815bbf1da 100644 --- a/server/tests/external-plugins/index.ts +++ b/server/tests/external-plugins/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import './akismet' | ||
1 | import './auth-ldap' | 2 | import './auth-ldap' |
2 | import './auto-block-videos' | 3 | import './auto-block-videos' |
3 | import './auto-mute' | 4 | import './auto-mute' |
diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts index 1d3c03d67..906dab1a3 100644 --- a/server/tests/feeds/feeds.ts +++ b/server/tests/feeds/feeds.ts | |||
@@ -9,6 +9,7 @@ import { | |||
9 | createSingleServer, | 9 | createSingleServer, |
10 | doubleFollow, | 10 | doubleFollow, |
11 | makeGetRequest, | 11 | makeGetRequest, |
12 | makeRawRequest, | ||
12 | PeerTubeServer, | 13 | PeerTubeServer, |
13 | setAccessTokensToServers, | 14 | setAccessTokensToServers, |
14 | setDefaultChannelAvatar, | 15 | setDefaultChannelAvatar, |
@@ -306,6 +307,15 @@ describe('Test syndication feeds', () => { | |||
306 | 307 | ||
307 | await stopFfmpeg(ffmpeg) | 308 | await stopFfmpeg(ffmpeg) |
308 | }) | 309 | }) |
310 | |||
311 | it('Should have the channel avatar as feed icon', async function () { | ||
312 | const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: rootChannelId }, ignoreCache: true }) | ||
313 | |||
314 | const jsonObj = JSON.parse(json) | ||
315 | const imageUrl = jsonObj.icon | ||
316 | expect(imageUrl).to.include('/lazy-static/avatars/') | ||
317 | await makeRawRequest({ url: imageUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
318 | }) | ||
309 | }) | 319 | }) |
310 | 320 | ||
311 | describe('Video comments feed', function () { | 321 | describe('Video comments feed', function () { |
diff --git a/server/tests/fixtures/peertube-plugin-test-four/main.js b/server/tests/fixtures/peertube-plugin-test-four/main.js index 5194e3e02..3e848c49e 100644 --- a/server/tests/fixtures/peertube-plugin-test-four/main.js +++ b/server/tests/fixtures/peertube-plugin-test-four/main.js | |||
@@ -128,6 +128,22 @@ async function register ({ | |||
128 | 128 | ||
129 | return res.json(result) | 129 | return res.json(result) |
130 | }) | 130 | }) |
131 | |||
132 | router.post('/send-notification', async (req, res) => { | ||
133 | peertubeHelpers.socket.sendNotification(req.body.userId, { | ||
134 | type: 1, | ||
135 | userId: req.body.userId | ||
136 | }) | ||
137 | |||
138 | return res.sendStatus(201) | ||
139 | }) | ||
140 | |||
141 | router.post('/send-video-live-new-state/:uuid', async (req, res) => { | ||
142 | const video = await peertubeHelpers.videos.loadByIdOrUUID(req.params.uuid) | ||
143 | peertubeHelpers.socket.sendVideoLiveNewState(video) | ||
144 | |||
145 | return res.sendStatus(201) | ||
146 | }) | ||
131 | } | 147 | } |
132 | 148 | ||
133 | } | 149 | } |
diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/main.js b/server/tests/fixtures/peertube-plugin-test-websocket/main.js new file mode 100644 index 000000000..3fde76cfe --- /dev/null +++ b/server/tests/fixtures/peertube-plugin-test-websocket/main.js | |||
@@ -0,0 +1,36 @@ | |||
1 | const WebSocketServer = require('ws').WebSocketServer | ||
2 | |||
3 | async function register ({ | ||
4 | registerWebSocketRoute | ||
5 | }) { | ||
6 | const wss = new WebSocketServer({ noServer: true }) | ||
7 | |||
8 | wss.on('connection', function connection(ws) { | ||
9 | ws.on('message', function message(data) { | ||
10 | if (data.toString() === 'ping') { | ||
11 | ws.send('pong') | ||
12 | } | ||
13 | }) | ||
14 | }) | ||
15 | |||
16 | registerWebSocketRoute({ | ||
17 | route: '/toto', | ||
18 | |||
19 | handler: (request, socket, head) => { | ||
20 | wss.handleUpgrade(request, socket, head, ws => { | ||
21 | wss.emit('connection', ws, request) | ||
22 | }) | ||
23 | } | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | async function unregister () { | ||
28 | return | ||
29 | } | ||
30 | |||
31 | module.exports = { | ||
32 | register, | ||
33 | unregister | ||
34 | } | ||
35 | |||
36 | // ########################################################################### | ||
diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/package.json b/server/tests/fixtures/peertube-plugin-test-websocket/package.json new file mode 100644 index 000000000..89c8baa04 --- /dev/null +++ b/server/tests/fixtures/peertube-plugin-test-websocket/package.json | |||
@@ -0,0 +1,20 @@ | |||
1 | { | ||
2 | "name": "peertube-plugin-test-websocket", | ||
3 | "version": "0.0.1", | ||
4 | "description": "Plugin test websocket", | ||
5 | "engine": { | ||
6 | "peertube": ">=1.3.0" | ||
7 | }, | ||
8 | "keywords": [ | ||
9 | "peertube", | ||
10 | "plugin" | ||
11 | ], | ||
12 | "homepage": "https://github.com/Chocobozzz/PeerTube", | ||
13 | "author": "Chocobozzz", | ||
14 | "bugs": "https://github.com/Chocobozzz/PeerTube/issues", | ||
15 | "library": "./main.js", | ||
16 | "staticDirs": {}, | ||
17 | "css": [], | ||
18 | "clientScripts": [], | ||
19 | "translations": {} | ||
20 | } | ||
diff --git a/server/tests/fixtures/peertube-plugin-test/main.js b/server/tests/fixtures/peertube-plugin-test/main.js index 813482a27..19dccf26e 100644 --- a/server/tests/fixtures/peertube-plugin-test/main.js +++ b/server/tests/fixtures/peertube-plugin-test/main.js | |||
@@ -178,6 +178,8 @@ async function register ({ registerHook, registerSetting, settingsManager, stora | |||
178 | } | 178 | } |
179 | }) | 179 | }) |
180 | 180 | ||
181 | // --------------------------------------------------------------------------- | ||
182 | |||
181 | registerHook({ | 183 | registerHook({ |
182 | target: 'filter:api.video-thread.create.accept.result', | 184 | target: 'filter:api.video-thread.create.accept.result', |
183 | handler: ({ accepted }, { commentBody }) => checkCommentBadWord(accepted, commentBody) | 185 | handler: ({ accepted }, { commentBody }) => checkCommentBadWord(accepted, commentBody) |
@@ -189,6 +191,13 @@ async function register ({ registerHook, registerSetting, settingsManager, stora | |||
189 | }) | 191 | }) |
190 | 192 | ||
191 | registerHook({ | 193 | registerHook({ |
194 | target: 'filter:activity-pub.remote-video-comment.create.accept.result', | ||
195 | handler: ({ accepted }, { comment }) => checkCommentBadWord(accepted, comment) | ||
196 | }) | ||
197 | |||
198 | // --------------------------------------------------------------------------- | ||
199 | |||
200 | registerHook({ | ||
192 | target: 'filter:api.video-threads.list.params', | 201 | target: 'filter:api.video-threads.list.params', |
193 | handler: obj => addToCount(obj) | 202 | handler: obj => addToCount(obj) |
194 | }) | 203 | }) |
diff --git a/server/tests/helpers/crypto.ts b/server/tests/helpers/crypto.ts new file mode 100644 index 000000000..b508c715b --- /dev/null +++ b/server/tests/helpers/crypto.ts | |||
@@ -0,0 +1,33 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { decrypt, encrypt } from '@server/helpers/peertube-crypto' | ||
5 | |||
6 | describe('Encrypt/Descrypt', function () { | ||
7 | |||
8 | it('Should encrypt and decrypt the string', async function () { | ||
9 | const secret = 'my_secret' | ||
10 | const str = 'my super string' | ||
11 | |||
12 | const encrypted = await encrypt(str, secret) | ||
13 | const decrypted = await decrypt(encrypted, secret) | ||
14 | |||
15 | expect(str).to.equal(decrypted) | ||
16 | }) | ||
17 | |||
18 | it('Should not decrypt without the same secret', async function () { | ||
19 | const str = 'my super string' | ||
20 | |||
21 | const encrypted = await encrypt(str, 'my_secret') | ||
22 | |||
23 | let error = false | ||
24 | |||
25 | try { | ||
26 | await decrypt(encrypted, 'my_sicret') | ||
27 | } catch (err) { | ||
28 | error = true | ||
29 | } | ||
30 | |||
31 | expect(error).to.be.true | ||
32 | }) | ||
33 | }) | ||
diff --git a/server/tests/helpers/index.ts b/server/tests/helpers/index.ts index 951208842..1f0e3098a 100644 --- a/server/tests/helpers/index.ts +++ b/server/tests/helpers/index.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | import './image' | 1 | import './comment-model' |
2 | import './core-utils' | 2 | import './core-utils' |
3 | import './crypto' | ||
3 | import './dns' | 4 | import './dns' |
4 | import './comment-model' | 5 | import './image' |
5 | import './markdown' | 6 | import './markdown' |
6 | import './request' | 7 | import './request' |
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts index 663ac044a..d2072342e 100644 --- a/server/tests/misc-endpoints.ts +++ b/server/tests/misc-endpoints.ts | |||
@@ -1,18 +1,24 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' | 4 | import { writeJson } from 'fs-extra' |
5 | import { join } from 'path' | ||
5 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | 6 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' |
7 | import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' | ||
6 | import { expectLogDoesNotContain } from './shared' | 8 | import { expectLogDoesNotContain } from './shared' |
7 | 9 | ||
8 | describe('Test misc endpoints', function () { | 10 | describe('Test misc endpoints', function () { |
9 | let server: PeerTubeServer | 11 | let server: PeerTubeServer |
12 | let wellKnownPath: string | ||
10 | 13 | ||
11 | before(async function () { | 14 | before(async function () { |
12 | this.timeout(120000) | 15 | this.timeout(120000) |
13 | 16 | ||
14 | server = await createSingleServer(1) | 17 | server = await createSingleServer(1) |
18 | |||
15 | await setAccessTokensToServers([ server ]) | 19 | await setAccessTokensToServers([ server ]) |
20 | |||
21 | wellKnownPath = server.getDirectoryPath('well-known') | ||
16 | }) | 22 | }) |
17 | 23 | ||
18 | describe('Test a well known endpoints', function () { | 24 | describe('Test a well known endpoints', function () { |
@@ -93,6 +99,28 @@ describe('Test misc endpoints', function () { | |||
93 | expect(remoteInteract).to.exist | 99 | expect(remoteInteract).to.exist |
94 | expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') | 100 | expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') |
95 | }) | 101 | }) |
102 | |||
103 | it('Should return 404 for non-existing files in /.well-known', async function () { | ||
104 | await makeGetRequest({ | ||
105 | url: server.url, | ||
106 | path: '/.well-known/non-existing-file', | ||
107 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
108 | }) | ||
109 | }) | ||
110 | |||
111 | it('Should return custom file from /.well-known', async function () { | ||
112 | const filename = 'existing-file.json' | ||
113 | |||
114 | await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' }) | ||
115 | |||
116 | const { body } = await makeGetRequest({ | ||
117 | url: server.url, | ||
118 | path: '/.well-known/' + filename, | ||
119 | expectedStatus: HttpStatusCode.OK_200 | ||
120 | }) | ||
121 | |||
122 | expect(body.iThink).to.equal('therefore I am') | ||
123 | }) | ||
96 | }) | 124 | }) |
97 | 125 | ||
98 | describe('Test classic static endpoints', function () { | 126 | describe('Test classic static endpoints', function () { |
diff --git a/server/tests/plugins/action-hooks.ts b/server/tests/plugins/action-hooks.ts index af65166d9..36f8052c0 100644 --- a/server/tests/plugins/action-hooks.ts +++ b/server/tests/plugins/action-hooks.ts | |||
@@ -21,7 +21,7 @@ describe('Test plugin action hooks', function () { | |||
21 | } | 21 | } |
22 | 22 | ||
23 | before(async function () { | 23 | before(async function () { |
24 | this.timeout(30000) | 24 | this.timeout(120000) |
25 | 25 | ||
26 | servers = await createMultipleServers(2) | 26 | servers = await createMultipleServers(2) |
27 | await setAccessTokensToServers(servers) | 27 | await setAccessTokensToServers(servers) |
diff --git a/server/tests/plugins/external-auth.ts b/server/tests/plugins/external-auth.ts index e08b83791..437777e90 100644 --- a/server/tests/plugins/external-auth.ts +++ b/server/tests/plugins/external-auth.ts | |||
@@ -155,7 +155,7 @@ describe('Test external auth plugins', function () { | |||
155 | expect(body.username).to.equal('cyan') | 155 | expect(body.username).to.equal('cyan') |
156 | expect(body.account.displayName).to.equal('cyan') | 156 | expect(body.account.displayName).to.equal('cyan') |
157 | expect(body.email).to.equal('cyan@example.com') | 157 | expect(body.email).to.equal('cyan@example.com') |
158 | expect(body.role).to.equal(UserRole.USER) | 158 | expect(body.role.id).to.equal(UserRole.USER) |
159 | } | 159 | } |
160 | }) | 160 | }) |
161 | 161 | ||
@@ -177,7 +177,7 @@ describe('Test external auth plugins', function () { | |||
177 | expect(body.username).to.equal('kefka') | 177 | expect(body.username).to.equal('kefka') |
178 | expect(body.account.displayName).to.equal('Kefka Palazzo') | 178 | expect(body.account.displayName).to.equal('Kefka Palazzo') |
179 | expect(body.email).to.equal('kefka@example.com') | 179 | expect(body.email).to.equal('kefka@example.com') |
180 | expect(body.role).to.equal(UserRole.ADMINISTRATOR) | 180 | expect(body.role.id).to.equal(UserRole.ADMINISTRATOR) |
181 | } | 181 | } |
182 | }) | 182 | }) |
183 | 183 | ||
@@ -237,7 +237,7 @@ describe('Test external auth plugins', function () { | |||
237 | expect(body.username).to.equal('cyan') | 237 | expect(body.username).to.equal('cyan') |
238 | expect(body.account.displayName).to.equal('Cyan Garamonde') | 238 | expect(body.account.displayName).to.equal('Cyan Garamonde') |
239 | expect(body.account.description).to.equal('Retainer to the king of Doma') | 239 | expect(body.account.description).to.equal('Retainer to the king of Doma') |
240 | expect(body.role).to.equal(UserRole.USER) | 240 | expect(body.role.id).to.equal(UserRole.USER) |
241 | }) | 241 | }) |
242 | 242 | ||
243 | it('Should not update an external auth email', async function () { | 243 | it('Should not update an external auth email', async function () { |
diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts index 026c7e856..083fd43ca 100644 --- a/server/tests/plugins/filter-hooks.ts +++ b/server/tests/plugins/filter-hooks.ts | |||
@@ -6,6 +6,7 @@ import { | |||
6 | cleanupTests, | 6 | cleanupTests, |
7 | createMultipleServers, | 7 | createMultipleServers, |
8 | doubleFollow, | 8 | doubleFollow, |
9 | makeGetRequest, | ||
9 | makeRawRequest, | 10 | makeRawRequest, |
10 | PeerTubeServer, | 11 | PeerTubeServer, |
11 | PluginsCommand, | 12 | PluginsCommand, |
@@ -64,232 +65,289 @@ describe('Test plugin filter hooks', function () { | |||
64 | }) | 65 | }) |
65 | }) | 66 | }) |
66 | 67 | ||
67 | it('Should run filter:api.videos.list.params', async function () { | 68 | describe('Videos', function () { |
68 | const { data } = await servers[0].videos.list({ start: 0, count: 2 }) | ||
69 | 69 | ||
70 | // 2 plugins do +1 to the count parameter | 70 | it('Should run filter:api.videos.list.params', async function () { |
71 | expect(data).to.have.lengthOf(4) | 71 | const { data } = await servers[0].videos.list({ start: 0, count: 2 }) |
72 | }) | ||
73 | 72 | ||
74 | it('Should run filter:api.videos.list.result', async function () { | 73 | // 2 plugins do +1 to the count parameter |
75 | const { total } = await servers[0].videos.list({ start: 0, count: 0 }) | 74 | expect(data).to.have.lengthOf(4) |
75 | }) | ||
76 | 76 | ||
77 | // Plugin do +1 to the total result | 77 | it('Should run filter:api.videos.list.result', async function () { |
78 | expect(total).to.equal(11) | 78 | const { total } = await servers[0].videos.list({ start: 0, count: 0 }) |
79 | }) | ||
80 | 79 | ||
81 | it('Should run filter:api.video-playlist.videos.list.params', async function () { | 80 | // Plugin do +1 to the total result |
82 | const { data } = await servers[0].playlists.listVideos({ | 81 | expect(total).to.equal(11) |
83 | count: 2, | ||
84 | playlistId: videoPlaylistUUID | ||
85 | }) | 82 | }) |
86 | 83 | ||
87 | // 1 plugin do +1 to the count parameter | 84 | it('Should run filter:api.video-playlist.videos.list.params', async function () { |
88 | expect(data).to.have.lengthOf(3) | 85 | const { data } = await servers[0].playlists.listVideos({ |
89 | }) | 86 | count: 2, |
87 | playlistId: videoPlaylistUUID | ||
88 | }) | ||
90 | 89 | ||
91 | it('Should run filter:api.video-playlist.videos.list.result', async function () { | 90 | // 1 plugin do +1 to the count parameter |
92 | const { total } = await servers[0].playlists.listVideos({ | 91 | expect(data).to.have.lengthOf(3) |
93 | count: 0, | ||
94 | playlistId: videoPlaylistUUID | ||
95 | }) | 92 | }) |
96 | 93 | ||
97 | // Plugin do +1 to the total result | 94 | it('Should run filter:api.video-playlist.videos.list.result', async function () { |
98 | expect(total).to.equal(11) | 95 | const { total } = await servers[0].playlists.listVideos({ |
99 | }) | 96 | count: 0, |
97 | playlistId: videoPlaylistUUID | ||
98 | }) | ||
100 | 99 | ||
101 | it('Should run filter:api.accounts.videos.list.params', async function () { | 100 | // Plugin do +1 to the total result |
102 | const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) | 101 | expect(total).to.equal(11) |
102 | }) | ||
103 | 103 | ||
104 | // 1 plugin do +1 to the count parameter | 104 | it('Should run filter:api.accounts.videos.list.params', async function () { |
105 | expect(data).to.have.lengthOf(3) | 105 | const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) |
106 | }) | ||
107 | 106 | ||
108 | it('Should run filter:api.accounts.videos.list.result', async function () { | 107 | // 1 plugin do +1 to the count parameter |
109 | const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) | 108 | expect(data).to.have.lengthOf(3) |
109 | }) | ||
110 | 110 | ||
111 | // Plugin do +2 to the total result | 111 | it('Should run filter:api.accounts.videos.list.result', async function () { |
112 | expect(total).to.equal(12) | 112 | const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) |
113 | }) | ||
114 | 113 | ||
115 | it('Should run filter:api.video-channels.videos.list.params', async function () { | 114 | // Plugin do +2 to the total result |
116 | const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) | 115 | expect(total).to.equal(12) |
116 | }) | ||
117 | 117 | ||
118 | // 1 plugin do +3 to the count parameter | 118 | it('Should run filter:api.video-channels.videos.list.params', async function () { |
119 | expect(data).to.have.lengthOf(5) | 119 | const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) |
120 | }) | ||
121 | 120 | ||
122 | it('Should run filter:api.video-channels.videos.list.result', async function () { | 121 | // 1 plugin do +3 to the count parameter |
123 | const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) | 122 | expect(data).to.have.lengthOf(5) |
123 | }) | ||
124 | 124 | ||
125 | // Plugin do +3 to the total result | 125 | it('Should run filter:api.video-channels.videos.list.result', async function () { |
126 | expect(total).to.equal(13) | 126 | const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) |
127 | }) | ||
128 | 127 | ||
129 | it('Should run filter:api.user.me.videos.list.params', async function () { | 128 | // Plugin do +3 to the total result |
130 | const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) | 129 | expect(total).to.equal(13) |
130 | }) | ||
131 | 131 | ||
132 | // 1 plugin do +4 to the count parameter | 132 | it('Should run filter:api.user.me.videos.list.params', async function () { |
133 | expect(data).to.have.lengthOf(6) | 133 | const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) |
134 | }) | ||
135 | 134 | ||
136 | it('Should run filter:api.user.me.videos.list.result', async function () { | 135 | // 1 plugin do +4 to the count parameter |
137 | const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) | 136 | expect(data).to.have.lengthOf(6) |
137 | }) | ||
138 | 138 | ||
139 | // Plugin do +4 to the total result | 139 | it('Should run filter:api.user.me.videos.list.result', async function () { |
140 | expect(total).to.equal(14) | 140 | const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) |
141 | }) | ||
142 | 141 | ||
143 | it('Should run filter:api.video.get.result', async function () { | 142 | // Plugin do +4 to the total result |
144 | const video = await servers[0].videos.get({ id: videoUUID }) | 143 | expect(total).to.equal(14) |
145 | expect(video.name).to.contain('<3') | 144 | }) |
146 | }) | ||
147 | 145 | ||
148 | it('Should run filter:api.video.upload.accept.result', async function () { | 146 | it('Should run filter:api.video.get.result', async function () { |
149 | await servers[0].videos.upload({ attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 147 | const video = await servers[0].videos.get({ id: videoUUID }) |
148 | expect(video.name).to.contain('<3') | ||
149 | }) | ||
150 | }) | 150 | }) |
151 | 151 | ||
152 | it('Should run filter:api.live-video.create.accept.result', async function () { | 152 | describe('Video/live/import accept', function () { |
153 | const attributes = { | ||
154 | name: 'video with bad word', | ||
155 | privacy: VideoPrivacy.PUBLIC, | ||
156 | channelId: servers[0].store.channel.id | ||
157 | } | ||
158 | 153 | ||
159 | await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 154 | it('Should run filter:api.video.upload.accept.result', async function () { |
160 | }) | 155 | await servers[0].videos.upload({ attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
161 | 156 | }) | |
162 | it('Should run filter:api.video.pre-import-url.accept.result', async function () { | ||
163 | const attributes = { | ||
164 | name: 'normal title', | ||
165 | privacy: VideoPrivacy.PUBLIC, | ||
166 | channelId: servers[0].store.channel.id, | ||
167 | targetUrl: FIXTURE_URLS.goodVideo + 'bad' | ||
168 | } | ||
169 | await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
170 | }) | ||
171 | 157 | ||
172 | it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { | 158 | it('Should run filter:api.live-video.create.accept.result', async function () { |
173 | const attributes = { | 159 | const attributes = { |
174 | name: 'bad torrent', | 160 | name: 'video with bad word', |
175 | privacy: VideoPrivacy.PUBLIC, | 161 | privacy: VideoPrivacy.PUBLIC, |
176 | channelId: servers[0].store.channel.id, | 162 | channelId: servers[0].store.channel.id |
177 | torrentfile: 'video-720p.torrent' as any | 163 | } |
178 | } | ||
179 | await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
180 | }) | ||
181 | 164 | ||
182 | it('Should run filter:api.video.post-import-url.accept.result', async function () { | 165 | await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
183 | this.timeout(60000) | 166 | }) |
184 | 167 | ||
185 | let videoImportId: number | 168 | it('Should run filter:api.video.pre-import-url.accept.result', async function () { |
169 | const attributes = { | ||
170 | name: 'normal title', | ||
171 | privacy: VideoPrivacy.PUBLIC, | ||
172 | channelId: servers[0].store.channel.id, | ||
173 | targetUrl: FIXTURE_URLS.goodVideo + 'bad' | ||
174 | } | ||
175 | await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
176 | }) | ||
186 | 177 | ||
187 | { | 178 | it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { |
188 | const attributes = { | 179 | const attributes = { |
189 | name: 'title with bad word', | 180 | name: 'bad torrent', |
190 | privacy: VideoPrivacy.PUBLIC, | 181 | privacy: VideoPrivacy.PUBLIC, |
191 | channelId: servers[0].store.channel.id, | 182 | channelId: servers[0].store.channel.id, |
192 | targetUrl: FIXTURE_URLS.goodVideo | 183 | torrentfile: 'video-720p.torrent' as any |
193 | } | 184 | } |
194 | const body = await servers[0].imports.importVideo({ attributes }) | 185 | await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
195 | videoImportId = body.id | 186 | }) |
196 | } | ||
197 | 187 | ||
198 | await waitJobs(servers) | 188 | it('Should run filter:api.video.post-import-url.accept.result', async function () { |
189 | this.timeout(60000) | ||
199 | 190 | ||
200 | { | 191 | let videoImportId: number |
201 | const body = await servers[0].imports.getMyVideoImports() | ||
202 | const videoImports = body.data | ||
203 | 192 | ||
204 | const videoImport = videoImports.find(i => i.id === videoImportId) | 193 | { |
194 | const attributes = { | ||
195 | name: 'title with bad word', | ||
196 | privacy: VideoPrivacy.PUBLIC, | ||
197 | channelId: servers[0].store.channel.id, | ||
198 | targetUrl: FIXTURE_URLS.goodVideo | ||
199 | } | ||
200 | const body = await servers[0].imports.importVideo({ attributes }) | ||
201 | videoImportId = body.id | ||
202 | } | ||
205 | 203 | ||
206 | expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) | 204 | await waitJobs(servers) |
207 | expect(videoImport.state.label).to.equal('Rejected') | ||
208 | } | ||
209 | }) | ||
210 | 205 | ||
211 | it('Should run filter:api.video.post-import-torrent.accept.result', async function () { | 206 | { |
212 | this.timeout(60000) | 207 | const body = await servers[0].imports.getMyVideoImports() |
208 | const videoImports = body.data | ||
213 | 209 | ||
214 | let videoImportId: number | 210 | const videoImport = videoImports.find(i => i.id === videoImportId) |
215 | 211 | ||
216 | { | 212 | expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) |
217 | const attributes = { | 213 | expect(videoImport.state.label).to.equal('Rejected') |
218 | name: 'title with bad word', | ||
219 | privacy: VideoPrivacy.PUBLIC, | ||
220 | channelId: servers[0].store.channel.id, | ||
221 | torrentfile: 'video-720p.torrent' as any | ||
222 | } | 214 | } |
223 | const body = await servers[0].imports.importVideo({ attributes }) | 215 | }) |
224 | videoImportId = body.id | 216 | |
225 | } | 217 | it('Should run filter:api.video.post-import-torrent.accept.result', async function () { |
218 | this.timeout(60000) | ||
226 | 219 | ||
227 | await waitJobs(servers) | 220 | let videoImportId: number |
228 | 221 | ||
229 | { | 222 | { |
230 | const { data: videoImports } = await servers[0].imports.getMyVideoImports() | 223 | const attributes = { |
224 | name: 'title with bad word', | ||
225 | privacy: VideoPrivacy.PUBLIC, | ||
226 | channelId: servers[0].store.channel.id, | ||
227 | torrentfile: 'video-720p.torrent' as any | ||
228 | } | ||
229 | const body = await servers[0].imports.importVideo({ attributes }) | ||
230 | videoImportId = body.id | ||
231 | } | ||
231 | 232 | ||
232 | const videoImport = videoImports.find(i => i.id === videoImportId) | 233 | await waitJobs(servers) |
233 | 234 | ||
234 | expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) | 235 | { |
235 | expect(videoImport.state.label).to.equal('Rejected') | 236 | const { data: videoImports } = await servers[0].imports.getMyVideoImports() |
236 | } | 237 | |
237 | }) | 238 | const videoImport = videoImports.find(i => i.id === videoImportId) |
238 | 239 | ||
239 | it('Should run filter:api.video-thread.create.accept.result', async function () { | 240 | expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) |
240 | await servers[0].comments.createThread({ | 241 | expect(videoImport.state.label).to.equal('Rejected') |
241 | videoId: videoUUID, | 242 | } |
242 | text: 'comment with bad word', | ||
243 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
244 | }) | 243 | }) |
245 | }) | 244 | }) |
246 | 245 | ||
247 | it('Should run filter:api.video-comment-reply.create.accept.result', async function () { | 246 | describe('Video comments accept', function () { |
248 | const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' }) | ||
249 | threadId = created.id | ||
250 | 247 | ||
251 | await servers[0].comments.addReply({ | 248 | it('Should run filter:api.video-thread.create.accept.result', async function () { |
252 | videoId: videoUUID, | 249 | await servers[0].comments.createThread({ |
253 | toCommentId: threadId, | 250 | videoId: videoUUID, |
254 | text: 'comment with bad word', | 251 | text: 'comment with bad word', |
255 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | 252 | expectedStatus: HttpStatusCode.FORBIDDEN_403 |
253 | }) | ||
256 | }) | 254 | }) |
257 | await servers[0].comments.addReply({ | 255 | |
258 | videoId: videoUUID, | 256 | it('Should run filter:api.video-comment-reply.create.accept.result', async function () { |
259 | toCommentId: threadId, | 257 | const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' }) |
260 | text: 'comment with good word', | 258 | threadId = created.id |
261 | expectedStatus: HttpStatusCode.OK_200 | 259 | |
260 | await servers[0].comments.addReply({ | ||
261 | videoId: videoUUID, | ||
262 | toCommentId: threadId, | ||
263 | text: 'comment with bad word', | ||
264 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
265 | }) | ||
266 | await servers[0].comments.addReply({ | ||
267 | videoId: videoUUID, | ||
268 | toCommentId: threadId, | ||
269 | text: 'comment with good word', | ||
270 | expectedStatus: HttpStatusCode.OK_200 | ||
271 | }) | ||
262 | }) | 272 | }) |
263 | }) | ||
264 | 273 | ||
265 | it('Should run filter:api.video-threads.list.params', async function () { | 274 | it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a thread creation', async function () { |
266 | const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) | 275 | this.timeout(30000) |
267 | 276 | ||
268 | // our plugin do +1 to the count parameter | 277 | await servers[1].comments.createThread({ videoId: videoUUID, text: 'comment with bad word' }) |
269 | expect(data).to.have.lengthOf(1) | ||
270 | }) | ||
271 | 278 | ||
272 | it('Should run filter:api.video-threads.list.result', async function () { | 279 | await waitJobs(servers) |
273 | const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) | ||
274 | 280 | ||
275 | // Plugin do +1 to the total result | 281 | { |
276 | expect(total).to.equal(2) | 282 | const thread = await servers[0].comments.listThreads({ videoId: videoUUID }) |
277 | }) | 283 | expect(thread.data).to.have.lengthOf(1) |
284 | expect(thread.data[0].text).to.not.include(' bad ') | ||
285 | } | ||
278 | 286 | ||
279 | it('Should run filter:api.video-thread-comments.list.params') | 287 | { |
288 | const thread = await servers[1].comments.listThreads({ videoId: videoUUID }) | ||
289 | expect(thread.data).to.have.lengthOf(2) | ||
290 | } | ||
291 | }) | ||
292 | |||
293 | it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a reply creation', async function () { | ||
294 | this.timeout(30000) | ||
280 | 295 | ||
281 | it('Should run filter:api.video-thread-comments.list.result', async function () { | 296 | const { data } = await servers[1].comments.listThreads({ videoId: videoUUID }) |
282 | const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) | 297 | const threadIdServer2 = data.find(t => t.text === 'thread').id |
283 | 298 | ||
284 | expect(thread.comment.text.endsWith(' <3')).to.be.true | 299 | await servers[1].comments.addReply({ |
300 | videoId: videoUUID, | ||
301 | toCommentId: threadIdServer2, | ||
302 | text: 'comment with bad word' | ||
303 | }) | ||
304 | |||
305 | await waitJobs(servers) | ||
306 | |||
307 | { | ||
308 | const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) | ||
309 | expect(tree.children).to.have.lengthOf(1) | ||
310 | expect(tree.children[0].comment.text).to.not.include(' bad ') | ||
311 | } | ||
312 | |||
313 | { | ||
314 | const tree = await servers[1].comments.getThread({ videoId: videoUUID, threadId: threadIdServer2 }) | ||
315 | expect(tree.children).to.have.lengthOf(2) | ||
316 | } | ||
317 | }) | ||
285 | }) | 318 | }) |
286 | 319 | ||
287 | it('Should run filter:api.overviews.videos.list.{params,result}', async function () { | 320 | describe('Video comments', function () { |
288 | await servers[0].overviews.getVideos({ page: 1 }) | ||
289 | 321 | ||
290 | // 3 because we get 3 samples per page | 322 | it('Should run filter:api.video-threads.list.params', async function () { |
291 | await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3) | 323 | const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) |
292 | await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3) | 324 | |
325 | // our plugin do +1 to the count parameter | ||
326 | expect(data).to.have.lengthOf(1) | ||
327 | }) | ||
328 | |||
329 | it('Should run filter:api.video-threads.list.result', async function () { | ||
330 | const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) | ||
331 | |||
332 | // Plugin do +1 to the total result | ||
333 | expect(total).to.equal(2) | ||
334 | }) | ||
335 | |||
336 | it('Should run filter:api.video-thread-comments.list.params') | ||
337 | |||
338 | it('Should run filter:api.video-thread-comments.list.result', async function () { | ||
339 | const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) | ||
340 | |||
341 | expect(thread.comment.text.endsWith(' <3')).to.be.true | ||
342 | }) | ||
343 | |||
344 | it('Should run filter:api.overviews.videos.list.{params,result}', async function () { | ||
345 | await servers[0].overviews.getVideos({ page: 1 }) | ||
346 | |||
347 | // 3 because we get 3 samples per page | ||
348 | await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3) | ||
349 | await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3) | ||
350 | }) | ||
293 | }) | 351 | }) |
294 | 352 | ||
295 | describe('filter:video.auto-blacklist.result', function () { | 353 | describe('filter:video.auto-blacklist.result', function () { |
@@ -404,30 +462,41 @@ describe('Test plugin filter hooks', function () { | |||
404 | }) | 462 | }) |
405 | 463 | ||
406 | it('Should run filter:api.download.torrent.allowed.result', async function () { | 464 | it('Should run filter:api.download.torrent.allowed.result', async function () { |
407 | const res = await makeRawRequest(downloadVideos[0].files[0].torrentDownloadUrl, 403) | 465 | const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
408 | expect(res.body.error).to.equal('Liu Bei') | 466 | expect(res.body.error).to.equal('Liu Bei') |
409 | 467 | ||
410 | await makeRawRequest(downloadVideos[1].files[0].torrentDownloadUrl, 200) | 468 | await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) |
411 | await makeRawRequest(downloadVideos[2].files[0].torrentDownloadUrl, 200) | 469 | await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) |
412 | }) | 470 | }) |
413 | 471 | ||
414 | it('Should run filter:api.download.video.allowed.result', async function () { | 472 | it('Should run filter:api.download.video.allowed.result', async function () { |
415 | { | 473 | { |
416 | const res = await makeRawRequest(downloadVideos[1].files[0].fileDownloadUrl, 403) | 474 | const res = await makeRawRequest({ url: downloadVideos[1].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
417 | expect(res.body.error).to.equal('Cao Cao') | 475 | expect(res.body.error).to.equal('Cao Cao') |
418 | 476 | ||
419 | await makeRawRequest(downloadVideos[0].files[0].fileDownloadUrl, 200) | 477 | await makeRawRequest({ url: downloadVideos[0].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) |
420 | await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200) | 478 | await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) |
421 | } | 479 | } |
422 | 480 | ||
423 | { | 481 | { |
424 | const res = await makeRawRequest(downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl, 403) | 482 | const res = await makeRawRequest({ |
483 | url: downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl, | ||
484 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
485 | }) | ||
486 | |||
425 | expect(res.body.error).to.equal('Sun Jian') | 487 | expect(res.body.error).to.equal('Sun Jian') |
426 | 488 | ||
427 | await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200) | 489 | await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) |
490 | |||
491 | await makeRawRequest({ | ||
492 | url: downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, | ||
493 | expectedStatus: HttpStatusCode.OK_200 | ||
494 | }) | ||
428 | 495 | ||
429 | await makeRawRequest(downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, 200) | 496 | await makeRawRequest({ |
430 | await makeRawRequest(downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl, 200) | 497 | url: downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl, |
498 | expectedStatus: HttpStatusCode.OK_200 | ||
499 | }) | ||
431 | } | 500 | } |
432 | }) | 501 | }) |
433 | }) | 502 | }) |
@@ -458,12 +527,12 @@ describe('Test plugin filter hooks', function () { | |||
458 | }) | 527 | }) |
459 | 528 | ||
460 | it('Should run filter:html.embed.video.allowed.result', async function () { | 529 | it('Should run filter:html.embed.video.allowed.result', async function () { |
461 | const res = await makeRawRequest(servers[0].url + embedVideos[0].embedPath, 200) | 530 | const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 }) |
462 | expect(res.text).to.equal('Lu Bu') | 531 | expect(res.text).to.equal('Lu Bu') |
463 | }) | 532 | }) |
464 | 533 | ||
465 | it('Should run filter:html.embed.video-playlist.allowed.result', async function () { | 534 | it('Should run filter:html.embed.video-playlist.allowed.result', async function () { |
466 | const res = await makeRawRequest(servers[0].url + embedPlaylists[0].embedPath, 200) | 535 | const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 }) |
467 | expect(res.text).to.equal('Diao Chan') | 536 | expect(res.text).to.equal('Diao Chan') |
468 | }) | 537 | }) |
469 | }) | 538 | }) |
diff --git a/server/tests/plugins/id-and-pass-auth.ts b/server/tests/plugins/id-and-pass-auth.ts index 85faac5a8..fc24a5656 100644 --- a/server/tests/plugins/id-and-pass-auth.ts +++ b/server/tests/plugins/id-and-pass-auth.ts | |||
@@ -48,7 +48,7 @@ describe('Test id and pass auth plugins', function () { | |||
48 | 48 | ||
49 | expect(body.username).to.equal('spyro') | 49 | expect(body.username).to.equal('spyro') |
50 | expect(body.account.displayName).to.equal('Spyro the Dragon') | 50 | expect(body.account.displayName).to.equal('Spyro the Dragon') |
51 | expect(body.role).to.equal(UserRole.USER) | 51 | expect(body.role.id).to.equal(UserRole.USER) |
52 | }) | 52 | }) |
53 | 53 | ||
54 | it('Should login Crash, create the user and use the token', async function () { | 54 | it('Should login Crash, create the user and use the token', async function () { |
@@ -63,7 +63,7 @@ describe('Test id and pass auth plugins', function () { | |||
63 | 63 | ||
64 | expect(body.username).to.equal('crash') | 64 | expect(body.username).to.equal('crash') |
65 | expect(body.account.displayName).to.equal('Crash Bandicoot') | 65 | expect(body.account.displayName).to.equal('Crash Bandicoot') |
66 | expect(body.role).to.equal(UserRole.MODERATOR) | 66 | expect(body.role.id).to.equal(UserRole.MODERATOR) |
67 | } | 67 | } |
68 | }) | 68 | }) |
69 | 69 | ||
@@ -79,7 +79,7 @@ describe('Test id and pass auth plugins', function () { | |||
79 | 79 | ||
80 | expect(body.username).to.equal('laguna') | 80 | expect(body.username).to.equal('laguna') |
81 | expect(body.account.displayName).to.equal('laguna') | 81 | expect(body.account.displayName).to.equal('laguna') |
82 | expect(body.role).to.equal(UserRole.USER) | 82 | expect(body.role.id).to.equal(UserRole.USER) |
83 | } | 83 | } |
84 | }) | 84 | }) |
85 | 85 | ||
@@ -129,7 +129,7 @@ describe('Test id and pass auth plugins', function () { | |||
129 | expect(body.username).to.equal('crash') | 129 | expect(body.username).to.equal('crash') |
130 | expect(body.account.displayName).to.equal('Beautiful Crash') | 130 | expect(body.account.displayName).to.equal('Beautiful Crash') |
131 | expect(body.account.description).to.equal('Mutant eastern barred bandicoot') | 131 | expect(body.account.description).to.equal('Mutant eastern barred bandicoot') |
132 | expect(body.role).to.equal(UserRole.MODERATOR) | 132 | expect(body.role.id).to.equal(UserRole.MODERATOR) |
133 | }) | 133 | }) |
134 | 134 | ||
135 | it('Should reject token of laguna by the plugin hook', async function () { | 135 | it('Should reject token of laguna by the plugin hook', async function () { |
diff --git a/server/tests/plugins/index.ts b/server/tests/plugins/index.ts index 4534120fd..210af7236 100644 --- a/server/tests/plugins/index.ts +++ b/server/tests/plugins/index.ts | |||
@@ -8,5 +8,6 @@ import './plugin-router' | |||
8 | import './plugin-storage' | 8 | import './plugin-storage' |
9 | import './plugin-transcoding' | 9 | import './plugin-transcoding' |
10 | import './plugin-unloading' | 10 | import './plugin-unloading' |
11 | import './plugin-websocket' | ||
11 | import './translations' | 12 | import './translations' |
12 | import './video-constants' | 13 | import './video-constants' |
diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts index 955d7ddfd..f2bada4ee 100644 --- a/server/tests/plugins/plugin-helpers.ts +++ b/server/tests/plugins/plugin-helpers.ts | |||
@@ -83,6 +83,33 @@ describe('Test plugin helpers', function () { | |||
83 | }) | 83 | }) |
84 | }) | 84 | }) |
85 | 85 | ||
86 | describe('Socket', function () { | ||
87 | |||
88 | it('Should sendNotification without any exceptions', async () => { | ||
89 | const user = await servers[0].users.create({ username: 'notis_redding', password: 'secret1234?' }) | ||
90 | await makePostBodyRequest({ | ||
91 | url: servers[0].url, | ||
92 | path: '/plugins/test-four/router/send-notification', | ||
93 | fields: { | ||
94 | userId: user.id | ||
95 | }, | ||
96 | expectedStatus: HttpStatusCode.CREATED_201 | ||
97 | }) | ||
98 | }) | ||
99 | |||
100 | it('Should sendVideoLiveNewState without any exceptions', async () => { | ||
101 | const res = await servers[0].videos.quickUpload({ name: 'video server 1' }) | ||
102 | |||
103 | await makePostBodyRequest({ | ||
104 | url: servers[0].url, | ||
105 | path: '/plugins/test-four/router/send-video-live-new-state/' + res.uuid, | ||
106 | expectedStatus: HttpStatusCode.CREATED_201 | ||
107 | }) | ||
108 | |||
109 | await servers[0].videos.remove({ id: res.uuid }) | ||
110 | }) | ||
111 | }) | ||
112 | |||
86 | describe('Plugin', function () { | 113 | describe('Plugin', function () { |
87 | 114 | ||
88 | it('Should get the base static route', async function () { | 115 | it('Should get the base static route', async function () { |
@@ -280,7 +307,7 @@ describe('Test plugin helpers', function () { | |||
280 | expect(file.fps).to.equal(25) | 307 | expect(file.fps).to.equal(25) |
281 | 308 | ||
282 | expect(await pathExists(file.path)).to.be.true | 309 | expect(await pathExists(file.path)).to.be.true |
283 | await makeRawRequest(file.url, HttpStatusCode.OK_200) | 310 | await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 }) |
284 | } | 311 | } |
285 | } | 312 | } |
286 | 313 | ||
@@ -294,12 +321,12 @@ describe('Test plugin helpers', function () { | |||
294 | const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE) | 321 | const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE) |
295 | expect(miniature).to.exist | 322 | expect(miniature).to.exist |
296 | expect(await pathExists(miniature.path)).to.be.true | 323 | expect(await pathExists(miniature.path)).to.be.true |
297 | await makeRawRequest(miniature.url, HttpStatusCode.OK_200) | 324 | await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 }) |
298 | 325 | ||
299 | const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW) | 326 | const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW) |
300 | expect(preview).to.exist | 327 | expect(preview).to.exist |
301 | expect(await pathExists(preview.path)).to.be.true | 328 | expect(await pathExists(preview.path)).to.be.true |
302 | await makeRawRequest(preview.url, HttpStatusCode.OK_200) | 329 | await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 }) |
303 | } | 330 | } |
304 | }) | 331 | }) |
305 | 332 | ||
diff --git a/server/tests/plugins/plugin-websocket.ts b/server/tests/plugins/plugin-websocket.ts new file mode 100644 index 000000000..adaa28b1d --- /dev/null +++ b/server/tests/plugins/plugin-websocket.ts | |||
@@ -0,0 +1,70 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import WebSocket from 'ws' | ||
4 | import { cleanupTests, createSingleServer, PeerTubeServer, PluginsCommand, setAccessTokensToServers } from '@shared/server-commands' | ||
5 | |||
6 | function buildWebSocket (server: PeerTubeServer, path: string) { | ||
7 | return new WebSocket('ws://' + server.host + path) | ||
8 | } | ||
9 | |||
10 | function expectErrorOrTimeout (server: PeerTubeServer, path: string, expectedTimeout: number) { | ||
11 | return new Promise<void>((res, rej) => { | ||
12 | const ws = buildWebSocket(server, path) | ||
13 | ws.on('error', () => res()) | ||
14 | |||
15 | const timeout = setTimeout(() => res(), expectedTimeout) | ||
16 | |||
17 | ws.on('open', () => { | ||
18 | clearTimeout(timeout) | ||
19 | |||
20 | return rej(new Error('Connect did not timeout')) | ||
21 | }) | ||
22 | }) | ||
23 | } | ||
24 | |||
25 | describe('Test plugin websocket', function () { | ||
26 | let server: PeerTubeServer | ||
27 | const basePaths = [ | ||
28 | '/plugins/test-websocket/ws/', | ||
29 | '/plugins/test-websocket/0.0.1/ws/' | ||
30 | ] | ||
31 | |||
32 | before(async function () { | ||
33 | this.timeout(30000) | ||
34 | |||
35 | server = await createSingleServer(1) | ||
36 | await setAccessTokensToServers([ server ]) | ||
37 | |||
38 | await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-websocket') }) | ||
39 | }) | ||
40 | |||
41 | it('Should not connect to the websocket without the appropriate path', async function () { | ||
42 | const paths = [ | ||
43 | '/plugins/unknown/ws/', | ||
44 | '/plugins/unknown/0.0.1/ws/' | ||
45 | ] | ||
46 | |||
47 | for (const path of paths) { | ||
48 | await expectErrorOrTimeout(server, path, 1000) | ||
49 | } | ||
50 | }) | ||
51 | |||
52 | it('Should not connect to the websocket without the appropriate sub path', async function () { | ||
53 | for (const path of basePaths) { | ||
54 | await expectErrorOrTimeout(server, path + '/unknown', 1000) | ||
55 | } | ||
56 | }) | ||
57 | |||
58 | it('Should connect to the websocket and receive pong', function (done) { | ||
59 | const ws = buildWebSocket(server, basePaths[0]) | ||
60 | |||
61 | ws.on('open', () => ws.send('ping')) | ||
62 | ws.on('message', data => { | ||
63 | if (data.toString() === 'pong') return done() | ||
64 | }) | ||
65 | }) | ||
66 | |||
67 | after(async function () { | ||
68 | await cleanupTests([ server ]) | ||
69 | }) | ||
70 | }) | ||
diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts index f8f4a5137..41fd72e89 100644 --- a/server/tests/shared/actors.ts +++ b/server/tests/shared/actors.ts | |||
@@ -2,8 +2,6 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { pathExists, readdir } from 'fs-extra' | 4 | import { pathExists, readdir } from 'fs-extra' |
5 | import { join } from 'path' | ||
6 | import { root } from '@shared/core-utils' | ||
7 | import { Account, VideoChannel } from '@shared/models' | 5 | import { Account, VideoChannel } from '@shared/models' |
8 | import { PeerTubeServer } from '@shared/server-commands' | 6 | import { PeerTubeServer } from '@shared/server-commands' |
9 | 7 | ||
@@ -31,11 +29,9 @@ async function expectAccountFollows (options: { | |||
31 | return expectActorFollow({ ...options, data }) | 29 | return expectActorFollow({ ...options, data }) |
32 | } | 30 | } |
33 | 31 | ||
34 | async function checkActorFilesWereRemoved (filename: string, serverNumber: number) { | 32 | async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) { |
35 | const testDirectory = 'test' + serverNumber | ||
36 | |||
37 | for (const directory of [ 'avatars' ]) { | 33 | for (const directory of [ 'avatars' ]) { |
38 | const directoryPath = join(root(), testDirectory, directory) | 34 | const directoryPath = server.getDirectoryPath(directory) |
39 | 35 | ||
40 | const directoryExists = await pathExists(directoryPath) | 36 | const directoryExists = await pathExists(directoryPath) |
41 | expect(directoryExists).to.be.true | 37 | expect(directoryExists).to.be.true |
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index c7065a767..90d534a06 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts | |||
@@ -2,22 +2,18 @@ | |||
2 | 2 | ||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { pathExists, readdir } from 'fs-extra' | 4 | import { pathExists, readdir } from 'fs-extra' |
5 | import { join } from 'path' | ||
6 | import { root } from '@shared/core-utils' | ||
7 | import { PeerTubeServer } from '@shared/server-commands' | 5 | import { PeerTubeServer } from '@shared/server-commands' |
8 | 6 | ||
9 | async function checkTmpIsEmpty (server: PeerTubeServer) { | 7 | async function checkTmpIsEmpty (server: PeerTubeServer) { |
10 | await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) | 8 | await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) |
11 | 9 | ||
12 | if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { | 10 | if (await pathExists(server.getDirectoryPath('tmp/hls'))) { |
13 | await checkDirectoryIsEmpty(server, 'tmp/hls') | 11 | await checkDirectoryIsEmpty(server, 'tmp/hls') |
14 | } | 12 | } |
15 | } | 13 | } |
16 | 14 | ||
17 | async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { | 15 | async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { |
18 | const testDirectory = 'test' + server.internalServerNumber | 16 | const directoryPath = server.getDirectoryPath(directory) |
19 | |||
20 | const directoryPath = join(root(), testDirectory, directory) | ||
21 | 17 | ||
22 | const directoryExists = await pathExists(directoryPath) | 18 | const directoryExists = await pathExists(directoryPath) |
23 | expect(directoryExists).to.be.true | 19 | expect(directoryExists).to.be.true |
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts index 4bd4786fc..47e0dc481 100644 --- a/server/tests/shared/live.ts +++ b/server/tests/shared/live.ts | |||
@@ -3,39 +3,118 @@ | |||
3 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
4 | import { pathExists, readdir } from 'fs-extra' | 4 | import { pathExists, readdir } from 'fs-extra' |
5 | import { join } from 'path' | 5 | import { join } from 'path' |
6 | import { LiveVideo } from '@shared/models' | 6 | import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models' |
7 | import { PeerTubeServer } from '@shared/server-commands' | 7 | import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands' |
8 | 8 | import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists' | |
9 | async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) { | 9 | |
10 | let live: LiveVideo | 10 | async function checkLiveCleanup (options: { |
11 | 11 | server: PeerTubeServer | |
12 | try { | 12 | videoUUID: string |
13 | live = await server.live.get({ videoId: videoUUID }) | 13 | permanent: boolean |
14 | } catch {} | 14 | savedResolutions?: number[] |
15 | }) { | ||
16 | const { server, videoUUID, permanent, savedResolutions = [] } = options | ||
15 | 17 | ||
16 | const basePath = server.servers.buildDirectory('streaming-playlists') | 18 | const basePath = server.servers.buildDirectory('streaming-playlists') |
17 | const hlsPath = join(basePath, 'hls', videoUUID) | 19 | const hlsPath = join(basePath, 'hls', videoUUID) |
18 | 20 | ||
19 | if (savedResolutions.length === 0) { | 21 | if (permanent) { |
22 | if (!await pathExists(hlsPath)) return | ||
20 | 23 | ||
21 | if (live?.permanentLive) { | 24 | const files = await readdir(hlsPath) |
22 | expect(await pathExists(hlsPath)).to.be.true | 25 | expect(files).to.have.lengthOf(0) |
26 | return | ||
27 | } | ||
23 | 28 | ||
24 | const hlsFiles = await readdir(hlsPath) | 29 | if (savedResolutions.length === 0) { |
25 | expect(hlsFiles).to.have.lengthOf(1) // Only replays directory | 30 | return checkUnsavedLiveCleanup(server, videoUUID, hlsPath) |
31 | } | ||
26 | 32 | ||
27 | const replayDir = join(hlsPath, 'replay') | 33 | return checkSavedLiveCleanup(hlsPath, savedResolutions) |
28 | expect(await pathExists(replayDir)).to.be.true | 34 | } |
29 | 35 | ||
30 | const replayFiles = await readdir(join(hlsPath, 'replay')) | 36 | // --------------------------------------------------------------------------- |
31 | expect(replayFiles).to.have.lengthOf(0) | 37 | |
32 | } else { | 38 | async function testVideoResolutions (options: { |
33 | expect(await pathExists(hlsPath)).to.be.false | 39 | originServer: PeerTubeServer |
40 | servers: PeerTubeServer[] | ||
41 | liveVideoId: string | ||
42 | resolutions: number[] | ||
43 | transcoded: boolean | ||
44 | objectStorage: boolean | ||
45 | }) { | ||
46 | const { originServer, servers, liveVideoId, resolutions, transcoded, objectStorage } = options | ||
47 | |||
48 | for (const server of servers) { | ||
49 | const { data } = await server.videos.list() | ||
50 | expect(data.find(v => v.uuid === liveVideoId)).to.exist | ||
51 | |||
52 | const video = await server.videos.get({ id: liveVideoId }) | ||
53 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
54 | |||
55 | const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) | ||
56 | expect(hlsPlaylist).to.exist | ||
57 | expect(hlsPlaylist.files).to.have.lengthOf(0) // Only fragmented mp4 files are displayed | ||
58 | |||
59 | await checkResolutionsInMasterPlaylist({ | ||
60 | server, | ||
61 | playlistUrl: hlsPlaylist.playlistUrl, | ||
62 | resolutions, | ||
63 | transcoded, | ||
64 | withRetry: objectStorage | ||
65 | }) | ||
66 | |||
67 | if (objectStorage) { | ||
68 | expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl()) | ||
34 | } | 69 | } |
35 | 70 | ||
36 | return | 71 | for (let i = 0; i < resolutions.length; i++) { |
72 | const segmentNum = 3 | ||
73 | const segmentName = `${i}-00000${segmentNum}.ts` | ||
74 | await originServer.live.waitUntilSegmentGeneration({ | ||
75 | server: originServer, | ||
76 | videoUUID: video.uuid, | ||
77 | playlistNumber: i, | ||
78 | segment: segmentNum, | ||
79 | objectStorage | ||
80 | }) | ||
81 | |||
82 | const baseUrl = objectStorage | ||
83 | ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls' | ||
84 | : originServer.url + '/static/streaming-playlists/hls' | ||
85 | |||
86 | if (objectStorage) { | ||
87 | expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl()) | ||
88 | } | ||
89 | |||
90 | const subPlaylist = await originServer.streamingPlaylists.get({ | ||
91 | url: `${baseUrl}/${video.uuid}/${i}.m3u8`, | ||
92 | withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3 | ||
93 | }) | ||
94 | |||
95 | expect(subPlaylist).to.contain(segmentName) | ||
96 | |||
97 | await checkLiveSegmentHash({ | ||
98 | server, | ||
99 | baseUrlSegment: baseUrl, | ||
100 | videoUUID: video.uuid, | ||
101 | segmentName, | ||
102 | hlsPlaylist | ||
103 | }) | ||
104 | } | ||
37 | } | 105 | } |
106 | } | ||
107 | |||
108 | // --------------------------------------------------------------------------- | ||
109 | |||
110 | export { | ||
111 | checkLiveCleanup, | ||
112 | testVideoResolutions | ||
113 | } | ||
114 | |||
115 | // --------------------------------------------------------------------------- | ||
38 | 116 | ||
117 | async function checkSavedLiveCleanup (hlsPath: string, savedResolutions: number[] = []) { | ||
39 | const files = await readdir(hlsPath) | 118 | const files = await readdir(hlsPath) |
40 | 119 | ||
41 | // fragmented file and playlist per resolution + master playlist + segments sha256 json file | 120 | // fragmented file and playlist per resolution + master playlist + segments sha256 json file |
@@ -56,6 +135,27 @@ async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, save | |||
56 | expect(shaFile).to.exist | 135 | expect(shaFile).to.exist |
57 | } | 136 | } |
58 | 137 | ||
59 | export { | 138 | async function checkUnsavedLiveCleanup (server: PeerTubeServer, videoUUID: string, hlsPath: string) { |
60 | checkLiveCleanup | 139 | let live: LiveVideo |
140 | |||
141 | try { | ||
142 | live = await server.live.get({ videoId: videoUUID }) | ||
143 | } catch {} | ||
144 | |||
145 | if (live?.permanentLive) { | ||
146 | expect(await pathExists(hlsPath)).to.be.true | ||
147 | |||
148 | const hlsFiles = await readdir(hlsPath) | ||
149 | expect(hlsFiles).to.have.lengthOf(1) // Only replays directory | ||
150 | |||
151 | const replayDir = join(hlsPath, 'replay') | ||
152 | expect(await pathExists(replayDir)).to.be.true | ||
153 | |||
154 | const replayFiles = await readdir(join(hlsPath, 'replay')) | ||
155 | expect(replayFiles).to.have.lengthOf(0) | ||
156 | |||
157 | return | ||
158 | } | ||
159 | |||
160 | expect(await pathExists(hlsPath)).to.be.false | ||
61 | } | 161 | } |
diff --git a/server/tests/shared/mock-servers/mock-object-storage.ts b/server/tests/shared/mock-servers/mock-object-storage.ts index 99d68e014..8c325bf11 100644 --- a/server/tests/shared/mock-servers/mock-object-storage.ts +++ b/server/tests/shared/mock-servers/mock-object-storage.ts | |||
@@ -12,7 +12,7 @@ export class MockObjectStorage { | |||
12 | const app = express() | 12 | const app = express() |
13 | 13 | ||
14 | app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { | 14 | app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { |
15 | const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` | 15 | const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}` |
16 | 16 | ||
17 | if (process.env.DEBUG) { | 17 | if (process.env.DEBUG) { |
18 | console.log('Receiving request on mocked server %s.', req.url) | 18 | console.log('Receiving request on mocked server %s.', req.url) |
diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts index fdd541d20..8db303fd8 100644 --- a/server/tests/shared/playlists.ts +++ b/server/tests/shared/playlists.ts | |||
@@ -1,17 +1,14 @@ | |||
1 | import { expect } from 'chai' | 1 | import { expect } from 'chai' |
2 | import { readdir } from 'fs-extra' | 2 | import { readdir } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { PeerTubeServer } from '@shared/server-commands' |
4 | import { root } from '@shared/core-utils' | ||
5 | 4 | ||
6 | async function checkPlaylistFilesWereRemoved ( | 5 | async function checkPlaylistFilesWereRemoved ( |
7 | playlistUUID: string, | 6 | playlistUUID: string, |
8 | internalServerNumber: number, | 7 | server: PeerTubeServer, |
9 | directories = [ 'thumbnails' ] | 8 | directories = [ 'thumbnails' ] |
10 | ) { | 9 | ) { |
11 | const testDirectory = 'test' + internalServerNumber | ||
12 | |||
13 | for (const directory of directories) { | 10 | for (const directory of directories) { |
14 | const directoryPath = join(root(), testDirectory, directory) | 11 | const directoryPath = server.getDirectoryPath(directory) |
15 | 12 | ||
16 | const files = await readdir(directoryPath) | 13 | const files = await readdir(directoryPath) |
17 | for (const file of files) { | 14 | for (const file of files) { |
diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts index 4d82b3654..824c3dcef 100644 --- a/server/tests/shared/streaming-playlists.ts +++ b/server/tests/shared/streaming-playlists.ts | |||
@@ -1,9 +1,13 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
1 | import { expect } from 'chai' | 3 | import { expect } from 'chai' |
2 | import { basename } from 'path' | 4 | import { basename } from 'path' |
3 | import { removeFragmentedMP4Ext } from '@shared/core-utils' | 5 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' |
4 | import { sha256 } from '@shared/extra-utils' | 6 | import { sha256 } from '@shared/extra-utils' |
5 | import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models' | 7 | import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models' |
6 | import { PeerTubeServer } from '@shared/server-commands' | 8 | import { makeRawRequest, PeerTubeServer, webtorrentAdd } from '@shared/server-commands' |
9 | import { expectStartWith } from './checks' | ||
10 | import { hlsInfohashExist } from './tracker' | ||
7 | 11 | ||
8 | async function checkSegmentHash (options: { | 12 | async function checkSegmentHash (options: { |
9 | server: PeerTubeServer | 13 | server: PeerTubeServer |
@@ -26,7 +30,7 @@ async function checkSegmentHash (options: { | |||
26 | const offset = parseInt(matches[2], 10) | 30 | const offset = parseInt(matches[2], 10) |
27 | const range = `${offset}-${offset + length - 1}` | 31 | const range = `${offset}-${offset + length - 1}` |
28 | 32 | ||
29 | const segmentBody = await command.getSegment({ | 33 | const segmentBody = await command.getFragmentedSegment({ |
30 | url: `${baseUrlSegment}/${videoName}`, | 34 | url: `${baseUrlSegment}/${videoName}`, |
31 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, | 35 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, |
32 | range: `bytes=${range}` | 36 | range: `bytes=${range}` |
@@ -46,7 +50,7 @@ async function checkLiveSegmentHash (options: { | |||
46 | const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options | 50 | const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options |
47 | const command = server.streamingPlaylists | 51 | const command = server.streamingPlaylists |
48 | 52 | ||
49 | const segmentBody = await command.getSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) | 53 | const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) |
50 | const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) | 54 | const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) |
51 | 55 | ||
52 | expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) | 56 | expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) |
@@ -56,15 +60,17 @@ async function checkResolutionsInMasterPlaylist (options: { | |||
56 | server: PeerTubeServer | 60 | server: PeerTubeServer |
57 | playlistUrl: string | 61 | playlistUrl: string |
58 | resolutions: number[] | 62 | resolutions: number[] |
63 | transcoded?: boolean // default true | ||
64 | withRetry?: boolean // default false | ||
59 | }) { | 65 | }) { |
60 | const { server, playlistUrl, resolutions } = options | 66 | const { server, playlistUrl, resolutions, withRetry = false, transcoded = true } = options |
61 | 67 | ||
62 | const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl }) | 68 | const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl, withRetry }) |
63 | 69 | ||
64 | for (const resolution of resolutions) { | 70 | for (const resolution of resolutions) { |
65 | const reg = new RegExp( | 71 | const reg = transcoded |
66 | '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"' | 72 | ? new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"') |
67 | ) | 73 | : new RegExp('#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + '') |
68 | 74 | ||
69 | expect(masterPlaylist).to.match(reg) | 75 | expect(masterPlaylist).to.match(reg) |
70 | } | 76 | } |
@@ -73,8 +79,118 @@ async function checkResolutionsInMasterPlaylist (options: { | |||
73 | expect(playlistsLength).to.have.lengthOf(resolutions.length) | 79 | expect(playlistsLength).to.have.lengthOf(resolutions.length) |
74 | } | 80 | } |
75 | 81 | ||
82 | async function completeCheckHlsPlaylist (options: { | ||
83 | servers: PeerTubeServer[] | ||
84 | videoUUID: string | ||
85 | hlsOnly: boolean | ||
86 | |||
87 | resolutions?: number[] | ||
88 | objectStorageBaseUrl: string | ||
89 | }) { | ||
90 | const { videoUUID, hlsOnly, objectStorageBaseUrl } = options | ||
91 | |||
92 | const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] | ||
93 | |||
94 | for (const server of options.servers) { | ||
95 | const videoDetails = await server.videos.get({ id: videoUUID }) | ||
96 | const baseUrl = `http://${videoDetails.account.host}` | ||
97 | |||
98 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | ||
99 | |||
100 | const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) | ||
101 | expect(hlsPlaylist).to.not.be.undefined | ||
102 | |||
103 | const hlsFiles = hlsPlaylist.files | ||
104 | expect(hlsFiles).to.have.lengthOf(resolutions.length) | ||
105 | |||
106 | if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0) | ||
107 | else expect(videoDetails.files).to.have.lengthOf(resolutions.length) | ||
108 | |||
109 | // Check JSON files | ||
110 | for (const resolution of resolutions) { | ||
111 | const file = hlsFiles.find(f => f.resolution.id === resolution) | ||
112 | expect(file).to.not.be.undefined | ||
113 | |||
114 | expect(file.magnetUri).to.have.lengthOf.above(2) | ||
115 | expect(file.torrentUrl).to.match( | ||
116 | new RegExp(`${server.url}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) | ||
117 | ) | ||
118 | |||
119 | if (objectStorageBaseUrl) { | ||
120 | expectStartWith(file.fileUrl, objectStorageBaseUrl) | ||
121 | } else { | ||
122 | expect(file.fileUrl).to.match( | ||
123 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | ||
124 | ) | ||
125 | } | ||
126 | |||
127 | expect(file.resolution.label).to.equal(resolution + 'p') | ||
128 | |||
129 | await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
130 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
131 | |||
132 | const torrent = await webtorrentAdd(file.magnetUri, true) | ||
133 | expect(torrent.files).to.be.an('array') | ||
134 | expect(torrent.files.length).to.equal(1) | ||
135 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | ||
136 | } | ||
137 | |||
138 | // Check master playlist | ||
139 | { | ||
140 | await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) | ||
141 | |||
142 | const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl }) | ||
143 | |||
144 | let i = 0 | ||
145 | for (const resolution of resolutions) { | ||
146 | expect(masterPlaylist).to.contain(`${resolution}.m3u8`) | ||
147 | expect(masterPlaylist).to.contain(`${resolution}.m3u8`) | ||
148 | |||
149 | const url = 'http://' + videoDetails.account.host | ||
150 | await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i) | ||
151 | |||
152 | i++ | ||
153 | } | ||
154 | } | ||
155 | |||
156 | // Check resolution playlists | ||
157 | { | ||
158 | for (const resolution of resolutions) { | ||
159 | const file = hlsFiles.find(f => f.resolution.id === resolution) | ||
160 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' | ||
161 | |||
162 | const url = objectStorageBaseUrl | ||
163 | ? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}` | ||
164 | : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` | ||
165 | |||
166 | const subPlaylist = await server.streamingPlaylists.get({ url }) | ||
167 | |||
168 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) | ||
169 | expect(subPlaylist).to.contain(basename(file.fileUrl)) | ||
170 | } | ||
171 | } | ||
172 | |||
173 | { | ||
174 | const baseUrlAndPath = objectStorageBaseUrl | ||
175 | ? objectStorageBaseUrl + 'hls/' + videoUUID | ||
176 | : baseUrl + '/static/streaming-playlists/hls/' + videoUUID | ||
177 | |||
178 | for (const resolution of resolutions) { | ||
179 | await checkSegmentHash({ | ||
180 | server, | ||
181 | baseUrlPlaylist: baseUrlAndPath, | ||
182 | baseUrlSegment: baseUrlAndPath, | ||
183 | resolution, | ||
184 | hlsPlaylist | ||
185 | }) | ||
186 | } | ||
187 | } | ||
188 | } | ||
189 | } | ||
190 | |||
76 | export { | 191 | export { |
77 | checkSegmentHash, | 192 | checkSegmentHash, |
78 | checkLiveSegmentHash, | 193 | checkLiveSegmentHash, |
79 | checkResolutionsInMasterPlaylist | 194 | checkResolutionsInMasterPlaylist, |
195 | completeCheckHlsPlaylist | ||
80 | } | 196 | } |
diff --git a/server/tests/shared/videos.ts b/server/tests/shared/videos.ts index e18329e07..c8339584b 100644 --- a/server/tests/shared/videos.ts +++ b/server/tests/shared/videos.ts | |||
@@ -125,9 +125,9 @@ async function completeVideoCheck ( | |||
125 | expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) | 125 | expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) |
126 | 126 | ||
127 | await Promise.all([ | 127 | await Promise.all([ |
128 | makeRawRequest(file.torrentUrl, 200), | 128 | makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }), |
129 | makeRawRequest(file.torrentDownloadUrl, 200), | 129 | makeRawRequest({ url: file.torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }), |
130 | makeRawRequest(file.metadataUrl, 200) | 130 | makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.OK_200 }) |
131 | ]) | 131 | ]) |
132 | 132 | ||
133 | expect(file.resolution.id).to.equal(attributeFile.resolution) | 133 | expect(file.resolution.id).to.equal(attributeFile.resolution) |
diff --git a/server/tools/cli.ts b/server/tools/cli.ts index a15d73fb4..4607d052a 100644 --- a/server/tools/cli.ts +++ b/server/tools/cli.ts | |||
@@ -20,7 +20,7 @@ async function getAdminTokenOrDie (server: PeerTubeServer, username: string, pas | |||
20 | const token = await server.login.getAccessToken(username, password) | 20 | const token = await server.login.getAccessToken(username, password) |
21 | const me = await server.users.getMyInfo({ token }) | 21 | const me = await server.users.getMyInfo({ token }) |
22 | 22 | ||
23 | if (me.role !== UserRole.ADMINISTRATOR) { | 23 | if (me.role.id !== UserRole.ADMINISTRATOR) { |
24 | console.error('You must be an administrator.') | 24 | console.error('You must be an administrator.') |
25 | process.exit(-1) | 25 | process.exit(-1) |
26 | } | 26 | } |
diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 27d60da72..3738ffc47 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts | |||
@@ -97,7 +97,7 @@ declare module 'express' { | |||
97 | 97 | ||
98 | title?: string | 98 | title?: string |
99 | status?: number | 99 | status?: number |
100 | type?: ServerErrorCode | 100 | type?: ServerErrorCode | string |
101 | instance?: string | 101 | instance?: string |
102 | 102 | ||
103 | data?: PeerTubeProblemDocumentData | 103 | data?: PeerTubeProblemDocumentData |
diff --git a/server/types/models/video/video-playlist.ts b/server/types/models/video/video-playlist.ts index 33fe5416a..40f0dfc14 100644 --- a/server/types/models/video/video-playlist.ts +++ b/server/types/models/video/video-playlist.ts | |||
@@ -14,6 +14,10 @@ export type MVideoPlaylist = Omit<VideoPlaylistModel, 'OwnerAccount' | 'VideoCha | |||
14 | // ############################################################################ | 14 | // ############################################################################ |
15 | 15 | ||
16 | export type MVideoPlaylistId = Pick<MVideoPlaylist, 'id'> | 16 | export type MVideoPlaylistId = Pick<MVideoPlaylist, 'id'> |
17 | export type MVideoPlaylistSummary = | ||
18 | Pick<MVideoPlaylist, 'id'> & | ||
19 | Pick<MVideoPlaylist, 'name'> & | ||
20 | Pick<MVideoPlaylist, 'uuid'> | ||
17 | export type MVideoPlaylistPrivacy = Pick<MVideoPlaylist, 'privacy'> | 21 | export type MVideoPlaylistPrivacy = Pick<MVideoPlaylist, 'privacy'> |
18 | export type MVideoPlaylistUUID = Pick<MVideoPlaylist, 'uuid'> | 22 | export type MVideoPlaylistUUID = Pick<MVideoPlaylist, 'uuid'> |
19 | export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: number } | 23 | export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: number } |
@@ -22,12 +26,8 @@ export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: numbe | |||
22 | 26 | ||
23 | // With elements | 27 | // With elements |
24 | 28 | ||
25 | export type MVideoPlaylistWithElements = | 29 | export type MVideoPlaylistSummaryWithElements = |
26 | MVideoPlaylist & | 30 | MVideoPlaylistSummary & |
27 | Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]> | ||
28 | |||
29 | export type MVideoPlaylistIdWithElements = | ||
30 | MVideoPlaylistId & | ||
31 | Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]> | 31 | Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]> |
32 | 32 | ||
33 | // ############################################################################ | 33 | // ############################################################################ |
diff --git a/server/types/plugins/index.ts b/server/types/plugins/index.ts index de30ff2ab..bf9c35d49 100644 --- a/server/types/plugins/index.ts +++ b/server/types/plugins/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | export * from './plugin-library.model' | 1 | export * from './plugin-library.model' |
2 | export * from './register-server-auth.model' | 2 | export * from './register-server-auth.model' |
3 | export * from './register-server-option.model' | 3 | export * from './register-server-option.model' |
4 | export * from './register-server-websocket-route.model' | ||
diff --git a/server/types/plugins/register-server-option.model.ts b/server/types/plugins/register-server-option.model.ts index fb4f12a4c..1e2bd830e 100644 --- a/server/types/plugins/register-server-option.model.ts +++ b/server/types/plugins/register-server-option.model.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | import { Response, Router } from 'express' | 1 | import { Response, Router } from 'express' |
2 | import { Server } from 'http' | ||
2 | import { Logger } from 'winston' | 3 | import { Logger } from 'winston' |
3 | import { ActorModel } from '@server/models/actor/actor' | 4 | import { ActorModel } from '@server/models/actor/actor' |
4 | import { | 5 | import { |
@@ -16,12 +17,13 @@ import { | |||
16 | ThumbnailType, | 17 | ThumbnailType, |
17 | VideoBlacklistCreate | 18 | VideoBlacklistCreate |
18 | } from '@shared/models' | 19 | } from '@shared/models' |
19 | import { MUserDefault, MVideoThumbnail } from '../models' | 20 | import { MUserDefault, MVideo, MVideoThumbnail, UserNotificationModelForApi } from '../models' |
20 | import { | 21 | import { |
21 | RegisterServerAuthExternalOptions, | 22 | RegisterServerAuthExternalOptions, |
22 | RegisterServerAuthExternalResult, | 23 | RegisterServerAuthExternalResult, |
23 | RegisterServerAuthPassOptions | 24 | RegisterServerAuthPassOptions |
24 | } from './register-server-auth.model' | 25 | } from './register-server-auth.model' |
26 | import { RegisterServerWebSocketRouteOptions } from './register-server-websocket-route.model' | ||
25 | 27 | ||
26 | export type PeerTubeHelpers = { | 28 | export type PeerTubeHelpers = { |
27 | logger: Logger | 29 | logger: Logger |
@@ -83,15 +85,25 @@ export type PeerTubeHelpers = { | |||
83 | } | 85 | } |
84 | 86 | ||
85 | server: { | 87 | server: { |
88 | // PeerTube >= 5.0 | ||
89 | getHTTPServer: () => Server | ||
90 | |||
86 | getServerActor: () => Promise<ActorModel> | 91 | getServerActor: () => Promise<ActorModel> |
87 | } | 92 | } |
88 | 93 | ||
94 | socket: { | ||
95 | sendNotification: (userId: number, notification: UserNotificationModelForApi) => void | ||
96 | sendVideoLiveNewState: (video: MVideo) => void | ||
97 | } | ||
98 | |||
89 | plugin: { | 99 | plugin: { |
90 | // PeerTube >= 3.2 | 100 | // PeerTube >= 3.2 |
91 | getBaseStaticRoute: () => string | 101 | getBaseStaticRoute: () => string |
92 | 102 | ||
93 | // PeerTube >= 3.2 | 103 | // PeerTube >= 3.2 |
94 | getBaseRouterRoute: () => string | 104 | getBaseRouterRoute: () => string |
105 | // PeerTube >= 5.0 | ||
106 | getBaseWebSocketRoute: () => string | ||
95 | 107 | ||
96 | // PeerTube >= 3.2 | 108 | // PeerTube >= 3.2 |
97 | getDataDirectoryPath: () => string | 109 | getDataDirectoryPath: () => string |
@@ -135,5 +147,12 @@ export type RegisterServerOptions = { | |||
135 | // * /plugins/:pluginName/router/... | 147 | // * /plugins/:pluginName/router/... |
136 | getRouter(): Router | 148 | getRouter(): Router |
137 | 149 | ||
150 | // PeerTube >= 5.0 | ||
151 | // Register WebSocket route | ||
152 | // Base routes of the WebSocket router are | ||
153 | // * /plugins/:pluginName/:pluginVersion/ws/... | ||
154 | // * /plugins/:pluginName/ws/... | ||
155 | registerWebSocketRoute: (options: RegisterServerWebSocketRouteOptions) => void | ||
156 | |||
138 | peertubeHelpers: PeerTubeHelpers | 157 | peertubeHelpers: PeerTubeHelpers |
139 | } | 158 | } |
diff --git a/server/types/plugins/register-server-websocket-route.model.ts b/server/types/plugins/register-server-websocket-route.model.ts new file mode 100644 index 000000000..edf64f66b --- /dev/null +++ b/server/types/plugins/register-server-websocket-route.model.ts | |||
@@ -0,0 +1,8 @@ | |||
1 | import { IncomingMessage } from 'http' | ||
2 | import { Duplex } from 'stream' | ||
3 | |||
4 | export type RegisterServerWebSocketRouteOptions = { | ||
5 | route: string | ||
6 | |||
7 | handler: (request: IncomingMessage, socket: Duplex, head: Buffer) => any | ||
8 | } | ||