aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-create.ts2
-rw-r--r--server/lib/activitypub/video-comments.ts35
-rw-r--r--server/lib/auth/oauth.ts27
-rw-r--r--server/lib/hls.ts6
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts10
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts22
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts27
-rw-r--r--server/lib/live/live-manager.ts12
-rw-r--r--server/lib/live/live-segment-sha-store.ts75
-rw-r--r--server/lib/live/live-utils.ts67
-rw-r--r--server/lib/live/shared/muxing-session.ts106
-rw-r--r--server/lib/moderation.ts42
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts25
-rw-r--r--server/lib/object-storage/videos.ts37
-rw-r--r--server/lib/plugins/plugin-helpers-builder.ts28
-rw-r--r--server/lib/plugins/plugin-manager.ts31
-rw-r--r--server/lib/plugins/register-helpers.ts21
-rw-r--r--server/lib/redis.ts25
-rw-r--r--server/lib/schedulers/video-channel-sync-latest-scheduler.ts35
-rw-r--r--server/lib/sync-channel.ts90
20 files changed, 517 insertions, 206 deletions
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 76ed37aae..1e6e8956c 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -109,8 +109,10 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc
109 let video: MVideoAccountLightBlacklistAllFiles 109 let video: MVideoAccountLightBlacklistAllFiles
110 let created: boolean 110 let created: boolean
111 let comment: MCommentOwnerVideo 111 let comment: MCommentOwnerVideo
112
112 try { 113 try {
113 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) 114 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
115 if (!resolveThreadResult) return // Comment not accepted
114 116
115 video = resolveThreadResult.video 117 video = resolveThreadResult.video
116 created = resolveThreadResult.commentCreated 118 created = resolveThreadResult.commentCreated
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 911c7cd30..b65baf0e9 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -4,7 +4,9 @@ import { logger } from '../../helpers/logger'
4import { doJSONRequest } from '../../helpers/requests' 4import { doJSONRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 7import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
8import { isRemoteVideoCommentAccepted } from '../moderation'
9import { Hooks } from '../plugins/hooks'
8import { getOrCreateAPActor } from './actors' 10import { getOrCreateAPActor } from './actors'
9import { checkUrlsSameHost } from './url' 11import { checkUrlsSameHost } from './url'
10import { getOrCreateAPVideo } from './videos' 12import { getOrCreateAPVideo } from './videos'
@@ -103,6 +105,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
103 firstReply.changed('updatedAt', true) 105 firstReply.changed('updatedAt', true)
104 firstReply.Video = video 106 firstReply.Video = video
105 107
108 if (await isRemoteCommentAccepted(firstReply) !== true) {
109 return undefined
110 }
111
106 comments[comments.length - 1] = await firstReply.save() 112 comments[comments.length - 1] = await firstReply.save()
107 113
108 for (let i = comments.length - 2; i >= 0; i--) { 114 for (let i = comments.length - 2; i >= 0; i--) {
@@ -113,6 +119,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
113 comment.changed('updatedAt', true) 119 comment.changed('updatedAt', true)
114 comment.Video = video 120 comment.Video = video
115 121
122 if (await isRemoteCommentAccepted(comment) !== true) {
123 return undefined
124 }
125
116 comments[i] = await comment.save() 126 comments[i] = await comment.save()
117 } 127 }
118 128
@@ -169,3 +179,26 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
169 commentCreated: true 179 commentCreated: true
170 }) 180 })
171} 181}
182
183async function isRemoteCommentAccepted (comment: MComment) {
184 // Already created
185 if (comment.id) return true
186
187 const acceptParameters = {
188 comment
189 }
190
191 const acceptedResult = await Hooks.wrapFun(
192 isRemoteVideoCommentAccepted,
193 acceptParameters,
194 'filter:activity-pub.remote-video-comment.create.accept.result'
195 )
196
197 if (!acceptedResult || acceptedResult.accepted !== true) {
198 logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters })
199
200 return false
201 }
202
203 return true
204}
diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts
index fa1887315..35b05ec5a 100644
--- a/server/lib/auth/oauth.ts
+++ b/server/lib/auth/oauth.ts
@@ -9,11 +9,23 @@ import OAuth2Server, {
9 UnsupportedGrantTypeError 9 UnsupportedGrantTypeError
10} from '@node-oauth/oauth2-server' 10} from '@node-oauth/oauth2-server'
11import { randomBytesPromise } from '@server/helpers/core-utils' 11import { randomBytesPromise } from '@server/helpers/core-utils'
12import { isOTPValid } from '@server/helpers/otp'
12import { MOAuthClient } from '@server/types/models' 13import { MOAuthClient } from '@server/types/models'
13import { sha1 } from '@shared/extra-utils' 14import { sha1 } from '@shared/extra-utils'
14import { OAUTH_LIFETIME } from '../../initializers/constants' 15import { HttpStatusCode } from '@shared/models'
16import { OAUTH_LIFETIME, OTP } from '../../initializers/constants'
15import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' 17import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model'
16 18
19class MissingTwoFactorError extends Error {
20 code = HttpStatusCode.UNAUTHORIZED_401
21 name = 'missing_two_factor'
22}
23
24class InvalidTwoFactorError extends Error {
25 code = HttpStatusCode.BAD_REQUEST_400
26 name = 'invalid_two_factor'
27}
28
17/** 29/**
18 * 30 *
19 * Reimplement some functions of OAuth2Server to inject external auth methods 31 * Reimplement some functions of OAuth2Server to inject external auth methods
@@ -94,6 +106,9 @@ function handleOAuthAuthenticate (
94} 106}
95 107
96export { 108export {
109 MissingTwoFactorError,
110 InvalidTwoFactorError,
111
97 handleOAuthToken, 112 handleOAuthToken,
98 handleOAuthAuthenticate 113 handleOAuthAuthenticate
99} 114}
@@ -118,6 +133,16 @@ async function handlePasswordGrant (options: {
118 const user = await getUser(request.body.username, request.body.password, bypassLogin) 133 const user = await getUser(request.body.username, request.body.password, bypassLogin)
119 if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid') 134 if (!user) throw new InvalidGrantError('Invalid grant: user credentials are invalid')
120 135
136 if (user.otpSecret) {
137 if (!request.headers[OTP.HEADER_NAME]) {
138 throw new MissingTwoFactorError('Missing two factor header')
139 }
140
141 if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) {
142 throw new InvalidTwoFactorError('Invalid two factor header')
143 }
144 }
145
121 const token = await buildToken() 146 const token = await buildToken()
122 147
123 return saveToken(token, client, user, { bypassLogin }) 148 return saveToken(token, client, user, { bypassLogin })
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index a0a5afc0f..a41f1ae48 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -15,7 +15,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
15import { sequelizeTypescript } from '../initializers/database' 15import { sequelizeTypescript } from '../initializers/database'
16import { VideoFileModel } from '../models/video/video-file' 16import { VideoFileModel } from '../models/video/video-file'
17import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 17import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
18import { storeHLSFile } from './object-storage' 18import { storeHLSFileFromFilename } from './object-storage'
19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' 19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
20import { VideoPathManager } from './video-path-manager' 20import { VideoPathManager } from './video-path-manager'
21 21
@@ -95,7 +95,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist
95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
96 96
97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) { 97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
98 playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) 98 playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
99 await remove(masterPlaylistPath) 99 await remove(masterPlaylistPath)
100 } 100 }
101 101
@@ -146,7 +146,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist
146 await outputJSON(outputPath, json) 146 await outputJSON(outputPath, json)
147 147
148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) { 148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
149 playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) 149 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
150 await remove(outputPath) 150 await remove(outputPath)
151 } 151 }
152 152
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index 25bdebeea..28c3d325d 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -5,7 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { updateTorrentMetadata } from '@server/helpers/webtorrent' 5import { updateTorrentMetadata } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' 7import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
8import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' 8import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 9import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' 10import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
@@ -88,10 +88,10 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
88 88
89 // Resolution playlist 89 // Resolution playlist
90 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 90 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
91 await storeHLSFile(playlistWithVideo, playlistFilename) 91 await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
92 92
93 // Resolution fragmented file 93 // Resolution fragmented file
94 const fileUrl = await storeHLSFile(playlistWithVideo, file.filename) 94 const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
95 95
96 const oldPath = join(getHLSDirectory(video), file.filename) 96 const oldPath = join(getHLSDirectory(video), file.filename)
97 97
@@ -113,9 +113,9 @@ async function doAfterLastJob (options: {
113 const playlistWithVideo = playlist.withVideo(video) 113 const playlistWithVideo = playlist.withVideo(video)
114 114
115 // Master playlist 115 // Master playlist
116 playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename) 116 playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
117 // Sha256 segments file 117 // Sha256 segments file
118 playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename) 118 playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
119 119
120 playlist.storage = VideoStorage.OBJECT_STORAGE 120 playlist.storage = VideoStorage.OBJECT_STORAGE
121 121
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
index 600292844..c3dd8a688 100644
--- a/server/lib/job-queue/handlers/video-channel-import.ts
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel' 5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
7import { MChannelSync } from '@server/types/models' 7import { MChannelSync } from '@server/types/models'
8import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' 8import { VideoChannelImportPayload } from '@shared/models'
9 9
10export async function processVideoChannelImport (job: Job) { 10export async function processVideoChannelImport (job: Job) {
11 const payload = job.data as VideoChannelImportPayload 11 const payload = job.data as VideoChannelImportPayload
@@ -32,17 +32,11 @@ export async function processVideoChannelImport (job: Job) {
32 32
33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) 33 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
34 34
35 try { 35 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
36 logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) 36
37 37 await synchronizeChannel({
38 await synchronizeChannel({ 38 channel: videoChannel,
39 channel: videoChannel, 39 externalChannelUrl: payload.externalChannelUrl,
40 externalChannelUrl: payload.externalChannelUrl, 40 channelSync
41 channelSync 41 })
42 })
43 } catch (err) {
44 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
45 channelSync.state = VideoChannelSyncState.FAILED
46 await channelSync.save()
47 }
48} 42}
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 8a3ee09a2..7dbffc955 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -4,7 +4,7 @@ import { join } from 'path'
4import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' 4import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg'
5import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 5import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
7import { cleanupPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' 7import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
8import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' 8import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
9import { generateVideoMiniature } from '@server/lib/thumbnail' 9import { generateVideoMiniature } from '@server/lib/thumbnail'
10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' 10import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding'
@@ -34,13 +34,13 @@ async function processVideoLiveEnding (job: Job) {
34 const live = await VideoLiveModel.loadByVideoId(payload.videoId) 34 const live = await VideoLiveModel.loadByVideoId(payload.videoId)
35 const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) 35 const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId)
36 36
37 const permanentLive = live.permanentLive
38
39 if (!video || !live || !liveSession) { 37 if (!video || !live || !liveSession) {
40 logError() 38 logError()
41 return 39 return
42 } 40 }
43 41
42 const permanentLive = live.permanentLive
43
44 liveSession.endingProcessed = true 44 liveSession.endingProcessed = true
45 await liveSession.save() 45 await liveSession.save()
46 46
@@ -141,23 +141,22 @@ async function replaceLiveByReplay (options: {
141}) { 141}) {
142 const { video, liveSession, live, permanentLive, replayDirectory } = options 142 const { video, liveSession, live, permanentLive, replayDirectory } = options
143 143
144 await cleanupTMPLiveFiles(video) 144 const videoWithFiles = await VideoModel.loadFull(video.id)
145 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
146
147 await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist)
145 148
146 await live.destroy() 149 await live.destroy()
147 150
148 video.isLive = false 151 videoWithFiles.isLive = false
149 video.waitTranscoding = true 152 videoWithFiles.waitTranscoding = true
150 video.state = VideoState.TO_TRANSCODE 153 videoWithFiles.state = VideoState.TO_TRANSCODE
151 154
152 await video.save() 155 await videoWithFiles.save()
153 156
154 liveSession.replayVideoId = video.id 157 liveSession.replayVideoId = videoWithFiles.id
155 await liveSession.save() 158 await liveSession.save()
156 159
157 // Remove old HLS playlist video files
158 const videoWithFiles = await VideoModel.loadFull(video.id)
159
160 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
161 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) 160 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
162 161
163 // Reset playlist 162 // Reset playlist
@@ -234,7 +233,7 @@ async function cleanupLiveAndFederate (options: {
234 233
235 if (streamingPlaylist) { 234 if (streamingPlaylist) {
236 if (permanentLive) { 235 if (permanentLive) {
237 await cleanupPermanentLive(video, streamingPlaylist) 236 await cleanupAndDestroyPermanentLive(video, streamingPlaylist)
238 } else { 237 } else {
239 await cleanupUnsavedNormalLive(video, streamingPlaylist) 238 await cleanupUnsavedNormalLive(video, streamingPlaylist)
240 } 239 }
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 16715862b..9470b530b 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -21,14 +21,14 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session'
21import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 21import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
22import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models' 22import { MStreamingPlaylistVideo, MVideo, MVideoLiveSession, MVideoLiveVideo } from '@server/types/models'
23import { pick, wait } from '@shared/core-utils' 23import { pick, wait } from '@shared/core-utils'
24import { LiveVideoError, VideoState, VideoStreamingPlaylistType } from '@shared/models' 24import { LiveVideoError, VideoState, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
25import { federateVideoIfNeeded } from '../activitypub/videos' 25import { federateVideoIfNeeded } from '../activitypub/videos'
26import { JobQueue } from '../job-queue' 26import { JobQueue } from '../job-queue'
27import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths' 27import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '../paths'
28import { PeerTubeSocket } from '../peertube-socket' 28import { PeerTubeSocket } from '../peertube-socket'
29import { Hooks } from '../plugins/hooks' 29import { Hooks } from '../plugins/hooks'
30import { LiveQuotaStore } from './live-quota-store' 30import { LiveQuotaStore } from './live-quota-store'
31import { cleanupPermanentLive } from './live-utils' 31import { cleanupAndDestroyPermanentLive } from './live-utils'
32import { MuxingSession } from './shared' 32import { MuxingSession } from './shared'
33 33
34const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') 34const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
@@ -224,7 +224,7 @@ class LiveManager {
224 if (oldStreamingPlaylist) { 224 if (oldStreamingPlaylist) {
225 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) 225 if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid)
226 226
227 await cleanupPermanentLive(video, oldStreamingPlaylist) 227 await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist)
228 } 228 }
229 229
230 this.videoSessions.set(video.id, sessionId) 230 this.videoSessions.set(video.id, sessionId)
@@ -301,7 +301,7 @@ class LiveManager {
301 ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) 301 ...pick(options, [ 'streamingPlaylist', 'inputUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ])
302 }) 302 })
303 303
304 muxingSession.on('master-playlist-created', () => this.publishAndFederateLive(videoLive, localLTags)) 304 muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags))
305 305
306 muxingSession.on('bad-socket-health', ({ videoId }) => { 306 muxingSession.on('bad-socket-health', ({ videoId }) => {
307 logger.error( 307 logger.error(
@@ -485,6 +485,10 @@ class LiveManager {
485 485
486 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) 486 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
487 487
488 playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED
489 ? VideoStorage.OBJECT_STORAGE
490 : VideoStorage.FILE_SYSTEM
491
488 return playlist.save() 492 return playlist.save()
489 } 493 }
490 494
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
index 4af6f3ebf..faf03dccf 100644
--- a/server/lib/live/live-segment-sha-store.ts
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -1,62 +1,73 @@
1import { writeJson } from 'fs-extra'
1import { basename } from 'path' 2import { basename } from 'path'
3import { mapToJSON } from '@server/helpers/core-utils'
2import { logger, loggerTagsFactory } from '@server/helpers/logger' 4import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { MStreamingPlaylistVideo } from '@server/types/models'
3import { buildSha256Segment } from '../hls' 6import { buildSha256Segment } from '../hls'
7import { storeHLSFileFromPath } from '../object-storage'
4 8
5const lTags = loggerTagsFactory('live') 9const lTags = loggerTagsFactory('live')
6 10
7class LiveSegmentShaStore { 11class LiveSegmentShaStore {
8 12
9 private static instance: LiveSegmentShaStore 13 private readonly segmentsSha256 = new Map<string, string>()
10 14
11 private readonly segmentsSha256 = new Map<string, Map<string, string>>() 15 private readonly videoUUID: string
12 16 private readonly sha256Path: string
13 private constructor () { 17 private readonly streamingPlaylist: MStreamingPlaylistVideo
18 private readonly sendToObjectStorage: boolean
19
20 constructor (options: {
21 videoUUID: string
22 sha256Path: string
23 streamingPlaylist: MStreamingPlaylistVideo
24 sendToObjectStorage: boolean
25 }) {
26 this.videoUUID = options.videoUUID
27 this.sha256Path = options.sha256Path
28 this.streamingPlaylist = options.streamingPlaylist
29 this.sendToObjectStorage = options.sendToObjectStorage
14 } 30 }
15 31
16 getSegmentsSha256 (videoUUID: string) { 32 async addSegmentSha (segmentPath: string) {
17 return this.segmentsSha256.get(videoUUID) 33 logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID))
18 }
19
20 async addSegmentSha (videoUUID: string, segmentPath: string) {
21 const segmentName = basename(segmentPath)
22 logger.debug('Adding live sha segment %s.', segmentPath, lTags(videoUUID))
23 34
24 const shaResult = await buildSha256Segment(segmentPath) 35 const shaResult = await buildSha256Segment(segmentPath)
25 36
26 if (!this.segmentsSha256.has(videoUUID)) { 37 const segmentName = basename(segmentPath)
27 this.segmentsSha256.set(videoUUID, new Map()) 38 this.segmentsSha256.set(segmentName, shaResult)
28 }
29 39
30 const filesMap = this.segmentsSha256.get(videoUUID) 40 await this.writeToDisk()
31 filesMap.set(segmentName, shaResult)
32 } 41 }
33 42
34 removeSegmentSha (videoUUID: string, segmentPath: string) { 43 async removeSegmentSha (segmentPath: string) {
35 const segmentName = basename(segmentPath) 44 const segmentName = basename(segmentPath)
36 45
37 logger.debug('Removing live sha segment %s.', segmentPath, lTags(videoUUID)) 46 logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID))
38 47
39 const filesMap = this.segmentsSha256.get(videoUUID) 48 if (!this.segmentsSha256.has(segmentName)) {
40 if (!filesMap) { 49 logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID))
41 logger.warn('Unknown files map to remove sha for %s.', videoUUID, lTags(videoUUID))
42 return 50 return
43 } 51 }
44 52
45 if (!filesMap.has(segmentName)) { 53 this.segmentsSha256.delete(segmentName)
46 logger.warn('Unknown segment in files map for video %s and segment %s.', videoUUID, segmentPath, lTags(videoUUID))
47 return
48 }
49 54
50 filesMap.delete(segmentName) 55 await this.writeToDisk()
51 } 56 }
52 57
53 cleanupShaSegments (videoUUID: string) { 58 private async writeToDisk () {
54 this.segmentsSha256.delete(videoUUID) 59 await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256))
55 }
56 60
57 static get Instance () { 61 if (this.sendToObjectStorage) {
58 return this.instance || (this.instance = new this()) 62 const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path)
63
64 if (this.streamingPlaylist.segmentsSha256Url !== url) {
65 this.streamingPlaylist.segmentsSha256Url = url
66 await this.streamingPlaylist.save()
67 }
68 }
59 } 69 }
70
60} 71}
61 72
62export { 73export {
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index bba876642..d2b8e3a55 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,9 +1,10 @@
1import { pathExists, readdir, remove } from 'fs-extra' 1import { pathExists, readdir, remove } from 'fs-extra'
2import { basename, join } from 'path' 2import { basename, join } from 'path'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { MStreamingPlaylist, MVideo } from '@server/types/models' 4import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
5import { VideoStorage } from '@shared/models'
6import { listHLSFileKeysOf, removeHLSFileObjectStorage, removeHLSObjectStorage } from '../object-storage'
5import { getLiveDirectory } from '../paths' 7import { getLiveDirectory } from '../paths'
6import { LiveSegmentShaStore } from './live-segment-sha-store'
7 8
8function buildConcatenatedName (segmentOrPlaylistPath: string) { 9function buildConcatenatedName (segmentOrPlaylistPath: string) {
9 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) 10 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
@@ -11,8 +12,8 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
11 return 'concat-' + num[1] + '.ts' 12 return 'concat-' + num[1] + '.ts'
12} 13}
13 14
14async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 15async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
15 await cleanupTMPLiveFiles(video) 16 await cleanupTMPLiveFiles(video, streamingPlaylist)
16 17
17 await streamingPlaylist.destroy() 18 await streamingPlaylist.destroy()
18} 19}
@@ -20,32 +21,51 @@ async function cleanupPermanentLive (video: MVideo, streamingPlaylist: MStreamin
20async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 21async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
21 const hlsDirectory = getLiveDirectory(video) 22 const hlsDirectory = getLiveDirectory(video)
22 23
24 // We uploaded files to object storage too, remove them
25 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
26 await removeHLSObjectStorage(streamingPlaylist.withVideo(video))
27 }
28
23 await remove(hlsDirectory) 29 await remove(hlsDirectory)
24 30
25 await streamingPlaylist.destroy() 31 await streamingPlaylist.destroy()
32}
26 33
27 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) 34async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
35 await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video))
36
37 await cleanupTMPLiveFilesFromFilesystem(video)
28} 38}
29 39
30async function cleanupTMPLiveFiles (video: MVideo) { 40export {
31 const hlsDirectory = getLiveDirectory(video) 41 cleanupAndDestroyPermanentLive,
42 cleanupUnsavedNormalLive,
43 cleanupTMPLiveFiles,
44 buildConcatenatedName
45}
46
47// ---------------------------------------------------------------------------
32 48
33 LiveSegmentShaStore.Instance.cleanupShaSegments(video.uuid) 49function isTMPLiveFile (name: string) {
50 return name.endsWith('.ts') ||
51 name.endsWith('.m3u8') ||
52 name.endsWith('.json') ||
53 name.endsWith('.mpd') ||
54 name.endsWith('.m4s') ||
55 name.endsWith('.tmp')
56}
57
58async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
59 const hlsDirectory = getLiveDirectory(video)
34 60
35 if (!await pathExists(hlsDirectory)) return 61 if (!await pathExists(hlsDirectory)) return
36 62
37 logger.info('Cleanup TMP live files of %s.', hlsDirectory) 63 logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory)
38 64
39 const files = await readdir(hlsDirectory) 65 const files = await readdir(hlsDirectory)
40 66
41 for (const filename of files) { 67 for (const filename of files) {
42 if ( 68 if (isTMPLiveFile(filename)) {
43 filename.endsWith('.ts') ||
44 filename.endsWith('.m3u8') ||
45 filename.endsWith('.mpd') ||
46 filename.endsWith('.m4s') ||
47 filename.endsWith('.tmp')
48 ) {
49 const p = join(hlsDirectory, filename) 69 const p = join(hlsDirectory, filename)
50 70
51 remove(p) 71 remove(p)
@@ -54,9 +74,14 @@ async function cleanupTMPLiveFiles (video: MVideo) {
54 } 74 }
55} 75}
56 76
57export { 77async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
58 cleanupPermanentLive, 78 if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
59 cleanupUnsavedNormalLive, 79
60 cleanupTMPLiveFiles, 80 const keys = await listHLSFileKeysOf(streamingPlaylist)
61 buildConcatenatedName 81
82 for (const key of keys) {
83 if (isTMPLiveFile(key)) {
84 await removeHLSFileObjectStorage(streamingPlaylist, key)
85 }
86 }
62} 87}
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index 505717dce..4c27d5dd8 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -9,8 +9,10 @@ import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers
9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' 9import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
10import { CONFIG } from '@server/initializers/config' 10import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { removeHLSFileObjectStorage, storeHLSFileFromFilename, storeHLSFileFromPath } from '@server/lib/object-storage'
12import { VideoFileModel } from '@server/models/video/video-file' 13import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 14import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
15import { VideoStorage } from '@shared/models'
14import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths' 16import { getLiveDirectory, getLiveReplayBaseDirectory } from '../../paths'
15import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' 17import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles'
16import { isAbleToUploadVideo } from '../../user' 18import { isAbleToUploadVideo } from '../../user'
@@ -21,7 +23,7 @@ import { buildConcatenatedName } from '../live-utils'
21import memoizee = require('memoizee') 23import memoizee = require('memoizee')
22 24
23interface MuxingSessionEvents { 25interface MuxingSessionEvents {
24 'master-playlist-created': (options: { videoId: number }) => void 26 'live-ready': (options: { videoId: number }) => void
25 27
26 'bad-socket-health': (options: { videoId: number }) => void 28 'bad-socket-health': (options: { videoId: number }) => void
27 'duration-exceeded': (options: { videoId: number }) => void 29 'duration-exceeded': (options: { videoId: number }) => void
@@ -68,12 +70,18 @@ class MuxingSession extends EventEmitter {
68 private readonly outDirectory: string 70 private readonly outDirectory: string
69 private readonly replayDirectory: string 71 private readonly replayDirectory: string
70 72
73 private readonly liveSegmentShaStore: LiveSegmentShaStore
74
71 private readonly lTags: LoggerTagsFn 75 private readonly lTags: LoggerTagsFn
72 76
73 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} 77 private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
74 78
75 private tsWatcher: FSWatcher 79 private tsWatcher: FSWatcher
76 private masterWatcher: FSWatcher 80 private masterWatcher: FSWatcher
81 private m3u8Watcher: FSWatcher
82
83 private masterPlaylistCreated = false
84 private liveReady = false
77 85
78 private aborted = false 86 private aborted = false
79 87
@@ -123,6 +131,13 @@ class MuxingSession extends EventEmitter {
123 this.outDirectory = getLiveDirectory(this.videoLive.Video) 131 this.outDirectory = getLiveDirectory(this.videoLive.Video)
124 this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) 132 this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString())
125 133
134 this.liveSegmentShaStore = new LiveSegmentShaStore({
135 videoUUID: this.videoLive.Video.uuid,
136 sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename),
137 streamingPlaylist: this.streamingPlaylist,
138 sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED
139 })
140
126 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) 141 this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID)
127 } 142 }
128 143
@@ -159,8 +174,9 @@ class MuxingSession extends EventEmitter {
159 174
160 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) 175 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags())
161 176
162 this.watchTSFiles()
163 this.watchMasterFile() 177 this.watchMasterFile()
178 this.watchTSFiles()
179 this.watchM3U8File()
164 180
165 let ffmpegShellCommand: string 181 let ffmpegShellCommand: string
166 this.ffmpegCommand.on('start', cmdline => { 182 this.ffmpegCommand.on('start', cmdline => {
@@ -219,7 +235,7 @@ class MuxingSession extends EventEmitter {
219 setTimeout(() => { 235 setTimeout(() => {
220 // Wait latest segments generation, and close watchers 236 // Wait latest segments generation, and close watchers
221 237
222 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close() ]) 238 Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ])
223 .then(() => { 239 .then(() => {
224 // Process remaining segments hash 240 // Process remaining segments hash
225 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { 241 for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
@@ -240,14 +256,41 @@ class MuxingSession extends EventEmitter {
240 private watchMasterFile () { 256 private watchMasterFile () {
241 this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) 257 this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
242 258
243 this.masterWatcher.on('add', () => { 259 this.masterWatcher.on('add', async () => {
244 this.emit('master-playlist-created', { videoId: this.videoId }) 260 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
261 try {
262 const url = await storeHLSFileFromFilename(this.streamingPlaylist, this.streamingPlaylist.playlistFilename)
263
264 this.streamingPlaylist.playlistUrl = url
265 await this.streamingPlaylist.save()
266 } catch (err) {
267 logger.error('Cannot upload live master file to object storage.', { err, ...this.lTags() })
268 }
269 }
270
271 this.masterPlaylistCreated = true
245 272
246 this.masterWatcher.close() 273 this.masterWatcher.close()
247 .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) 274 .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
248 }) 275 })
249 } 276 }
250 277
278 private watchM3U8File () {
279 this.m3u8Watcher = watch(this.outDirectory + '/*.m3u8')
280
281 const onChangeOrAdd = async (m3u8Path: string) => {
282 if (this.streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
283
284 try {
285 await storeHLSFileFromPath(this.streamingPlaylist, m3u8Path)
286 } catch (err) {
287 logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() })
288 }
289 }
290
291 this.m3u8Watcher.on('change', onChangeOrAdd)
292 }
293
251 private watchTSFiles () { 294 private watchTSFiles () {
252 const startStreamDateTime = new Date().getTime() 295 const startStreamDateTime = new Date().getTime()
253 296
@@ -282,7 +325,21 @@ class MuxingSession extends EventEmitter {
282 } 325 }
283 } 326 }
284 327
285 const deleteHandler = (segmentPath: string) => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath) 328 const deleteHandler = async (segmentPath: string) => {
329 try {
330 await this.liveSegmentShaStore.removeSegmentSha(segmentPath)
331 } catch (err) {
332 logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() })
333 }
334
335 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
336 try {
337 await removeHLSFileObjectStorage(this.streamingPlaylist, segmentPath)
338 } catch (err) {
339 logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() })
340 }
341 }
342 }
286 343
287 this.tsWatcher.on('add', p => addHandler(p)) 344 this.tsWatcher.on('add', p => addHandler(p))
288 this.tsWatcher.on('unlink', p => deleteHandler(p)) 345 this.tsWatcher.on('unlink', p => deleteHandler(p))
@@ -315,6 +372,7 @@ class MuxingSession extends EventEmitter {
315 extname: '.ts', 372 extname: '.ts',
316 infoHash: null, 373 infoHash: null,
317 fps: this.fps, 374 fps: this.fps,
375 storage: this.streamingPlaylist.storage,
318 videoStreamingPlaylistId: this.streamingPlaylist.id 376 videoStreamingPlaylistId: this.streamingPlaylist.id
319 }) 377 })
320 378
@@ -343,18 +401,36 @@ class MuxingSession extends EventEmitter {
343 } 401 }
344 402
345 private processSegments (segmentPaths: string[]) { 403 private processSegments (segmentPaths: string[]) {
346 mapSeries(segmentPaths, async previousSegment => { 404 mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment))
347 // Add sha hash of previous segments, because ffmpeg should have finished generating them 405 .catch(err => {
348 await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment) 406 if (this.aborted) return
407
408 logger.error('Cannot process segments', { err, ...this.lTags() })
409 })
410 }
349 411
350 if (this.saveReplay) { 412 private async processSegment (segmentPath: string) {
351 await this.addSegmentToReplay(previousSegment) 413 // Add sha hash of previous segments, because ffmpeg should have finished generating them
414 await this.liveSegmentShaStore.addSegmentSha(segmentPath)
415
416 if (this.saveReplay) {
417 await this.addSegmentToReplay(segmentPath)
418 }
419
420 if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
421 try {
422 await storeHLSFileFromPath(this.streamingPlaylist, segmentPath)
423 } catch (err) {
424 logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })
352 } 425 }
353 }).catch(err => { 426 }
354 if (this.aborted) return
355 427
356 logger.error('Cannot process segments', { err, ...this.lTags() }) 428 // Master playlist and segment JSON file are created, live is ready
357 }) 429 if (this.masterPlaylistCreated && !this.liveReady) {
430 this.liveReady = true
431
432 this.emit('live-ready', { videoId: this.videoId })
433 }
358 } 434 }
359 435
360 private hasClientSocketInBadHealth (sessionId: string) { 436 private hasClientSocketInBadHealth (sessionId: string) {
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index c23f5b6a6..3cc92ca30 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -1,4 +1,4 @@
1import { VideoUploadFile } from 'express' 1import express, { VideoUploadFile } from 'express'
2import { PathLike } from 'fs-extra' 2import { PathLike } from 'fs-extra'
3import { Transaction } from 'sequelize/types' 3import { Transaction } from 'sequelize/types'
4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' 4import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
@@ -13,18 +13,15 @@ import {
13 MAbuseFull, 13 MAbuseFull,
14 MAccountDefault, 14 MAccountDefault,
15 MAccountLight, 15 MAccountLight,
16 MComment,
16 MCommentAbuseAccountVideo, 17 MCommentAbuseAccountVideo,
17 MCommentOwnerVideo, 18 MCommentOwnerVideo,
18 MUser, 19 MUser,
19 MVideoAbuseVideoFull, 20 MVideoAbuseVideoFull,
20 MVideoAccountLightBlacklistAllFiles 21 MVideoAccountLightBlacklistAllFiles
21} from '@server/types/models' 22} from '@server/types/models'
22import { ActivityCreate } from '../../shared/models/activitypub'
23import { VideoObject } from '../../shared/models/activitypub/objects'
24import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
25import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' 23import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos'
26import { VideoCommentCreate } from '../../shared/models/videos/comment' 24import { VideoCommentCreate } from '../../shared/models/videos/comment'
27import { ActorModel } from '../models/actor/actor'
28import { UserModel } from '../models/user/user' 25import { UserModel } from '../models/user/user'
29import { VideoModel } from '../models/video/video' 26import { VideoModel } from '../models/video/video'
30import { VideoCommentModel } from '../models/video/video-comment' 27import { VideoCommentModel } from '../models/video/video-comment'
@@ -36,7 +33,9 @@ export type AcceptResult = {
36 errorMessage?: string 33 errorMessage?: string
37} 34}
38 35
39// Can be filtered by plugins 36// ---------------------------------------------------------------------------
37
38// Stub function that can be filtered by plugins
40function isLocalVideoAccepted (object: { 39function isLocalVideoAccepted (object: {
41 videoBody: VideoCreate 40 videoBody: VideoCreate
42 videoFile: VideoUploadFile 41 videoFile: VideoUploadFile
@@ -45,6 +44,9 @@ function isLocalVideoAccepted (object: {
45 return { accepted: true } 44 return { accepted: true }
46} 45}
47 46
47// ---------------------------------------------------------------------------
48
49// Stub function that can be filtered by plugins
48function isLocalLiveVideoAccepted (object: { 50function isLocalLiveVideoAccepted (object: {
49 liveVideoBody: LiveVideoCreate 51 liveVideoBody: LiveVideoCreate
50 user: UserModel 52 user: UserModel
@@ -52,7 +54,11 @@ function isLocalLiveVideoAccepted (object: {
52 return { accepted: true } 54 return { accepted: true }
53} 55}
54 56
57// ---------------------------------------------------------------------------
58
59// Stub function that can be filtered by plugins
55function isLocalVideoThreadAccepted (_object: { 60function isLocalVideoThreadAccepted (_object: {
61 req: express.Request
56 commentBody: VideoCommentCreate 62 commentBody: VideoCommentCreate
57 video: VideoModel 63 video: VideoModel
58 user: UserModel 64 user: UserModel
@@ -60,7 +66,9 @@ function isLocalVideoThreadAccepted (_object: {
60 return { accepted: true } 66 return { accepted: true }
61} 67}
62 68
69// Stub function that can be filtered by plugins
63function isLocalVideoCommentReplyAccepted (_object: { 70function isLocalVideoCommentReplyAccepted (_object: {
71 req: express.Request
64 commentBody: VideoCommentCreate 72 commentBody: VideoCommentCreate
65 parentComment: VideoCommentModel 73 parentComment: VideoCommentModel
66 video: VideoModel 74 video: VideoModel
@@ -69,22 +77,18 @@ function isLocalVideoCommentReplyAccepted (_object: {
69 return { accepted: true } 77 return { accepted: true }
70} 78}
71 79
72function isRemoteVideoAccepted (_object: { 80// ---------------------------------------------------------------------------
73 activity: ActivityCreate
74 videoAP: VideoObject
75 byActor: ActorModel
76}): AcceptResult {
77 return { accepted: true }
78}
79 81
82// Stub function that can be filtered by plugins
80function isRemoteVideoCommentAccepted (_object: { 83function isRemoteVideoCommentAccepted (_object: {
81 activity: ActivityCreate 84 comment: MComment
82 commentAP: VideoCommentObject
83 byActor: ActorModel
84}): AcceptResult { 85}): AcceptResult {
85 return { accepted: true } 86 return { accepted: true }
86} 87}
87 88
89// ---------------------------------------------------------------------------
90
91// Stub function that can be filtered by plugins
88function isPreImportVideoAccepted (object: { 92function isPreImportVideoAccepted (object: {
89 videoImportBody: VideoImportCreate 93 videoImportBody: VideoImportCreate
90 user: MUser 94 user: MUser
@@ -92,6 +96,7 @@ function isPreImportVideoAccepted (object: {
92 return { accepted: true } 96 return { accepted: true }
93} 97}
94 98
99// Stub function that can be filtered by plugins
95function isPostImportVideoAccepted (object: { 100function isPostImportVideoAccepted (object: {
96 videoFilePath: PathLike 101 videoFilePath: PathLike
97 videoFile: VideoFileModel 102 videoFile: VideoFileModel
@@ -100,6 +105,8 @@ function isPostImportVideoAccepted (object: {
100 return { accepted: true } 105 return { accepted: true }
101} 106}
102 107
108// ---------------------------------------------------------------------------
109
103async function createVideoAbuse (options: { 110async function createVideoAbuse (options: {
104 baseAbuse: FilteredModelAttributes<AbuseModel> 111 baseAbuse: FilteredModelAttributes<AbuseModel>
105 videoInstance: MVideoAccountLightBlacklistAllFiles 112 videoInstance: MVideoAccountLightBlacklistAllFiles
@@ -189,12 +196,13 @@ function createAccountAbuse (options: {
189 }) 196 })
190} 197}
191 198
199// ---------------------------------------------------------------------------
200
192export { 201export {
193 isLocalLiveVideoAccepted, 202 isLocalLiveVideoAccepted,
194 203
195 isLocalVideoAccepted, 204 isLocalVideoAccepted,
196 isLocalVideoThreadAccepted, 205 isLocalVideoThreadAccepted,
197 isRemoteVideoAccepted,
198 isRemoteVideoCommentAccepted, 206 isRemoteVideoCommentAccepted,
199 isLocalVideoCommentReplyAccepted, 207 isLocalVideoCommentReplyAccepted,
200 isPreImportVideoAccepted, 208 isPreImportVideoAccepted,
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index 16161362c..c131977e8 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -22,6 +22,24 @@ type BucketInfo = {
22 PREFIX?: string 22 PREFIX?: string
23} 23}
24 24
25async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
26 const s3Client = getClient()
27
28 const commandPrefix = bucketInfo.PREFIX + prefix
29 const listCommand = new ListObjectsV2Command({
30 Bucket: bucketInfo.BUCKET_NAME,
31 Prefix: commandPrefix
32 })
33
34 const listedObjects = await s3Client.send(listCommand)
35
36 if (isArray(listedObjects.Contents) !== true) return []
37
38 return listedObjects.Contents.map(c => c.Key)
39}
40
41// ---------------------------------------------------------------------------
42
25async function storeObject (options: { 43async function storeObject (options: {
26 inputPath: string 44 inputPath: string
27 objectStorageKey: string 45 objectStorageKey: string
@@ -36,6 +54,8 @@ async function storeObject (options: {
36 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) 54 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo })
37} 55}
38 56
57// ---------------------------------------------------------------------------
58
39async function removeObject (filename: string, bucketInfo: BucketInfo) { 59async function removeObject (filename: string, bucketInfo: BucketInfo) {
40 const command = new DeleteObjectCommand({ 60 const command = new DeleteObjectCommand({
41 Bucket: bucketInfo.BUCKET_NAME, 61 Bucket: bucketInfo.BUCKET_NAME,
@@ -89,6 +109,8 @@ async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
89 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) 109 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
90} 110}
91 111
112// ---------------------------------------------------------------------------
113
92async function makeAvailable (options: { 114async function makeAvailable (options: {
93 key: string 115 key: string
94 destination: string 116 destination: string
@@ -122,7 +144,8 @@ export {
122 storeObject, 144 storeObject,
123 removeObject, 145 removeObject,
124 removePrefix, 146 removePrefix,
125 makeAvailable 147 makeAvailable,
148 listKeysOfPrefix
126} 149}
127 150
128// --------------------------------------------------------------------------- 151// ---------------------------------------------------------------------------
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 66e738200..62aae248b 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,19 +1,35 @@
1import { join } from 'path' 1import { basename, join } from 'path'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config' 3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' 4import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 7import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8 8
9function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) { 9function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
10 return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
11}
12
13// ---------------------------------------------------------------------------
14
15function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
10 return storeObject({ 16 return storeObject({
11 inputPath: path ?? join(getHLSDirectory(playlist.Video), filename), 17 inputPath: join(getHLSDirectory(playlist.Video), filename),
12 objectStorageKey: generateHLSObjectStorageKey(playlist, filename), 18 objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
13 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 19 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
14 }) 20 })
15} 21}
16 22
23function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
24 return storeObject({
25 inputPath: path,
26 objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
27 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
28 })
29}
30
31// ---------------------------------------------------------------------------
32
17function storeWebTorrentFile (filename: string) { 33function storeWebTorrentFile (filename: string) {
18 return storeObject({ 34 return storeObject({
19 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), 35 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
@@ -22,6 +38,8 @@ function storeWebTorrentFile (filename: string) {
22 }) 38 })
23} 39}
24 40
41// ---------------------------------------------------------------------------
42
25function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { 43function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
26 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 44 return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
27} 45}
@@ -30,10 +48,14 @@ function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename
30 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 48 return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
31} 49}
32 50
51// ---------------------------------------------------------------------------
52
33function removeWebTorrentObjectStorage (videoFile: MVideoFile) { 53function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
34 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) 54 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
35} 55}
36 56
57// ---------------------------------------------------------------------------
58
37async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { 59async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
38 const key = generateHLSObjectStorageKey(playlist, filename) 60 const key = generateHLSObjectStorageKey(playlist, filename)
39 61
@@ -62,9 +84,14 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
62 return destination 84 return destination
63} 85}
64 86
87// ---------------------------------------------------------------------------
88
65export { 89export {
90 listHLSFileKeysOf,
91
66 storeWebTorrentFile, 92 storeWebTorrentFile,
67 storeHLSFile, 93 storeHLSFileFromFilename,
94 storeHLSFileFromPath,
68 95
69 removeHLSObjectStorage, 96 removeHLSObjectStorage,
70 removeHLSFileObjectStorage, 97 removeHLSFileObjectStorage,
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts
index 4e799b3d4..7b1def6e3 100644
--- a/server/lib/plugins/plugin-helpers-builder.ts
+++ b/server/lib/plugins/plugin-helpers-builder.ts
@@ -1,4 +1,5 @@
1import express from 'express' 1import express from 'express'
2import { Server } from 'http'
2import { join } from 'path' 3import { join } from 'path'
3import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' 4import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils'
4import { buildLogger } from '@server/helpers/logger' 5import { buildLogger } from '@server/helpers/logger'
@@ -13,15 +14,16 @@ import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
13import { UserModel } from '@server/models/user/user' 14import { UserModel } from '@server/models/user/user'
14import { VideoModel } from '@server/models/video/video' 15import { VideoModel } from '@server/models/video/video'
15import { VideoBlacklistModel } from '@server/models/video/video-blacklist' 16import { VideoBlacklistModel } from '@server/models/video/video-blacklist'
16import { MPlugin } from '@server/types/models' 17import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models'
17import { PeerTubeHelpers } from '@server/types/plugins' 18import { PeerTubeHelpers } from '@server/types/plugins'
18import { VideoBlacklistCreate, VideoStorage } from '@shared/models' 19import { VideoBlacklistCreate, VideoStorage } from '@shared/models'
19import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' 20import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist'
21import { PeerTubeSocket } from '../peertube-socket'
20import { ServerConfigManager } from '../server-config-manager' 22import { ServerConfigManager } from '../server-config-manager'
21import { blacklistVideo, unblacklistVideo } from '../video-blacklist' 23import { blacklistVideo, unblacklistVideo } from '../video-blacklist'
22import { VideoPathManager } from '../video-path-manager' 24import { VideoPathManager } from '../video-path-manager'
23 25
24function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHelpers { 26function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers {
25 const logger = buildPluginLogger(npmName) 27 const logger = buildPluginLogger(npmName)
26 28
27 const database = buildDatabaseHelpers() 29 const database = buildDatabaseHelpers()
@@ -29,12 +31,14 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel
29 31
30 const config = buildConfigHelpers() 32 const config = buildConfigHelpers()
31 33
32 const server = buildServerHelpers() 34 const server = buildServerHelpers(httpServer)
33 35
34 const moderation = buildModerationHelpers() 36 const moderation = buildModerationHelpers()
35 37
36 const plugin = buildPluginRelatedHelpers(pluginModel, npmName) 38 const plugin = buildPluginRelatedHelpers(pluginModel, npmName)
37 39
40 const socket = buildSocketHelpers()
41
38 const user = buildUserHelpers() 42 const user = buildUserHelpers()
39 43
40 return { 44 return {
@@ -45,6 +49,7 @@ function buildPluginHelpers (pluginModel: MPlugin, npmName: string): PeerTubeHel
45 moderation, 49 moderation,
46 plugin, 50 plugin,
47 server, 51 server,
52 socket,
48 user 53 user
49 } 54 }
50} 55}
@@ -65,8 +70,10 @@ function buildDatabaseHelpers () {
65 } 70 }
66} 71}
67 72
68function buildServerHelpers () { 73function buildServerHelpers (httpServer: Server) {
69 return { 74 return {
75 getHTTPServer: () => httpServer,
76
70 getServerActor: () => getServerActor() 77 getServerActor: () => getServerActor()
71 } 78 }
72} 79}
@@ -214,10 +221,23 @@ function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) {
214 221
215 getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, 222 getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`,
216 223
224 getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`,
225
217 getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) 226 getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName)
218 } 227 }
219} 228}
220 229
230function buildSocketHelpers () {
231 return {
232 sendNotification: (userId: number, notification: UserNotificationModelForApi) => {
233 PeerTubeSocket.Instance.sendNotification(userId, notification)
234 },
235 sendVideoLiveNewState: (video: MVideo) => {
236 PeerTubeSocket.Instance.sendVideoLiveNewState(video)
237 }
238 }
239}
240
221function buildUserHelpers () { 241function buildUserHelpers () {
222 return { 242 return {
223 loadById: (id: number) => { 243 loadById: (id: number) => {
diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts
index a46b97fa4..c4d9b6574 100644
--- a/server/lib/plugins/plugin-manager.ts
+++ b/server/lib/plugins/plugin-manager.ts
@@ -1,6 +1,7 @@
1import express from 'express' 1import express from 'express'
2import { createReadStream, createWriteStream } from 'fs' 2import { createReadStream, createWriteStream } from 'fs'
3import { ensureDir, outputFile, readJSON } from 'fs-extra' 3import { ensureDir, outputFile, readJSON } from 'fs-extra'
4import { Server } from 'http'
4import { basename, join } from 'path' 5import { basename, join } from 'path'
5import { decachePlugin } from '@server/helpers/decache' 6import { decachePlugin } from '@server/helpers/decache'
6import { ApplicationModel } from '@server/models/application/application' 7import { ApplicationModel } from '@server/models/application/application'
@@ -67,9 +68,37 @@ export class PluginManager implements ServerHook {
67 private hooks: { [name: string]: HookInformationValue[] } = {} 68 private hooks: { [name: string]: HookInformationValue[] } = {}
68 private translations: PluginLocalesTranslations = {} 69 private translations: PluginLocalesTranslations = {}
69 70
71 private server: Server
72
70 private constructor () { 73 private constructor () {
71 } 74 }
72 75
76 init (server: Server) {
77 this.server = server
78 }
79
80 registerWebSocketRouter () {
81 this.server.on('upgrade', (request, socket, head) => {
82 const url = request.url
83
84 const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`)
85 if (!matched) return
86
87 const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN)
88 const subRoute = matched[3]
89
90 const result = this.getRegisteredPluginOrTheme(npmName)
91 if (!result) return
92
93 const routes = result.registerHelpers.getWebSocketRoutes()
94
95 const wss = routes.find(r => r.route.startsWith(subRoute))
96 if (!wss) return
97
98 wss.handler(request, socket, head)
99 })
100 }
101
73 // ###################### Getters ###################### 102 // ###################### Getters ######################
74 103
75 isRegistered (npmName: string) { 104 isRegistered (npmName: string) {
@@ -581,7 +610,7 @@ export class PluginManager implements ServerHook {
581 }) 610 })
582 } 611 }
583 612
584 const registerHelpers = new RegisterHelpers(npmName, plugin, onHookAdded.bind(this)) 613 const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this))
585 614
586 return { 615 return {
587 registerStore: registerHelpers, 616 registerStore: registerHelpers,
diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts
index f4d405676..1aaef3606 100644
--- a/server/lib/plugins/register-helpers.ts
+++ b/server/lib/plugins/register-helpers.ts
@@ -1,4 +1,5 @@
1import express from 'express' 1import express from 'express'
2import { Server } from 'http'
2import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
3import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' 4import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth'
4import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' 5import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory'
@@ -8,7 +9,8 @@ import {
8 RegisterServerAuthExternalResult, 9 RegisterServerAuthExternalResult,
9 RegisterServerAuthPassOptions, 10 RegisterServerAuthPassOptions,
10 RegisterServerExternalAuthenticatedResult, 11 RegisterServerExternalAuthenticatedResult,
11 RegisterServerOptions 12 RegisterServerOptions,
13 RegisterServerWebSocketRouteOptions
12} from '@server/types/plugins' 14} from '@server/types/plugins'
13import { 15import {
14 EncoderOptionsBuilder, 16 EncoderOptionsBuilder,
@@ -49,12 +51,15 @@ export class RegisterHelpers {
49 51
50 private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] 52 private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = []
51 53
54 private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = []
55
52 private readonly router: express.Router 56 private readonly router: express.Router
53 private readonly videoConstantManagerFactory: VideoConstantManagerFactory 57 private readonly videoConstantManagerFactory: VideoConstantManagerFactory
54 58
55 constructor ( 59 constructor (
56 private readonly npmName: string, 60 private readonly npmName: string,
57 private readonly plugin: PluginModel, 61 private readonly plugin: PluginModel,
62 private readonly server: Server,
58 private readonly onHookAdded: (options: RegisterServerHookOptions) => void 63 private readonly onHookAdded: (options: RegisterServerHookOptions) => void
59 ) { 64 ) {
60 this.router = express.Router() 65 this.router = express.Router()
@@ -66,6 +71,7 @@ export class RegisterHelpers {
66 const registerSetting = this.buildRegisterSetting() 71 const registerSetting = this.buildRegisterSetting()
67 72
68 const getRouter = this.buildGetRouter() 73 const getRouter = this.buildGetRouter()
74 const registerWebSocketRoute = this.buildRegisterWebSocketRoute()
69 75
70 const settingsManager = this.buildSettingsManager() 76 const settingsManager = this.buildSettingsManager()
71 const storageManager = this.buildStorageManager() 77 const storageManager = this.buildStorageManager()
@@ -85,13 +91,14 @@ export class RegisterHelpers {
85 const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() 91 const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth()
86 const unregisterExternalAuth = this.buildUnregisterExternalAuth() 92 const unregisterExternalAuth = this.buildUnregisterExternalAuth()
87 93
88 const peertubeHelpers = buildPluginHelpers(this.plugin, this.npmName) 94 const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName)
89 95
90 return { 96 return {
91 registerHook, 97 registerHook,
92 registerSetting, 98 registerSetting,
93 99
94 getRouter, 100 getRouter,
101 registerWebSocketRoute,
95 102
96 settingsManager, 103 settingsManager,
97 storageManager, 104 storageManager,
@@ -180,10 +187,20 @@ export class RegisterHelpers {
180 return this.onSettingsChangeCallbacks 187 return this.onSettingsChangeCallbacks
181 } 188 }
182 189
190 getWebSocketRoutes () {
191 return this.webSocketRoutes
192 }
193
183 private buildGetRouter () { 194 private buildGetRouter () {
184 return () => this.router 195 return () => this.router
185 } 196 }
186 197
198 private buildRegisterWebSocketRoute () {
199 return (options: RegisterServerWebSocketRouteOptions) => {
200 this.webSocketRoutes.push(options)
201 }
202 }
203
187 private buildRegisterSetting () { 204 private buildRegisterSetting () {
188 return (options: RegisterServerSettingOptions) => { 205 return (options: RegisterServerSettingOptions) => {
189 this.settings.push(options) 206 this.settings.push(options)
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 9b3c72300..b7523492a 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -9,6 +9,7 @@ import {
9 CONTACT_FORM_LIFETIME, 9 CONTACT_FORM_LIFETIME,
10 RESUMABLE_UPLOAD_SESSION_LIFETIME, 10 RESUMABLE_UPLOAD_SESSION_LIFETIME,
11 TRACKER_RATE_LIMITS, 11 TRACKER_RATE_LIMITS,
12 TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
12 USER_EMAIL_VERIFY_LIFETIME, 13 USER_EMAIL_VERIFY_LIFETIME,
13 USER_PASSWORD_CREATE_LIFETIME, 14 USER_PASSWORD_CREATE_LIFETIME,
14 USER_PASSWORD_RESET_LIFETIME, 15 USER_PASSWORD_RESET_LIFETIME,
@@ -108,10 +109,24 @@ class Redis {
108 return this.removeValue(this.generateResetPasswordKey(userId)) 109 return this.removeValue(this.generateResetPasswordKey(userId))
109 } 110 }
110 111
111 async getResetPasswordLink (userId: number) { 112 async getResetPasswordVerificationString (userId: number) {
112 return this.getValue(this.generateResetPasswordKey(userId)) 113 return this.getValue(this.generateResetPasswordKey(userId))
113 } 114 }
114 115
116 /* ************ Two factor auth request ************ */
117
118 async setTwoFactorRequest (userId: number, otpSecret: string) {
119 const requestToken = await generateRandomString(32)
120
121 await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME)
122
123 return requestToken
124 }
125
126 async getTwoFactorRequestToken (userId: number, requestToken: string) {
127 return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken))
128 }
129
115 /* ************ Email verification ************ */ 130 /* ************ Email verification ************ */
116 131
117 async setVerifyEmailVerificationString (userId: number) { 132 async setVerifyEmailVerificationString (userId: number) {
@@ -342,6 +357,10 @@ class Redis {
342 return 'reset-password-' + userId 357 return 'reset-password-' + userId
343 } 358 }
344 359
360 private generateTwoFactorRequestKey (userId: number, token: string) {
361 return 'two-factor-request-' + userId + '-' + token
362 }
363
345 private generateVerifyEmailKey (userId: number) { 364 private generateVerifyEmailKey (userId: number) {
346 return 'verify-email-' + userId 365 return 'verify-email-' + userId
347 } 366 }
@@ -391,8 +410,8 @@ class Redis {
391 return JSON.parse(value) 410 return JSON.parse(value)
392 } 411 }
393 412
394 private setObject (key: string, value: { [ id: string ]: number | string }) { 413 private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) {
395 return this.setValue(key, JSON.stringify(value)) 414 return this.setValue(key, JSON.stringify(value), expirationMilliseconds)
396 } 415 }
397 416
398 private async setValue (key: string, value: string, expirationMilliseconds?: number) { 417 private async setValue (key: string, value: string, expirationMilliseconds?: number) {
diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
index a527f68b5..efb957fac 100644
--- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { VideoChannelModel } from '@server/models/video/video-channel' 3import { VideoChannelModel } from '@server/models/video/video-channel'
4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 4import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
5import { VideoChannelSyncState } from '@shared/models'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 5import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { synchronizeChannel } from '../sync-channel' 6import { synchronizeChannel } from '../sync-channel'
8import { AbstractScheduler } from './abstract-scheduler' 7import { AbstractScheduler } from './abstract-scheduler'
@@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
28 for (const sync of channelSyncs) { 27 for (const sync of channelSyncs) {
29 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) 28 const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
30 29
31 try { 30 logger.info(
32 logger.info( 31 'Creating video import jobs for "%s" sync with external channel "%s"',
33 'Creating video import jobs for "%s" sync with external channel "%s"', 32 channel.Actor.preferredUsername, sync.externalChannelUrl
34 channel.Actor.preferredUsername, sync.externalChannelUrl 33 )
35 ) 34
36 35 const onlyAfter = sync.lastSyncAt || sync.createdAt
37 const onlyAfter = sync.lastSyncAt || sync.createdAt 36
38 37 await synchronizeChannel({
39 await synchronizeChannel({ 38 channel,
40 channel, 39 externalChannelUrl: sync.externalChannelUrl,
41 externalChannelUrl: sync.externalChannelUrl, 40 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
42 videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, 41 channelSync: sync,
43 channelSync: sync, 42 onlyAfter
44 onlyAfter 43 })
45 })
46 } catch (err) {
47 logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
48 sync.state = VideoChannelSyncState.FAILED
49 await sync.save()
50 }
51 } 44 }
52 } 45 }
53 46
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index f91599c14..35af91429 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -24,56 +24,62 @@ export async function synchronizeChannel (options: {
24 await channelSync.save() 24 await channelSync.save()
25 } 25 }
26 26
27 const user = await UserModel.loadByChannelActorId(channel.actorId) 27 try {
28 const youtubeDL = new YoutubeDLWrapper( 28 const user = await UserModel.loadByChannelActorId(channel.actorId)
29 externalChannelUrl, 29 const youtubeDL = new YoutubeDLWrapper(
30 ServerConfigManager.Instance.getEnabledResolutions('vod'), 30 externalChannelUrl,
31 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION 31 ServerConfigManager.Instance.getEnabledResolutions('vod'),
32 ) 32 CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
33 33 )
34 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
35
36 logger.info(
37 'Fetched %d candidate URLs for sync channel %s.',
38 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
39 )
40
41 if (targetUrls.length === 0) {
42 if (channelSync) {
43 channelSync.state = VideoChannelSyncState.SYNCED
44 await channelSync.save()
45 }
46
47 return
48 }
49 34
50 const children: CreateJobArgument[] = [] 35 const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
51 36
52 for (const targetUrl of targetUrls) { 37 logger.info(
53 if (await skipImport(channel, targetUrl, onlyAfter)) continue 38 'Fetched %d candidate URLs for sync channel %s.',
39 targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
40 )
54 41
55 const { job } = await buildYoutubeDLImport({ 42 if (targetUrls.length === 0) {
56 user, 43 if (channelSync) {
57 channel, 44 channelSync.state = VideoChannelSyncState.SYNCED
58 targetUrl, 45 await channelSync.save()
59 channelSync,
60 importDataOverride: {
61 privacy: VideoPrivacy.PUBLIC
62 } 46 }
63 })
64 47
65 children.push(job) 48 return
66 } 49 }
50
51 const children: CreateJobArgument[] = []
52
53 for (const targetUrl of targetUrls) {
54 if (await skipImport(channel, targetUrl, onlyAfter)) continue
67 55
68 // Will update the channel sync status 56 const { job } = await buildYoutubeDLImport({
69 const parent: CreateJobArgument = { 57 user,
70 type: 'after-video-channel-import', 58 channel,
71 payload: { 59 targetUrl,
72 channelSyncId: channelSync?.id 60 channelSync,
61 importDataOverride: {
62 privacy: VideoPrivacy.PUBLIC
63 }
64 })
65
66 children.push(job)
73 } 67 }
74 }
75 68
76 await JobQueue.Instance.createJobWithChildren(parent, children) 69 // Will update the channel sync status
70 const parent: CreateJobArgument = {
71 type: 'after-video-channel-import',
72 payload: {
73 channelSyncId: channelSync?.id
74 }
75 }
76
77 await JobQueue.Instance.createJobWithChildren(parent, children)
78 } catch (err) {
79 logger.error(`Failed to import channel ${channel.name}`, { err })
80 channelSync.state = VideoChannelSyncState.FAILED
81 await channelSync.save()
82 }
77} 83}
78 84
79// --------------------------------------------------------------------------- 85// ---------------------------------------------------------------------------
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-12 12:16:27 +0000