aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/schedulers/videos-redundancy-scheduler.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib/schedulers/videos-redundancy-scheduler.ts')
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts57
1 files changed, 38 insertions, 19 deletions
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 5f4aad66e..1e30f6ebc 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -3,7 +3,6 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER }
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file'
7import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 6import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
8import { join } from 'path' 7import { join } from 'path'
9import { move } from 'fs-extra' 8import { move } from 'fs-extra'
@@ -12,16 +11,31 @@ import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' 11import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
13import { removeVideoRedundancy } from '../redundancy' 12import { removeVideoRedundancy } from '../redundancy'
14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 13import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { VideoModel } from '../../models/video/video'
17import { downloadPlaylistSegments } from '../hls' 14import { downloadPlaylistSegments } from '../hls'
18import { CONFIG } from '../../initializers/config' 15import { CONFIG } from '../../initializers/config'
16import {
17 MStreamingPlaylist,
18 MStreamingPlaylistVideo,
19 MVideoAccountLight,
20 MVideoFile,
21 MVideoFileVideo,
22 MVideoRedundancyFileVideo,
23 MVideoRedundancyStreamingPlaylistVideo,
24 MVideoRedundancyVideo,
25 MVideoWithAllFiles
26} from '@server/typings/models'
19 27
20type CandidateToDuplicate = { 28type CandidateToDuplicate = {
21 redundancy: VideosRedundancy, 29 redundancy: VideosRedundancy,
22 video: VideoModel, 30 video: MVideoWithAllFiles,
23 files: VideoFileModel[], 31 files: MVideoFile[],
24 streamingPlaylists: VideoStreamingPlaylistModel[] 32 streamingPlaylists: MStreamingPlaylist[]
33}
34
35function isMVideoRedundancyFileVideo (
36 o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo
37): o is MVideoRedundancyFileVideo {
38 return !!(o as MVideoRedundancyFileVideo).VideoFile
25} 39}
26 40
27export class VideosRedundancyScheduler extends AbstractScheduler { 41export class VideosRedundancyScheduler extends AbstractScheduler {
@@ -102,7 +116,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
102 } 116 }
103 } 117 }
104 118
105 private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { 119 private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) {
106 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 120 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
107 // Redundancy strategy disabled, remove our redundancy instead of extending expiration 121 // Redundancy strategy disabled, remove our redundancy instead of extending expiration
108 if (!redundancy) { 122 if (!redundancy) {
@@ -172,7 +186,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
172 } 186 }
173 } 187 }
174 188
175 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { 189 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) {
190 const file = fileArg as MVideoFileVideo
176 file.Video = video 191 file.Video = video
177 192
178 const serverActor = await getServerActor() 193 const serverActor = await getServerActor()
@@ -187,7 +202,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
187 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) 202 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
188 await move(tmpPath, destPath, { overwrite: true }) 203 await move(tmpPath, destPath, { overwrite: true })
189 204
190 const createdModel = await VideoRedundancyModel.create({ 205 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
191 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 206 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
192 url: getVideoCacheFileActivityPubUrl(file), 207 url: getVideoCacheFileActivityPubUrl(file),
193 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), 208 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
@@ -203,7 +218,12 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
203 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) 218 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
204 } 219 }
205 220
206 private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { 221 private async createStreamingPlaylistRedundancy (
222 redundancy: VideosRedundancy,
223 video: MVideoAccountLight,
224 playlistArg: MStreamingPlaylist
225 ) {
226 const playlist = playlistArg as MStreamingPlaylistVideo
207 playlist.Video = video 227 playlist.Video = video
208 228
209 const serverActor = await getServerActor() 229 const serverActor = await getServerActor()
@@ -213,7 +233,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
213 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 233 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
214 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) 234 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
215 235
216 const createdModel = await VideoRedundancyModel.create({ 236 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
217 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 237 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
218 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), 238 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
219 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), 239 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
@@ -229,7 +249,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
229 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) 249 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
230 } 250 }
231 251
232 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { 252 private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
233 logger.info('Extending expiration of %s.', redundancy.url) 253 logger.info('Extending expiration of %s.', redundancy.url)
234 254
235 const serverActor = await getServerActor() 255 const serverActor = await getServerActor()
@@ -243,7 +263,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
243 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { 263 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
244 while (await this.isTooHeavy(candidateToDuplicate)) { 264 while (await this.isTooHeavy(candidateToDuplicate)) {
245 const redundancy = candidateToDuplicate.redundancy 265 const redundancy = candidateToDuplicate.redundancy
246 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) 266 const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime)
247 if (!toDelete) return 267 if (!toDelete) return
248 268
249 await removeVideoRedundancy(toDelete) 269 await removeVideoRedundancy(toDelete)
@@ -263,19 +283,18 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
263 return new Date(Date.now() + expiresAfterMs) 283 return new Date(Date.now() + expiresAfterMs)
264 } 284 }
265 285
266 private buildEntryLogId (object: VideoRedundancyModel) { 286 private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) {
267 if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 287 if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
268 288
269 return `${object.VideoStreamingPlaylist.playlistUrl}` 289 return `${object.VideoStreamingPlaylist.playlistUrl}`
270 } 290 }
271 291
272 private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { 292 private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylist[]) {
273 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size 293 const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
274 294
275 const totalSize = files.reduce(fileReducer, 0) 295 const totalSize = files.reduce(fileReducer, 0)
276 if (playlists.length === 0) return totalSize
277 296
278 return totalSize * playlists.length 297 return totalSize + (totalSize * playlists.length)
279 } 298 }
280 299
281 private async loadAndRefreshVideo (videoUrl: string) { 300 private async loadAndRefreshVideo (videoUrl: string) {