diff options
Diffstat (limited to 'server/lib/schedulers/videos-redundancy-scheduler.ts')
-rw-r--r-- | server/lib/schedulers/videos-redundancy-scheduler.ts | 54 |
1 files changed, 35 insertions, 19 deletions
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index ebfd015b5..d9988157d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -14,7 +14,7 @@ import { | |||
14 | MVideoWithAllFiles | 14 | MVideoWithAllFiles |
15 | } from '@server/types/models' | 15 | } from '@server/types/models' |
16 | import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' | 16 | import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' |
17 | import { logger } from '../../helpers/logger' | 17 | import { logger, loggerTagsFactory } from '../../helpers/logger' |
18 | import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' | 18 | import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' |
19 | import { CONFIG } from '../../initializers/config' | 19 | import { CONFIG } from '../../initializers/config' |
20 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' | 20 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' |
@@ -27,6 +27,8 @@ import { removeVideoRedundancy } from '../redundancy' | |||
27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls' | 27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls' |
28 | import { AbstractScheduler } from './abstract-scheduler' | 28 | import { AbstractScheduler } from './abstract-scheduler' |
29 | 29 | ||
30 | const lTags = loggerTagsFactory('redundancy') | ||
31 | |||
30 | type CandidateToDuplicate = { | 32 | type CandidateToDuplicate = { |
31 | redundancy: VideosRedundancyStrategy | 33 | redundancy: VideosRedundancyStrategy |
32 | video: MVideoWithAllFiles | 34 | video: MVideoWithAllFiles |
@@ -54,7 +56,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
54 | const videoToDuplicate = await VideoModel.loadWithFiles(videoId) | 56 | const videoToDuplicate = await VideoModel.loadWithFiles(videoId) |
55 | 57 | ||
56 | if (!videoToDuplicate) { | 58 | if (!videoToDuplicate) { |
57 | logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) | 59 | logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags()) |
58 | return | 60 | return |
59 | } | 61 | } |
60 | 62 | ||
@@ -68,7 +70,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
68 | 70 | ||
69 | protected async internalExecute () { | 71 | protected async internalExecute () { |
70 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 72 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
71 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) | 73 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags()) |
72 | 74 | ||
73 | try { | 75 | try { |
74 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) | 76 | const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) |
@@ -84,15 +86,18 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
84 | await this.purgeCacheIfNeeded(candidateToDuplicate) | 86 | await this.purgeCacheIfNeeded(candidateToDuplicate) |
85 | 87 | ||
86 | if (await this.isTooHeavy(candidateToDuplicate)) { | 88 | if (await this.isTooHeavy(candidateToDuplicate)) { |
87 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 89 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid)) |
88 | continue | 90 | continue |
89 | } | 91 | } |
90 | 92 | ||
91 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy) | 93 | logger.info( |
94 | 'Will duplicate video %s in redundancy scheduler "%s".', | ||
95 | videoToDuplicate.url, redundancyConfig.strategy, lTags(videoToDuplicate.uuid) | ||
96 | ) | ||
92 | 97 | ||
93 | await this.createVideoRedundancies(candidateToDuplicate) | 98 | await this.createVideoRedundancies(candidateToDuplicate) |
94 | } catch (err) { | 99 | } catch (err) { |
95 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err }) | 100 | logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() }) |
96 | } | 101 | } |
97 | } | 102 | } |
98 | 103 | ||
@@ -120,15 +125,19 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
120 | 125 | ||
121 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it | 126 | // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it |
122 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { | 127 | if (!redundancyConfig || await this.isTooHeavy(candidate)) { |
123 | logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) | 128 | logger.info( |
129 | 'Destroying redundancy %s because the cache size %s is too heavy.', | ||
130 | redundancyModel.url, redundancyModel.strategy, lTags(candidate.video.uuid) | ||
131 | ) | ||
132 | |||
124 | await removeVideoRedundancy(redundancyModel) | 133 | await removeVideoRedundancy(redundancyModel) |
125 | } else { | 134 | } else { |
126 | await this.extendsRedundancy(redundancyModel) | 135 | await this.extendsRedundancy(redundancyModel) |
127 | } | 136 | } |
128 | } catch (err) { | 137 | } catch (err) { |
129 | logger.error( | 138 | logger.error( |
130 | 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel), | 139 | 'Cannot extend or remove expiration of %s video from our redundancy system.', |
131 | { err } | 140 | this.buildEntryLogId(redundancyModel), { err, ...lTags(redundancyModel.getVideoUUID()) } |
132 | ) | 141 | ) |
133 | } | 142 | } |
134 | } | 143 | } |
@@ -152,7 +161,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
152 | try { | 161 | try { |
153 | await removeVideoRedundancy(redundancyModel) | 162 | await removeVideoRedundancy(redundancyModel) |
154 | } catch (err) { | 163 | } catch (err) { |
155 | logger.error('Cannot remove redundancy %s from our redundancy system.', this.buildEntryLogId(redundancyModel)) | 164 | logger.error( |
165 | 'Cannot remove redundancy %s from our redundancy system.', | ||
166 | this.buildEntryLogId(redundancyModel), lTags(redundancyModel.getVideoUUID()) | ||
167 | ) | ||
156 | } | 168 | } |
157 | } | 169 | } |
158 | } | 170 | } |
@@ -176,7 +188,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
176 | const video = await this.loadAndRefreshVideo(data.video.url) | 188 | const video = await this.loadAndRefreshVideo(data.video.url) |
177 | 189 | ||
178 | if (!video) { | 190 | if (!video) { |
179 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url) | 191 | logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid)) |
180 | 192 | ||
181 | return | 193 | return |
182 | } | 194 | } |
@@ -218,7 +230,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
218 | 230 | ||
219 | const serverActor = await getServerActor() | 231 | const serverActor = await getServerActor() |
220 | 232 | ||
221 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) | 233 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) |
222 | 234 | ||
223 | const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) | 235 | const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) |
224 | const magnetUri = generateMagnetUri(video, file, trackerUrls) | 236 | const magnetUri = generateMagnetUri(video, file, trackerUrls) |
@@ -241,7 +253,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
241 | 253 | ||
242 | await sendCreateCacheFile(serverActor, video, createdModel) | 254 | await sendCreateCacheFile(serverActor, video, createdModel) |
243 | 255 | ||
244 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 256 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid)) |
245 | } | 257 | } |
246 | 258 | ||
247 | private async createStreamingPlaylistRedundancy ( | 259 | private async createStreamingPlaylistRedundancy ( |
@@ -260,7 +272,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
260 | const playlist = Object.assign(playlistArg, { Video: video }) | 272 | const playlist = Object.assign(playlistArg, { Video: video }) |
261 | const serverActor = await getServerActor() | 273 | const serverActor = await getServerActor() |
262 | 274 | ||
263 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) | 275 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) |
264 | 276 | ||
265 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | 277 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) |
266 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) | 278 | const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) |
@@ -282,11 +294,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
282 | 294 | ||
283 | await sendCreateCacheFile(serverActor, video, createdModel) | 295 | await sendCreateCacheFile(serverActor, video, createdModel) |
284 | 296 | ||
285 | logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url) | 297 | logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid)) |
286 | } | 298 | } |
287 | 299 | ||
288 | private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { | 300 | private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { |
289 | logger.info('Extending expiration of %s.', redundancy.url) | 301 | logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID())) |
290 | 302 | ||
291 | const serverActor = await getServerActor() | 303 | const serverActor = await getServerActor() |
292 | 304 | ||
@@ -317,10 +329,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
317 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { | 329 | private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { |
318 | const maxSize = candidateToDuplicate.redundancy.size | 330 | const maxSize = candidateToDuplicate.redundancy.size |
319 | 331 | ||
320 | const { totalUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) | 332 | const { totalUsed: used } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) |
321 | const totalWillDuplicate = totalUsed + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) | 333 | |
334 | const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) | ||
335 | const total = used + videoSize | ||
336 | |||
337 | logger.debug('Checking candidate size.', { used, videoSize, total, ...lTags(candidateToDuplicate.video.uuid) }) | ||
322 | 338 | ||
323 | return totalWillDuplicate > maxSize | 339 | return total > maxSize |
324 | } | 340 | } |
325 | 341 | ||
326 | private buildNewExpiration (expiresAfterMs: number) { | 342 | private buildNewExpiration (expiresAfterMs: number) { |