diff options
Diffstat (limited to 'server/lib/schedulers/videos-redundancy-scheduler.ts')
-rw-r--r-- | server/lib/schedulers/videos-redundancy-scheduler.ts | 57 |
1 files changed, 38 insertions, 19 deletions
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 5f4aad66e..1e30f6ebc 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -3,7 +3,6 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } | |||
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | import { VideoFileModel } from '../../models/video/video-file' | ||
7 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | 6 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' |
8 | import { join } from 'path' | 7 | import { join } from 'path' |
9 | import { move } from 'fs-extra' | 8 | import { move } from 'fs-extra' |
@@ -12,16 +11,31 @@ import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | |||
12 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' | 11 | import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' |
13 | import { removeVideoRedundancy } from '../redundancy' | 12 | import { removeVideoRedundancy } from '../redundancy' |
14 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | 13 | import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
16 | import { VideoModel } from '../../models/video/video' | ||
17 | import { downloadPlaylistSegments } from '../hls' | 14 | import { downloadPlaylistSegments } from '../hls' |
18 | import { CONFIG } from '../../initializers/config' | 15 | import { CONFIG } from '../../initializers/config' |
16 | import { | ||
17 | MStreamingPlaylist, | ||
18 | MStreamingPlaylistVideo, | ||
19 | MVideoAccountLight, | ||
20 | MVideoFile, | ||
21 | MVideoFileVideo, | ||
22 | MVideoRedundancyFileVideo, | ||
23 | MVideoRedundancyStreamingPlaylistVideo, | ||
24 | MVideoRedundancyVideo, | ||
25 | MVideoWithAllFiles | ||
26 | } from '@server/typings/models' | ||
19 | 27 | ||
20 | type CandidateToDuplicate = { | 28 | type CandidateToDuplicate = { |
21 | redundancy: VideosRedundancy, | 29 | redundancy: VideosRedundancy, |
22 | video: VideoModel, | 30 | video: MVideoWithAllFiles, |
23 | files: VideoFileModel[], | 31 | files: MVideoFile[], |
24 | streamingPlaylists: VideoStreamingPlaylistModel[] | 32 | streamingPlaylists: MStreamingPlaylist[] |
33 | } | ||
34 | |||
35 | function isMVideoRedundancyFileVideo ( | ||
36 | o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo | ||
37 | ): o is MVideoRedundancyFileVideo { | ||
38 | return !!(o as MVideoRedundancyFileVideo).VideoFile | ||
25 | } | 39 | } |
26 | 40 | ||
27 | export class VideosRedundancyScheduler extends AbstractScheduler { | 41 | export class VideosRedundancyScheduler extends AbstractScheduler { |
@@ -102,7 +116,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
102 | } | 116 | } |
103 | } | 117 | } |
104 | 118 | ||
105 | private async extendsRedundancy (redundancyModel: VideoRedundancyModel) { | 119 | private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) { |
106 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 120 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
107 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration | 121 | // Redundancy strategy disabled, remove our redundancy instead of extending expiration |
108 | if (!redundancy) { | 122 | if (!redundancy) { |
@@ -172,7 +186,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
172 | } | 186 | } |
173 | } | 187 | } |
174 | 188 | ||
175 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) { | 189 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { |
190 | const file = fileArg as MVideoFileVideo | ||
176 | file.Video = video | 191 | file.Video = video |
177 | 192 | ||
178 | const serverActor = await getServerActor() | 193 | const serverActor = await getServerActor() |
@@ -187,7 +202,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
187 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) | 202 | const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) |
188 | await move(tmpPath, destPath, { overwrite: true }) | 203 | await move(tmpPath, destPath, { overwrite: true }) |
189 | 204 | ||
190 | const createdModel = await VideoRedundancyModel.create({ | 205 | const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ |
191 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 206 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), |
192 | url: getVideoCacheFileActivityPubUrl(file), | 207 | url: getVideoCacheFileActivityPubUrl(file), |
193 | fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), | 208 | fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), |
@@ -203,7 +218,12 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
203 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) | 218 | logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) |
204 | } | 219 | } |
205 | 220 | ||
206 | private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) { | 221 | private async createStreamingPlaylistRedundancy ( |
222 | redundancy: VideosRedundancy, | ||
223 | video: MVideoAccountLight, | ||
224 | playlistArg: MStreamingPlaylist | ||
225 | ) { | ||
226 | const playlist = playlistArg as MStreamingPlaylistVideo | ||
207 | playlist.Video = video | 227 | playlist.Video = video |
208 | 228 | ||
209 | const serverActor = await getServerActor() | 229 | const serverActor = await getServerActor() |
@@ -213,7 +233,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
213 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | 233 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) |
214 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | 234 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) |
215 | 235 | ||
216 | const createdModel = await VideoRedundancyModel.create({ | 236 | const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ |
217 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 237 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), |
218 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | 238 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), |
219 | fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), | 239 | fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), |
@@ -229,7 +249,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
229 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) | 249 | logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) |
230 | } | 250 | } |
231 | 251 | ||
232 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { | 252 | private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { |
233 | logger.info('Extending expiration of %s.', redundancy.url) | 253 | logger.info('Extending expiration of %s.', redundancy.url) |
234 | 254 | ||
235 | const serverActor = await getServerActor() | 255 | const serverActor = await getServerActor() |
@@ -243,7 +263,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
243 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { | 263 | private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { |
244 | while (await this.isTooHeavy(candidateToDuplicate)) { | 264 | while (await this.isTooHeavy(candidateToDuplicate)) { |
245 | const redundancy = candidateToDuplicate.redundancy | 265 | const redundancy = candidateToDuplicate.redundancy |
246 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | 266 | const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) |
247 | if (!toDelete) return | 267 | if (!toDelete) return |
248 | 268 | ||
249 | await removeVideoRedundancy(toDelete) | 269 | await removeVideoRedundancy(toDelete) |
@@ -263,19 +283,18 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
263 | return new Date(Date.now() + expiresAfterMs) | 283 | return new Date(Date.now() + expiresAfterMs) |
264 | } | 284 | } |
265 | 285 | ||
266 | private buildEntryLogId (object: VideoRedundancyModel) { | 286 | private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { |
267 | if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | 287 | if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` |
268 | 288 | ||
269 | return `${object.VideoStreamingPlaylist.playlistUrl}` | 289 | return `${object.VideoStreamingPlaylist.playlistUrl}` |
270 | } | 290 | } |
271 | 291 | ||
272 | private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) { | 292 | private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylist[]) { |
273 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | 293 | const fileReducer = (previous: number, current: MVideoFile) => previous + current.size |
274 | 294 | ||
275 | const totalSize = files.reduce(fileReducer, 0) | 295 | const totalSize = files.reduce(fileReducer, 0) |
276 | if (playlists.length === 0) return totalSize | ||
277 | 296 | ||
278 | return totalSize * playlists.length | 297 | return totalSize + (totalSize * playlists.length) |
279 | } | 298 | } |
280 | 299 | ||
281 | private async loadAndRefreshVideo (videoUrl: string) { | 300 | private async loadAndRefreshVideo (videoUrl: string) { |