]>
Commit | Line | Data |
---|---|---|
c48e82b5 C |
1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' | |
3 | import { logger } from '../../helpers/logger' | |
3f6b6a56 | 4 | import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' |
c48e82b5 C |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | import { VideoFileModel } from '../../models/video/video-file' | |
7 | import { sortBy } from 'lodash' | |
8 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | |
9 | import { join } from 'path' | |
10 | import { rename } from 'fs-extra' | |
11 | import { getServerActor } from '../../helpers/utils' | |
12 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | |
13 | import { VideoModel } from '../../models/video/video' | |
14 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | |
15 | import { removeVideoRedundancy } from '../redundancy' | |
16 | import { isTestInstance } from '../../helpers/core-utils' | |
17 | ||
18 | export class VideosRedundancyScheduler extends AbstractScheduler { | |
19 | ||
20 | private static instance: AbstractScheduler | |
21 | private executing = false | |
22 | ||
23 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy | |
24 | ||
25 | private constructor () { | |
26 | super() | |
27 | } | |
28 | ||
29 | async execute () { | |
30 | if (this.executing) return | |
31 | ||
32 | this.executing = true | |
33 | ||
34 | for (const obj of CONFIG.REDUNDANCY.VIDEOS) { | |
c48e82b5 | 35 | try { |
3f6b6a56 | 36 | const videoToDuplicate = await this.findVideoToDuplicate(obj) |
c48e82b5 C |
37 | if (!videoToDuplicate) continue |
38 | ||
39 | const videoFiles = videoToDuplicate.VideoFiles | |
40 | videoFiles.forEach(f => f.Video = videoToDuplicate) | |
41 | ||
3f6b6a56 | 42 | if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { |
c48e82b5 C |
43 | if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
44 | continue | |
45 | } | |
46 | ||
47 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | |
48 | ||
49 | await this.createVideoRedundancy(obj.strategy, videoFiles) | |
50 | } catch (err) { | |
51 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | |
52 | } | |
53 | } | |
54 | ||
55 | const expired = await VideoRedundancyModel.listAllExpired() | |
56 | ||
57 | for (const m of expired) { | |
58 | logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m)) | |
59 | ||
60 | try { | |
61 | await m.destroy() | |
62 | } catch (err) { | |
63 | logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) | |
64 | } | |
65 | } | |
66 | ||
67 | this.executing = false | |
68 | } | |
69 | ||
70 | static get Instance () { | |
71 | return this.instance || (this.instance = new this()) | |
72 | } | |
73 | ||
3f6b6a56 C |
74 | private findVideoToDuplicate (cache: VideosRedundancy) { |
75 | if (cache.strategy === 'most-views') { | |
76 | return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | |
77 | } | |
78 | ||
79 | if (cache.strategy === 'trending') { | |
80 | return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | |
81 | } | |
b36f41ca | 82 | |
3f6b6a56 C |
83 | if (cache.strategy === 'recently-added') { |
84 | const minViews = (cache as RecentlyAddedStrategy).minViews | |
85 | return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) | |
86 | } | |
c48e82b5 C |
87 | } |
88 | ||
89 | private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { | |
90 | const serverActor = await getServerActor() | |
91 | ||
92 | for (const file of filesToDuplicate) { | |
93 | const existing = await VideoRedundancyModel.loadByFileId(file.id) | |
94 | if (existing) { | |
95 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy) | |
96 | ||
97 | existing.expiresOn = this.buildNewExpiration() | |
98 | await existing.save() | |
99 | ||
100 | await sendUpdateCacheFile(serverActor, existing) | |
101 | continue | |
102 | } | |
103 | ||
104 | // We need more attributes and check if the video still exists | |
105 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) | |
106 | if (!video) continue | |
107 | ||
108 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) | |
109 | ||
110 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | |
111 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | |
112 | ||
113 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import']) | |
114 | ||
115 | const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) | |
116 | await rename(tmpPath, destPath) | |
117 | ||
118 | const createdModel = await VideoRedundancyModel.create({ | |
119 | expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS), | |
120 | url: getVideoCacheFileActivityPubUrl(file), | |
121 | fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), | |
122 | strategy, | |
123 | videoFileId: file.id, | |
124 | actorId: serverActor.id | |
125 | }) | |
126 | createdModel.VideoFile = file | |
127 | ||
128 | await sendCreateCacheFile(serverActor, createdModel) | |
129 | } | |
130 | } | |
131 | ||
3f6b6a56 | 132 | private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { |
c48e82b5 C |
133 | const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) |
134 | ||
3f6b6a56 | 135 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) |
c48e82b5 C |
136 | |
137 | return totalDuplicated > maxSize | |
138 | } | |
139 | ||
140 | private buildNewExpiration () { | |
141 | return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS) | |
142 | } | |
143 | ||
144 | private buildEntryLogId (object: VideoRedundancyModel) { | |
145 | return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` | |
146 | } | |
147 | ||
148 | private getTotalFileSizes (files: VideoFileModel[]) { | |
149 | const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size | |
150 | ||
151 | return files.reduce(fileReducer, 0) | |
152 | } | |
153 | } |