aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/cache-file.ts4
-rw-r--r--server/lib/activitypub/videos.ts1
-rw-r--r--server/lib/job-queue/handlers/video-redundancy.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts16
-rw-r--r--server/lib/redundancy.ts8
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts1
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts59
7 files changed, 81 insertions, 28 deletions
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 65b2dcb49..8252e95e9 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) 13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14 14
15 return { 15 return {
16 expiresOn: new Date(cacheFileObject.expires), 16 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
17 url: cacheFileObject.id, 17 url: cacheFileObject.id,
18 fileUrl: url.href, 18 fileUrl: url.href,
19 strategy: null, 19 strategy: null,
@@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) 30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
31 31
32 return { 32 return {
33 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
34 url: cacheFileObject.id, 34 url: cacheFileObject.id,
35 fileUrl: url.href, 35 fileUrl: url.href,
36 strategy: null, 36 strategy: null,
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index ade93150f..7a9d5168b 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -639,7 +639,6 @@ async function videoActivityObjectToDBAttributes (videoChannel: MChannelId, vide
639 createdAt: new Date(videoObject.published), 639 createdAt: new Date(videoObject.published),
640 publishedAt: new Date(videoObject.published), 640 publishedAt: new Date(videoObject.published),
641 originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null, 641 originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
642 // FIXME: updatedAt does not seems to be considered by Sequelize
643 updatedAt: new Date(videoObject.updated), 642 updatedAt: new Date(videoObject.updated),
644 views: videoObject.views, 643 views: videoObject.views,
645 likes: 0, 644 likes: 0,
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts
new file mode 100644
index 000000000..319d7090e
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-redundancy.ts
@@ -0,0 +1,20 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'
4
5export type VideoRedundancyPayload = {
6 videoId: number
7}
8
9async function processVideoRedundancy (job: Bull.Job) {
10 const payload = job.data as VideoRedundancyPayload
11 logger.info('Processing video redundancy in job %d.', job.id)
12
13 return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 processVideoRedundancy
20}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ec601e9ea..a1c623b25 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViews } from './handlers/video-views' 13import { processVideosViews } from './handlers/video-views'
14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' 14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' 15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
16import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'
16 17
17type CreateJobArgument = 18type CreateJobArgument =
18 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 19 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -24,20 +25,21 @@ type CreateJobArgument =
24 { type: 'email', payload: EmailPayload } | 25 { type: 'email', payload: EmailPayload } |
25 { type: 'video-import', payload: VideoImportPayload } | 26 { type: 'video-import', payload: VideoImportPayload } |
26 { type: 'activitypub-refresher', payload: RefreshPayload } | 27 { type: 'activitypub-refresher', payload: RefreshPayload } |
27 { type: 'videos-views', payload: {} } 28 { type: 'videos-views', payload: {} } |
29 { type: 'video-redundancy', payload: VideoRedundancyPayload }
28 30
29const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = { 31const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
30 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 32 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
31 'activitypub-http-unicast': processActivityPubHttpUnicast, 33 'activitypub-http-unicast': processActivityPubHttpUnicast,
32 'activitypub-http-fetcher': processActivityPubHttpFetcher, 34 'activitypub-http-fetcher': processActivityPubHttpFetcher,
33 'activitypub-follow': processActivityPubFollow, 35 'activitypub-follow': processActivityPubFollow,
34 'video-file-import': processVideoFileImport, 36 'video-file-import': processVideoFileImport,
35 'video-transcoding': processVideoTranscoding, 37 'video-transcoding': processVideoTranscoding,
36 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
37 'email': processEmail, 38 'email': processEmail,
38 'video-import': processVideoImport, 39 'video-import': processVideoImport,
39 'videos-views': processVideosViews, 40 'videos-views': processVideosViews,
40 'activitypub-refresher': refreshAPObject 41 'activitypub-refresher': refreshAPObject,
42 'video-redundancy': processVideoRedundancy
41} 43}
42 44
43const jobTypes: JobType[] = [ 45const jobTypes: JobType[] = [
@@ -50,7 +52,8 @@ const jobTypes: JobType[] = [
50 'video-file-import', 52 'video-file-import',
51 'video-import', 53 'video-import',
52 'videos-views', 54 'videos-views',
53 'activitypub-refresher' 55 'activitypub-refresher',
56 'video-redundancy'
54] 57]
55 58
56class JobQueue { 59class JobQueue {
@@ -141,8 +144,7 @@ class JobQueue {
141 continue 144 continue
142 } 145 }
143 146
144 // FIXME: Bull queue typings does not have getJobs method 147 const jobs = await queue.getJobs([ state ], 0, start + count, asc)
145 const jobs = await (queue as any).getJobs(state, 0, start + count, asc)
146 results = results.concat(jobs) 148 results = results.concat(jobs)
147 } 149 }
148 150
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 1b4ecd7c0..78d84e02e 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?
13 await videoRedundancy.destroy({ transaction: t }) 13 await videoRedundancy.destroy({ transaction: t })
14} 14}
15 15
16async function removeRedundancyOf (serverId: number) { 16async function removeRedundanciesOfServer (serverId: number) {
17 const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) 17 const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId)
18 18
19 for (const redundancy of videosRedundancy) { 19 for (const redundancy of redundancies) {
20 await removeVideoRedundancy(redundancy) 20 await removeVideoRedundancy(redundancy)
21 } 21 }
22} 22}
@@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) {
24// --------------------------------------------------------------------------- 24// ---------------------------------------------------------------------------
25 25
26export { 26export {
27 removeRedundancyOf, 27 removeRedundanciesOfServer,
28 removeVideoRedundancy 28 removeVideoRedundancy
29} 29}
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 350a335d3..956780a77 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda
4import { retryTransactionWrapper } from '../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../helpers/database-utils'
5import { federateVideoIfNeeded } from '../activitypub' 5import { federateVideoIfNeeded } from '../activitypub'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { VideoPrivacy } from '../../../shared/models/videos'
8import { Notifier } from '../notifier' 7import { Notifier } from '../notifier'
9import { sequelizeTypescript } from '../../initializers/database' 8import { sequelizeTypescript } from '../../initializers/database'
10import { MVideoFullLight } from '@server/typings/models' 9import { MVideoFullLight } from '@server/typings/models'
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c1c91b656..6e61cbe7d 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' 2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' 6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
7import { join } from 'path' 7import { join } from 'path'
@@ -25,9 +25,10 @@ import {
25 MVideoWithAllFiles 25 MVideoWithAllFiles
26} from '@server/typings/models' 26} from '@server/typings/models'
27import { getVideoFilename } from '../video-paths' 27import { getVideoFilename } from '../video-paths'
28import { VideoModel } from '@server/models/video/video'
28 29
29type CandidateToDuplicate = { 30type CandidateToDuplicate = {
30 redundancy: VideosRedundancy, 31 redundancy: VideosRedundancyStrategy,
31 video: MVideoWithAllFiles, 32 video: MVideoWithAllFiles,
32 files: MVideoFile[], 33 files: MVideoFile[],
33 streamingPlaylists: MStreamingPlaylistFiles[] 34 streamingPlaylists: MStreamingPlaylistFiles[]
@@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo (
41 42
42export class VideosRedundancyScheduler extends AbstractScheduler { 43export class VideosRedundancyScheduler extends AbstractScheduler {
43 44
44 private static instance: AbstractScheduler 45 private static instance: VideosRedundancyScheduler
45 46
46 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL 47 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
47 48
@@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
49 super() 50 super()
50 } 51 }
51 52
53 async createManualRedundancy (videoId: number) {
54 const videoToDuplicate = await VideoModel.loadWithFiles(videoId)
55
56 if (!videoToDuplicate) {
57 logger.warn('Video to manually duplicate %d does not exist anymore.', videoId)
58 return
59 }
60
61 return this.createVideoRedundancies({
62 video: videoToDuplicate,
63 redundancy: null,
64 files: videoToDuplicate.VideoFiles,
65 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
66 })
67 }
68
52 protected async internalExecute () { 69 protected async internalExecute () {
53 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 70 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
54 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) 71 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
@@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
94 for (const redundancyModel of expired) { 111 for (const redundancyModel of expired) {
95 try { 112 try {
96 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 113 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
97 const candidate = { 114 const candidate: CandidateToDuplicate = {
98 redundancy: redundancyConfig, 115 redundancy: redundancyConfig,
99 video: null, 116 video: null,
100 files: [], 117 files: [],
@@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
140 } 157 }
141 } 158 }
142 159
143 private findVideoToDuplicate (cache: VideosRedundancy) { 160 private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
144 if (cache.strategy === 'most-views') { 161 if (cache.strategy === 'most-views') {
145 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 162 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
146 } 163 }
@@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
187 } 204 }
188 } 205 }
189 206
190 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { 207 private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) {
208 let strategy = 'manual'
209 let expiresOn: Date = null
210
211 if (redundancy) {
212 strategy = redundancy.strategy
213 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
214 }
215
191 const file = fileArg as MVideoFileVideo 216 const file = fileArg as MVideoFileVideo
192 file.Video = video 217 file.Video = video
193 218
194 const serverActor = await getServerActor() 219 const serverActor = await getServerActor()
195 220
196 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 221 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
197 222
198 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 223 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
199 const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) 224 const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs)
@@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
204 await move(tmpPath, destPath, { overwrite: true }) 229 await move(tmpPath, destPath, { overwrite: true })
205 230
206 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ 231 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
207 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 232 expiresOn,
208 url: getVideoCacheFileActivityPubUrl(file), 233 url: getVideoCacheFileActivityPubUrl(file),
209 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), 234 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
210 strategy: redundancy.strategy, 235 strategy,
211 videoFileId: file.id, 236 videoFileId: file.id,
212 actorId: serverActor.id 237 actorId: serverActor.id
213 }) 238 })
@@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
220 } 245 }
221 246
222 private async createStreamingPlaylistRedundancy ( 247 private async createStreamingPlaylistRedundancy (
223 redundancy: VideosRedundancy, 248 redundancy: VideosRedundancyStrategy,
224 video: MVideoAccountLight, 249 video: MVideoAccountLight,
225 playlistArg: MStreamingPlaylist 250 playlistArg: MStreamingPlaylist
226 ) { 251 ) {
252 let strategy = 'manual'
253 let expiresOn: Date = null
254
255 if (redundancy) {
256 strategy = redundancy.strategy
257 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
258 }
259
227 const playlist = playlistArg as MStreamingPlaylistVideo 260 const playlist = playlistArg as MStreamingPlaylistVideo
228 playlist.Video = video 261 playlist.Video = video
229 262
230 const serverActor = await getServerActor() 263 const serverActor = await getServerActor()
231 264
232 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) 265 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
233 266
234 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 267 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
235 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) 268 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
236 269
237 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ 270 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
238 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 271 expiresOn,
239 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), 272 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
240 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), 273 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
241 strategy: redundancy.strategy, 274 strategy,
242 videoStreamingPlaylistId: playlist.id, 275 videoStreamingPlaylistId: playlist.id,
243 actorId: serverActor.id 276 actorId: serverActor.id
244 }) 277 })