aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/server/follows.ts4
-rw-r--r--server/controllers/api/server/redundancy.ts84
-rw-r--r--server/controllers/api/server/stats.ts10
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts2
-rw-r--r--server/helpers/custom-validators/video-redundancies.ts12
-rw-r--r--server/helpers/webtorrent.ts4
-rw-r--r--server/initializers/config.ts4
-rw-r--r--server/initializers/constants.ts25
-rw-r--r--server/initializers/migrations/0475-redundancy-expires-on.ts27
-rw-r--r--server/lib/activitypub/cache-file.ts4
-rw-r--r--server/lib/job-queue/handlers/video-redundancy.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts13
-rw-r--r--server/lib/redundancy.ts8
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts1
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts59
-rw-r--r--server/middlewares/sort.ts23
-rw-r--r--server/middlewares/validators/redundancy.ts74
-rw-r--r--server/middlewares/validators/sort.ts3
-rw-r--r--server/models/redundancy/video-redundancy.ts186
-rw-r--r--server/tests/api/check-params/redundancy.ts141
-rw-r--r--server/tests/api/redundancy/index.ts1
-rw-r--r--server/tests/api/redundancy/manage-redundancy.ts373
-rw-r--r--server/tests/api/redundancy/redundancy.ts138
-rw-r--r--server/typings/models/video/video-file.ts5
-rw-r--r--server/typings/models/video/video-streaming-playlist.ts6
-rw-r--r--server/typings/models/video/video.ts13
26 files changed, 1137 insertions, 103 deletions
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index 29a403a43..c69de21fb 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -24,7 +24,7 @@ import {
24} from '../../../middlewares/validators' 24} from '../../../middlewares/validators'
25import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 25import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
26import { JobQueue } from '../../../lib/job-queue' 26import { JobQueue } from '../../../lib/job-queue'
27import { removeRedundancyOf } from '../../../lib/redundancy' 27import { removeRedundanciesOfServer } from '../../../lib/redundancy'
28import { sequelizeTypescript } from '../../../initializers/database' 28import { sequelizeTypescript } from '../../../initializers/database'
29import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' 29import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow'
30 30
@@ -153,7 +153,7 @@ async function removeFollowing (req: express.Request, res: express.Response) {
153 await server.save({ transaction: t }) 153 await server.save({ transaction: t })
154 154
155 // Async, could be long 155 // Async, could be long
156 removeRedundancyOf(server.id) 156 removeRedundanciesOfServer(server.id)
157 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) 157 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err))
158 158
159 await follow.destroy({ transaction: t }) 159 await follow.destroy({ transaction: t })
diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts
index 4ea6164a3..a11c1a74f 100644
--- a/server/controllers/api/server/redundancy.ts
+++ b/server/controllers/api/server/redundancy.ts
@@ -1,9 +1,24 @@
1import * as express from 'express' 1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users' 2import { UserRight } from '../../../../shared/models/users'
3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' 3import {
4import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' 4 asyncMiddleware,
5import { removeRedundancyOf } from '../../../lib/redundancy' 5 authenticate,
6 ensureUserHasRight,
7 paginationValidator,
8 setDefaultPagination,
9 setDefaultVideoRedundanciesSort,
10 videoRedundanciesSortValidator
11} from '../../../middlewares'
12import {
13 listVideoRedundanciesValidator,
14 updateServerRedundancyValidator,
15 addVideoRedundancyValidator,
16 removeVideoRedundancyValidator
17} from '../../../middlewares/validators/redundancy'
18import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy'
6import { logger } from '../../../helpers/logger' 19import { logger } from '../../../helpers/logger'
20import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy'
21import { JobQueue } from '@server/lib/job-queue'
7 22
8const serverRedundancyRouter = express.Router() 23const serverRedundancyRouter = express.Router()
9 24
@@ -14,6 +29,31 @@ serverRedundancyRouter.put('/redundancy/:host',
14 asyncMiddleware(updateRedundancy) 29 asyncMiddleware(updateRedundancy)
15) 30)
16 31
32serverRedundancyRouter.get('/redundancy/videos',
33 authenticate,
34 ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
35 listVideoRedundanciesValidator,
36 paginationValidator,
37 videoRedundanciesSortValidator,
38 setDefaultVideoRedundanciesSort,
39 setDefaultPagination,
40 asyncMiddleware(listVideoRedundancies)
41)
42
43serverRedundancyRouter.post('/redundancy/videos',
44 authenticate,
45 ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
46 addVideoRedundancyValidator,
47 asyncMiddleware(addVideoRedundancy)
48)
49
50serverRedundancyRouter.delete('/redundancy/videos/:redundancyId',
51 authenticate,
52 ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
53 removeVideoRedundancyValidator,
54 asyncMiddleware(removeVideoRedundancyController)
55)
56
17// --------------------------------------------------------------------------- 57// ---------------------------------------------------------------------------
18 58
19export { 59export {
@@ -22,6 +62,42 @@ export {
22 62
23// --------------------------------------------------------------------------- 63// ---------------------------------------------------------------------------
24 64
65async function listVideoRedundancies (req: express.Request, res: express.Response) {
66 const resultList = await VideoRedundancyModel.listForApi({
67 start: req.query.start,
68 count: req.query.count,
69 sort: req.query.sort,
70 target: req.query.target,
71 strategy: req.query.strategy
72 })
73
74 const result = {
75 total: resultList.total,
76 data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r))
77 }
78
79 return res.json(result)
80}
81
82async function addVideoRedundancy (req: express.Request, res: express.Response) {
83 const payload = {
84 videoId: res.locals.onlyVideo.id
85 }
86
87 await JobQueue.Instance.createJob({
88 type: 'video-redundancy',
89 payload
90 })
91
92 return res.sendStatus(204)
93}
94
95async function removeVideoRedundancyController (req: express.Request, res: express.Response) {
96 await removeVideoRedundancy(res.locals.videoRedundancy)
97
98 return res.sendStatus(204)
99}
100
25async function updateRedundancy (req: express.Request, res: express.Response) { 101async function updateRedundancy (req: express.Request, res: express.Response) {
26 const server = res.locals.server 102 const server = res.locals.server
27 103
@@ -30,7 +106,7 @@ async function updateRedundancy (req: express.Request, res: express.Response) {
30 await server.save() 106 await server.save()
31 107
32 // Async, could be long 108 // Async, could be long
33 removeRedundancyOf(server.id) 109 removeRedundanciesOfServer(server.id)
34 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) 110 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err }))
35 111
36 return res.sendStatus(204) 112 return res.sendStatus(204)
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts
index 3616c074d..6d508a481 100644
--- a/server/controllers/api/server/stats.ts
+++ b/server/controllers/api/server/stats.ts
@@ -10,6 +10,7 @@ import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
10import { cacheRoute } from '../../../middlewares/cache' 10import { cacheRoute } from '../../../middlewares/cache'
11import { VideoFileModel } from '../../../models/video/video-file' 11import { VideoFileModel } from '../../../models/video/video-file'
12import { CONFIG } from '../../../initializers/config' 12import { CONFIG } from '../../../initializers/config'
13import { VideoRedundancyStrategyWithManual } from '@shared/models'
13 14
14const statsRouter = express.Router() 15const statsRouter = express.Router()
15 16
@@ -25,8 +26,15 @@ async function getStats (req: express.Request, res: express.Response) {
25 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() 26 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
26 const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() 27 const { totalLocalVideoFilesSize } = await VideoFileModel.getStats()
27 28
29 const strategies: { strategy: VideoRedundancyStrategyWithManual, size: number }[] = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
30 .map(r => ({
31 strategy: r.strategy,
32 size: r.size
33 }))
34 strategies.push({ strategy: 'manual', size: null })
35
28 const videosRedundancyStats = await Promise.all( 36 const videosRedundancyStats = await Promise.all(
29 CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { 37 strategies.map(r => {
30 return VideoRedundancyModel.getStats(r.strategy) 38 return VideoRedundancyModel.getStats(r.strategy)
31 .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) 39 .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
32 }) 40 })
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index 21d5c53ca..c5b3b4d9f 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
6function isCacheFileObjectValid (object: CacheFileObject) { 6function isCacheFileObjectValid (object: CacheFileObject) {
7 return exists(object) && 7 return exists(object) &&
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 (object.expires === null || isDateValid(object.expires)) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts
new file mode 100644
index 000000000..50a559c4f
--- /dev/null
+++ b/server/helpers/custom-validators/video-redundancies.ts
@@ -0,0 +1,12 @@
1import { exists } from './misc'
2
3function isVideoRedundancyTarget (value: any) {
4 return exists(value) &&
5 (value === 'my-videos' || value === 'remote-videos')
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isVideoRedundancyTarget
12}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 3a99518c6..8a5d030df 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -9,12 +9,12 @@ import { promisify2 } from './core-utils'
9import { MVideo } from '@server/typings/models/video/video' 9import { MVideo } from '@server/typings/models/video/video'
10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' 10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file'
11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' 11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist'
12import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' 12import { WEBSERVER } from '@server/initializers/constants'
13import * as parseTorrent from 'parse-torrent' 13import * as parseTorrent from 'parse-torrent'
14import * as magnetUtil from 'magnet-uri' 14import * as magnetUtil from 'magnet-uri'
15import { isArray } from '@server/helpers/custom-validators/misc' 15import { isArray } from '@server/helpers/custom-validators/misc'
16import { extractVideo } from '@server/lib/videos' 16import { extractVideo } from '@server/lib/videos'
17import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 17import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'
18 18
19const createTorrentPromise = promisify2<string, any, any>(createTorrent) 19const createTorrentPromise = promisify2<string, any, any>(createTorrent)
20 20
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 7fd77f3e8..fd8bf09fc 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -1,6 +1,6 @@
1import { IConfig } from 'config' 1import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { VideosRedundancy } from '../../shared/models' 3import { VideosRedundancyStrategy } from '../../shared/models'
4// Do not use barrels, remain constants as independent as possible 4// Do not use barrels, remain constants as independent as possible
5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' 5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
@@ -304,7 +304,7 @@ function getLocalConfigFilePath () {
304 return join(dirname(configSources[ 0 ].name), filename + '.json') 304 return join(dirname(configSources[ 0 ].name), filename + '.json')
305} 305}
306 306
307function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { 307function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] {
308 if (!objs) return [] 308 if (!objs) return []
309 309
310 if (!Array.isArray(objs)) return objs 310 if (!Array.isArray(objs)) return objs
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 032f63c8f..e01ab8943 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
14 14
15// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
16 16
17const LAST_MIGRATION_VERSION = 470 17const LAST_MIGRATION_VERSION = 475
18 18
19// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
20 20
@@ -73,7 +73,9 @@ const SORTABLE_COLUMNS = {
73 73
74 PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], 74 PLUGINS: [ 'name', 'createdAt', 'updatedAt' ],
75 75
76 AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ] 76 AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ],
77
78 VIDEO_REDUNDANCIES: [ 'name' ]
77} 79}
78 80
79const OAUTH_LIFETIME = { 81const OAUTH_LIFETIME = {
@@ -117,45 +119,44 @@ const REMOTE_SCHEME = {
117 WS: 'wss' 119 WS: 'wss'
118} 120}
119 121
120// TODO: remove 'video-file' 122const JOB_ATTEMPTS: { [id in JobType]: number } = {
121const JOB_ATTEMPTS: { [id in (JobType | 'video-file')]: number } = {
122 'activitypub-http-broadcast': 5, 123 'activitypub-http-broadcast': 5,
123 'activitypub-http-unicast': 5, 124 'activitypub-http-unicast': 5,
124 'activitypub-http-fetcher': 5, 125 'activitypub-http-fetcher': 5,
125 'activitypub-follow': 5, 126 'activitypub-follow': 5,
126 'video-file-import': 1, 127 'video-file-import': 1,
127 'video-transcoding': 1, 128 'video-transcoding': 1,
128 'video-file': 1,
129 'video-import': 1, 129 'video-import': 1,
130 'email': 5, 130 'email': 5,
131 'videos-views': 1, 131 'videos-views': 1,
132 'activitypub-refresher': 1 132 'activitypub-refresher': 1,
133 'video-redundancy': 1
133} 134}
134const JOB_CONCURRENCY: { [id in (JobType | 'video-file')]: number } = { 135const JOB_CONCURRENCY: { [id in JobType]: number } = {
135 'activitypub-http-broadcast': 1, 136 'activitypub-http-broadcast': 1,
136 'activitypub-http-unicast': 5, 137 'activitypub-http-unicast': 5,
137 'activitypub-http-fetcher': 1, 138 'activitypub-http-fetcher': 1,
138 'activitypub-follow': 1, 139 'activitypub-follow': 1,
139 'video-file-import': 1, 140 'video-file-import': 1,
140 'video-transcoding': 1, 141 'video-transcoding': 1,
141 'video-file': 1,
142 'video-import': 1, 142 'video-import': 1,
143 'email': 5, 143 'email': 5,
144 'videos-views': 1, 144 'videos-views': 1,
145 'activitypub-refresher': 1 145 'activitypub-refresher': 1,
146 'video-redundancy': 1
146} 147}
147const JOB_TTL: { [id in (JobType | 'video-file')]: number } = { 148const JOB_TTL: { [id in JobType]: number } = {
148 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 149 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
149 'activitypub-http-unicast': 60000 * 10, // 10 minutes 150 'activitypub-http-unicast': 60000 * 10, // 10 minutes
150 'activitypub-http-fetcher': 60000 * 10, // 10 minutes 151 'activitypub-http-fetcher': 60000 * 10, // 10 minutes
151 'activitypub-follow': 60000 * 10, // 10 minutes 152 'activitypub-follow': 60000 * 10, // 10 minutes
152 'video-file-import': 1000 * 3600, // 1 hour 153 'video-file-import': 1000 * 3600, // 1 hour
153 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long 154 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
154 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
155 'video-import': 1000 * 3600 * 2, // hours 155 'video-import': 1000 * 3600 * 2, // hours
156 'email': 60000 * 10, // 10 minutes 156 'email': 60000 * 10, // 10 minutes
157 'videos-views': undefined, // Unlimited 157 'videos-views': undefined, // Unlimited
158 'activitypub-refresher': 60000 * 10 // 10 minutes 158 'activitypub-refresher': 60000 * 10, // 10 minutes
159 'video-redundancy': 1000 * 3600 * 3 // 3 hours
159} 160}
160const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 161const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
161 'videos-views': { 162 'videos-views': {
diff --git a/server/initializers/migrations/0475-redundancy-expires-on.ts b/server/initializers/migrations/0475-redundancy-expires-on.ts
new file mode 100644
index 000000000..7e392c8c0
--- /dev/null
+++ b/server/initializers/migrations/0475-redundancy-expires-on.ts
@@ -0,0 +1,27 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize,
7 db: any
8}): Promise<void> {
9 {
10 const data = {
11 type: Sequelize.DATE,
12 allowNull: true,
13 defaultValue: null
14 }
15
16 await utils.queryInterface.changeColumn('videoRedundancy', 'expiresOn', data)
17 }
18}
19
20function down (options) {
21 throw new Error('Not implemented.')
22}
23
24export {
25 up,
26 down
27}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 65b2dcb49..8252e95e9 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) 13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14 14
15 return { 15 return {
16 expiresOn: new Date(cacheFileObject.expires), 16 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
17 url: cacheFileObject.id, 17 url: cacheFileObject.id,
18 fileUrl: url.href, 18 fileUrl: url.href,
19 strategy: null, 19 strategy: null,
@@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) 30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
31 31
32 return { 32 return {
33 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
34 url: cacheFileObject.id, 34 url: cacheFileObject.id,
35 fileUrl: url.href, 35 fileUrl: url.href,
36 strategy: null, 36 strategy: null,
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts
new file mode 100644
index 000000000..319d7090e
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-redundancy.ts
@@ -0,0 +1,20 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'
4
5export type VideoRedundancyPayload = {
6 videoId: number
7}
8
9async function processVideoRedundancy (job: Bull.Job) {
10 const payload = job.data as VideoRedundancyPayload
11 logger.info('Processing video redundancy in job %d.', job.id)
12
13 return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 processVideoRedundancy
20}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ec601e9ea..8bbf58f2b 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViews } from './handlers/video-views' 13import { processVideosViews } from './handlers/video-views'
14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' 14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' 15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
16import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'
16 17
17type CreateJobArgument = 18type CreateJobArgument =
18 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 19 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -24,20 +25,21 @@ type CreateJobArgument =
24 { type: 'email', payload: EmailPayload } | 25 { type: 'email', payload: EmailPayload } |
25 { type: 'video-import', payload: VideoImportPayload } | 26 { type: 'video-import', payload: VideoImportPayload } |
26 { type: 'activitypub-refresher', payload: RefreshPayload } | 27 { type: 'activitypub-refresher', payload: RefreshPayload } |
27 { type: 'videos-views', payload: {} } 28 { type: 'videos-views', payload: {} } |
29 { type: 'video-redundancy', payload: VideoRedundancyPayload }
28 30
29const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = { 31const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
30 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 32 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
31 'activitypub-http-unicast': processActivityPubHttpUnicast, 33 'activitypub-http-unicast': processActivityPubHttpUnicast,
32 'activitypub-http-fetcher': processActivityPubHttpFetcher, 34 'activitypub-http-fetcher': processActivityPubHttpFetcher,
33 'activitypub-follow': processActivityPubFollow, 35 'activitypub-follow': processActivityPubFollow,
34 'video-file-import': processVideoFileImport, 36 'video-file-import': processVideoFileImport,
35 'video-transcoding': processVideoTranscoding, 37 'video-transcoding': processVideoTranscoding,
36 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
37 'email': processEmail, 38 'email': processEmail,
38 'video-import': processVideoImport, 39 'video-import': processVideoImport,
39 'videos-views': processVideosViews, 40 'videos-views': processVideosViews,
40 'activitypub-refresher': refreshAPObject 41 'activitypub-refresher': refreshAPObject,
42 'video-redundancy': processVideoRedundancy
41} 43}
42 44
43const jobTypes: JobType[] = [ 45const jobTypes: JobType[] = [
@@ -50,7 +52,8 @@ const jobTypes: JobType[] = [
50 'video-file-import', 52 'video-file-import',
51 'video-import', 53 'video-import',
52 'videos-views', 54 'videos-views',
53 'activitypub-refresher' 55 'activitypub-refresher',
56 'video-redundancy'
54] 57]
55 58
56class JobQueue { 59class JobQueue {
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 1b4ecd7c0..78d84e02e 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?
13 await videoRedundancy.destroy({ transaction: t }) 13 await videoRedundancy.destroy({ transaction: t })
14} 14}
15 15
16async function removeRedundancyOf (serverId: number) { 16async function removeRedundanciesOfServer (serverId: number) {
17 const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) 17 const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId)
18 18
19 for (const redundancy of videosRedundancy) { 19 for (const redundancy of redundancies) {
20 await removeVideoRedundancy(redundancy) 20 await removeVideoRedundancy(redundancy)
21 } 21 }
22} 22}
@@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) {
24// --------------------------------------------------------------------------- 24// ---------------------------------------------------------------------------
25 25
26export { 26export {
27 removeRedundancyOf, 27 removeRedundanciesOfServer,
28 removeVideoRedundancy 28 removeVideoRedundancy
29} 29}
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 350a335d3..956780a77 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda
4import { retryTransactionWrapper } from '../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../helpers/database-utils'
5import { federateVideoIfNeeded } from '../activitypub' 5import { federateVideoIfNeeded } from '../activitypub'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { VideoPrivacy } from '../../../shared/models/videos'
8import { Notifier } from '../notifier' 7import { Notifier } from '../notifier'
9import { sequelizeTypescript } from '../../initializers/database' 8import { sequelizeTypescript } from '../../initializers/database'
10import { MVideoFullLight } from '@server/typings/models' 9import { MVideoFullLight } from '@server/typings/models'
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c1c91b656..6e61cbe7d 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' 2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' 6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
7import { join } from 'path' 7import { join } from 'path'
@@ -25,9 +25,10 @@ import {
25 MVideoWithAllFiles 25 MVideoWithAllFiles
26} from '@server/typings/models' 26} from '@server/typings/models'
27import { getVideoFilename } from '../video-paths' 27import { getVideoFilename } from '../video-paths'
28import { VideoModel } from '@server/models/video/video'
28 29
29type CandidateToDuplicate = { 30type CandidateToDuplicate = {
30 redundancy: VideosRedundancy, 31 redundancy: VideosRedundancyStrategy,
31 video: MVideoWithAllFiles, 32 video: MVideoWithAllFiles,
32 files: MVideoFile[], 33 files: MVideoFile[],
33 streamingPlaylists: MStreamingPlaylistFiles[] 34 streamingPlaylists: MStreamingPlaylistFiles[]
@@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo (
41 42
42export class VideosRedundancyScheduler extends AbstractScheduler { 43export class VideosRedundancyScheduler extends AbstractScheduler {
43 44
44 private static instance: AbstractScheduler 45 private static instance: VideosRedundancyScheduler
45 46
46 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL 47 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
47 48
@@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
49 super() 50 super()
50 } 51 }
51 52
53 async createManualRedundancy (videoId: number) {
54 const videoToDuplicate = await VideoModel.loadWithFiles(videoId)
55
56 if (!videoToDuplicate) {
57 logger.warn('Video to manually duplicate %d does not exist anymore.', videoId)
58 return
59 }
60
61 return this.createVideoRedundancies({
62 video: videoToDuplicate,
63 redundancy: null,
64 files: videoToDuplicate.VideoFiles,
65 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
66 })
67 }
68
52 protected async internalExecute () { 69 protected async internalExecute () {
53 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 70 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
54 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) 71 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
@@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
94 for (const redundancyModel of expired) { 111 for (const redundancyModel of expired) {
95 try { 112 try {
96 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 113 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
97 const candidate = { 114 const candidate: CandidateToDuplicate = {
98 redundancy: redundancyConfig, 115 redundancy: redundancyConfig,
99 video: null, 116 video: null,
100 files: [], 117 files: [],
@@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
140 } 157 }
141 } 158 }
142 159
143 private findVideoToDuplicate (cache: VideosRedundancy) { 160 private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
144 if (cache.strategy === 'most-views') { 161 if (cache.strategy === 'most-views') {
145 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 162 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
146 } 163 }
@@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
187 } 204 }
188 } 205 }
189 206
190 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { 207 private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) {
208 let strategy = 'manual'
209 let expiresOn: Date = null
210
211 if (redundancy) {
212 strategy = redundancy.strategy
213 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
214 }
215
191 const file = fileArg as MVideoFileVideo 216 const file = fileArg as MVideoFileVideo
192 file.Video = video 217 file.Video = video
193 218
194 const serverActor = await getServerActor() 219 const serverActor = await getServerActor()
195 220
196 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 221 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
197 222
198 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 223 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
199 const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) 224 const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs)
@@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
204 await move(tmpPath, destPath, { overwrite: true }) 229 await move(tmpPath, destPath, { overwrite: true })
205 230
206 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ 231 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
207 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 232 expiresOn,
208 url: getVideoCacheFileActivityPubUrl(file), 233 url: getVideoCacheFileActivityPubUrl(file),
209 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), 234 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
210 strategy: redundancy.strategy, 235 strategy,
211 videoFileId: file.id, 236 videoFileId: file.id,
212 actorId: serverActor.id 237 actorId: serverActor.id
213 }) 238 })
@@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
220 } 245 }
221 246
222 private async createStreamingPlaylistRedundancy ( 247 private async createStreamingPlaylistRedundancy (
223 redundancy: VideosRedundancy, 248 redundancy: VideosRedundancyStrategy,
224 video: MVideoAccountLight, 249 video: MVideoAccountLight,
225 playlistArg: MStreamingPlaylist 250 playlistArg: MStreamingPlaylist
226 ) { 251 ) {
252 let strategy = 'manual'
253 let expiresOn: Date = null
254
255 if (redundancy) {
256 strategy = redundancy.strategy
257 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
258 }
259
227 const playlist = playlistArg as MStreamingPlaylistVideo 260 const playlist = playlistArg as MStreamingPlaylistVideo
228 playlist.Video = video 261 playlist.Video = video
229 262
230 const serverActor = await getServerActor() 263 const serverActor = await getServerActor()
231 264
232 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) 265 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
233 266
234 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 267 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
235 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) 268 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
236 269
237 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ 270 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
238 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 271 expiresOn,
239 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), 272 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
240 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), 273 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
241 strategy: redundancy.strategy, 274 strategy,
242 videoStreamingPlaylistId: playlist.id, 275 videoStreamingPlaylistId: playlist.id,
243 actorId: serverActor.id 276 actorId: serverActor.id
244 }) 277 })
diff --git a/server/middlewares/sort.ts b/server/middlewares/sort.ts
index 8c27e8237..75238228f 100644
--- a/server/middlewares/sort.ts
+++ b/server/middlewares/sort.ts
@@ -1,17 +1,11 @@
1import * as express from 'express' 1import * as express from 'express'
2import { SortType } from '../models/utils' 2import { SortType } from '../models/utils'
3 3
4function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) { 4const setDefaultSort = setDefaultSortFactory('-createdAt')
5 if (!req.query.sort) req.query.sort = '-createdAt'
6
7 return next()
8}
9 5
10function setDefaultSearchSort (req: express.Request, res: express.Response, next: express.NextFunction) { 6const setDefaultVideoRedundanciesSort = setDefaultSortFactory('name')
11 if (!req.query.sort) req.query.sort = '-match'
12 7
13 return next() 8const setDefaultSearchSort = setDefaultSortFactory('-match')
14}
15 9
16function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) { 10function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) {
17 let newSort: SortType = { sortModel: undefined, sortValue: '' } 11 let newSort: SortType = { sortModel: undefined, sortValue: '' }
@@ -39,5 +33,16 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex
39export { 33export {
40 setDefaultSort, 34 setDefaultSort,
41 setDefaultSearchSort, 35 setDefaultSearchSort,
36 setDefaultVideoRedundanciesSort,
42 setBlacklistSort 37 setBlacklistSort
43} 38}
39
40// ---------------------------------------------------------------------------
41
42function setDefaultSortFactory (sort: string) {
43 return (req: express.Request, res: express.Response, next: express.NextFunction) => {
44 if (!req.query.sort) req.query.sort = sort
45
46 return next()
47 }
48}
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
index 8098e3a44..16b42fc0d 100644
--- a/server/middlewares/validators/redundancy.ts
+++ b/server/middlewares/validators/redundancy.ts
@@ -1,12 +1,13 @@
1import * as express from 'express' 1import * as express from 'express'
2import { body, param } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { exists, isBooleanValid, isIdOrUUIDValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' 3import { exists, isBooleanValid, isIdOrUUIDValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
4import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
5import { areValidationErrors } from './utils' 5import { areValidationErrors } from './utils'
6import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 6import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
7import { isHostValid } from '../../helpers/custom-validators/servers' 7import { isHostValid } from '../../helpers/custom-validators/servers'
8import { ServerModel } from '../../models/server/server' 8import { ServerModel } from '../../models/server/server'
9import { doesVideoExist } from '../../helpers/middlewares' 9import { doesVideoExist } from '../../helpers/middlewares'
10import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies'
10 11
11const videoFileRedundancyGetValidator = [ 12const videoFileRedundancyGetValidator = [
12 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), 13 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
@@ -101,10 +102,77 @@ const updateServerRedundancyValidator = [
101 } 102 }
102] 103]
103 104
105const listVideoRedundanciesValidator = [
106 query('target')
107 .custom(isVideoRedundancyTarget).withMessage('Should have a valid video redundancies target'),
108
109 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
110 logger.debug('Checking listVideoRedundanciesValidator parameters', { parameters: req.query })
111
112 if (areValidationErrors(req, res)) return
113
114 return next()
115 }
116]
117
118const addVideoRedundancyValidator = [
119 body('videoId')
120 .custom(isIdValid)
121 .withMessage('Should have a valid video id'),
122
123 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
124 logger.debug('Checking addVideoRedundancyValidator parameters', { parameters: req.query })
125
126 if (areValidationErrors(req, res)) return
127
128 if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return
129
130 if (res.locals.onlyVideo.remote === false) {
131 return res.status(400)
132 .json({ error: 'Cannot create a redundancy on a local video' })
133 .end()
134 }
135
136 const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid)
137 if (alreadyExists) {
138 return res.status(409)
139 .json({ error: 'This video is already duplicated by your instance.' })
140 }
141
142 return next()
143 }
144]
145
146const removeVideoRedundancyValidator = [
147 param('redundancyId')
148 .custom(isIdValid)
149 .withMessage('Should have a valid redundancy id'),
150
151 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
152 logger.debug('Checking removeVideoRedundancyValidator parameters', { parameters: req.query })
153
154 if (areValidationErrors(req, res)) return
155
156 const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10))
157 if (!redundancy) {
158 return res.status(404)
159 .json({ error: 'Video redundancy not found' })
160 .end()
161 }
162
163 res.locals.videoRedundancy = redundancy
164
165 return next()
166 }
167]
168
104// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
105 170
106export { 171export {
107 videoFileRedundancyGetValidator, 172 videoFileRedundancyGetValidator,
108 videoPlaylistRedundancyGetValidator, 173 videoPlaylistRedundancyGetValidator,
109 updateServerRedundancyValidator 174 updateServerRedundancyValidator,
175 listVideoRedundanciesValidator,
176 addVideoRedundancyValidator,
177 removeVideoRedundancyValidator
110} 178}
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index c75e701d6..b76dab722 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -23,6 +23,7 @@ const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUM
23const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) 23const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS)
24const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) 24const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS)
25const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) 25const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
26const SORTABLE_VIDEO_REDUNDANCIES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
26 27
27const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) 28const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS)
28const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) 29const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS)
@@ -45,6 +46,7 @@ const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COL
45const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) 46const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS)
46const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) 47const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS)
47const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) 48const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS)
49const videoRedundanciesSortValidator = checkSort(SORTABLE_VIDEO_REDUNDANCIES_COLUMNS)
48 50
49// --------------------------------------------------------------------------- 51// ---------------------------------------------------------------------------
50 52
@@ -69,5 +71,6 @@ export {
69 serversBlocklistSortValidator, 71 serversBlocklistSortValidator,
70 userNotificationsSortValidator, 72 userNotificationsSortValidator,
71 videoPlaylistsSortValidator, 73 videoPlaylistsSortValidator,
74 videoRedundanciesSortValidator,
72 pluginsSortValidator 75 pluginsSortValidator
73} 76}
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 8c9a7eabf..4e66d72e3 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -13,13 +13,13 @@ import {
13 UpdatedAt 13 UpdatedAt
14} from 'sequelize-typescript' 14} from 'sequelize-typescript'
15import { ActorModel } from '../activitypub/actor' 15import { ActorModel } from '../activitypub/actor'
16import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' 16import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
17import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' 17import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' 18import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
19import { VideoFileModel } from '../video/video-file' 19import { VideoFileModel } from '../video/video-file'
20import { getServerActor } from '../../helpers/utils' 20import { getServerActor } from '../../helpers/utils'
21import { VideoModel } from '../video/video' 21import { VideoModel } from '../video/video'
22import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 22import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
23import { logger } from '../../helpers/logger' 23import { logger } from '../../helpers/logger'
24import { CacheFileObject, VideoPrivacy } from '../../../shared' 24import { CacheFileObject, VideoPrivacy } from '../../../shared'
25import { VideoChannelModel } from '../video/video-channel' 25import { VideoChannelModel } from '../video/video-channel'
@@ -27,10 +27,16 @@ import { ServerModel } from '../server/server'
27import { sample } from 'lodash' 27import { sample } from 'lodash'
28import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
29import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize' 30import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' 31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
32import { CONFIG } from '../../initializers/config' 32import { CONFIG } from '../../initializers/config'
33import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' 33import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
34import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
35import {
36 FileRedundancyInformation,
37 StreamingPlaylistRedundancyInformation,
38 VideoRedundancy
39} from '@shared/models/redundancy/video-redundancy.model'
34 40
35export enum ScopeNames { 41export enum ScopeNames {
36 WITH_VIDEO = 'WITH_VIDEO' 42 WITH_VIDEO = 'WITH_VIDEO'
@@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
86 @UpdatedAt 92 @UpdatedAt
87 updatedAt: Date 93 updatedAt: Date
88 94
89 @AllowNull(false) 95 @AllowNull(true)
90 @Column 96 @Column
91 expiresOn: Date 97 expiresOn: Date
92 98
@@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
193 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 199 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
194 } 200 }
195 201
202 static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
203 const query = {
204 where: { id },
205 transaction
206 }
207
208 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
209 }
210
196 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> { 211 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
197 const query = { 212 const query = {
198 where: { 213 where: {
@@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
394 [Op.ne]: actor.id 409 [Op.ne]: actor.id
395 }, 410 },
396 expiresOn: { 411 expiresOn: {
397 [ Op.lt ]: new Date() 412 [ Op.lt ]: new Date(),
413 [ Op.ne ]: null
398 } 414 }
399 } 415 }
400 } 416 }
@@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
447 return VideoRedundancyModel.findAll(query) 463 return VideoRedundancyModel.findAll(query)
448 } 464 }
449 465
450 static async getStats (strategy: VideoRedundancyStrategy) { 466 static listForApi (options: {
467 start: number,
468 count: number,
469 sort: string,
470 target: VideoRedundanciesTarget,
471 strategy?: string
472 }) {
473 const { start, count, sort, target, strategy } = options
474 let redundancyWhere: WhereOptions = {}
475 let videosWhere: WhereOptions = {}
476 let redundancySqlSuffix = ''
477
478 if (target === 'my-videos') {
479 Object.assign(videosWhere, { remote: false })
480 } else if (target === 'remote-videos') {
481 Object.assign(videosWhere, { remote: true })
482 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
483 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
484 }
485
486 if (strategy) {
487 Object.assign(redundancyWhere, { strategy: strategy })
488 }
489
490 const videoFilterWhere = {
491 [Op.and]: [
492 {
493 [ Op.or ]: [
494 {
495 id: {
496 [ Op.in ]: literal(
497 '(' +
498 'SELECT "videoId" FROM "videoFile" ' +
499 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
500 redundancySqlSuffix +
501 ')'
502 )
503 }
504 },
505 {
506 id: {
507 [ Op.in ]: literal(
508 '(' +
509 'select "videoId" FROM "videoStreamingPlaylist" ' +
510 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
511 redundancySqlSuffix +
512 ')'
513 )
514 }
515 }
516 ]
517 },
518
519 videosWhere
520 ]
521 }
522
523 // /!\ On video model /!\
524 const findOptions = {
525 offset: start,
526 limit: count,
527 order: getSort(sort),
528 include: [
529 {
530 required: false,
531 model: VideoFileModel.unscoped(),
532 include: [
533 {
534 model: VideoRedundancyModel.unscoped(),
535 required: false,
536 where: redundancyWhere
537 }
538 ]
539 },
540 {
541 required: false,
542 model: VideoStreamingPlaylistModel.unscoped(),
543 include: [
544 {
545 model: VideoRedundancyModel.unscoped(),
546 required: false,
547 where: redundancyWhere
548 },
549 {
550 model: VideoFileModel.unscoped(),
551 required: false
552 }
553 ]
554 }
555 ],
556 where: videoFilterWhere
557 }
558
559 // /!\ On video model /!\
560 const countOptions = {
561 where: videoFilterWhere
562 }
563
564 return Promise.all([
565 VideoModel.findAll(findOptions),
566
567 VideoModel.count(countOptions)
568 ]).then(([ data, total ]) => ({ total, data }))
569 }
570
571 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
451 const actor = await getServerActor() 572 const actor = await getServerActor()
452 573
453 const query: FindOptions = { 574 const query: FindOptions = {
@@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
478 })) 599 }))
479 } 600 }
480 601
602 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
603 let filesRedundancies: FileRedundancyInformation[] = []
604 let streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
605
606 for (const file of video.VideoFiles) {
607 for (const redundancy of file.RedundancyVideos) {
608 filesRedundancies.push({
609 id: redundancy.id,
610 fileUrl: redundancy.fileUrl,
611 strategy: redundancy.strategy,
612 createdAt: redundancy.createdAt,
613 updatedAt: redundancy.updatedAt,
614 expiresOn: redundancy.expiresOn,
615 size: file.size
616 })
617 }
618 }
619
620 for (const playlist of video.VideoStreamingPlaylists) {
621 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
622
623 for (const redundancy of playlist.RedundancyVideos) {
624 streamingPlaylistsRedundancies.push({
625 id: redundancy.id,
626 fileUrl: redundancy.fileUrl,
627 strategy: redundancy.strategy,
628 createdAt: redundancy.createdAt,
629 updatedAt: redundancy.updatedAt,
630 expiresOn: redundancy.expiresOn,
631 size
632 })
633 }
634 }
635
636 return {
637 id: video.id,
638 name: video.name,
639 url: video.url,
640 uuid: video.uuid,
641
642 redundancies: {
643 files: filesRedundancies,
644 streamingPlaylists: streamingPlaylistsRedundancies
645 }
646 }
647 }
648
481 getVideo () { 649 getVideo () {
482 if (this.VideoFile) return this.VideoFile.Video 650 if (this.VideoFile) return this.VideoFile.Video
483 651
@@ -494,7 +662,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
494 id: this.url, 662 id: this.url,
495 type: 'CacheFile' as 'CacheFile', 663 type: 'CacheFile' as 'CacheFile',
496 object: this.VideoStreamingPlaylist.Video.url, 664 object: this.VideoStreamingPlaylist.Video.url,
497 expires: this.expiresOn.toISOString(), 665 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
498 url: { 666 url: {
499 type: 'Link', 667 type: 'Link',
500 mediaType: 'application/x-mpegURL', 668 mediaType: 'application/x-mpegURL',
@@ -507,7 +675,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
507 id: this.url, 675 id: this.url,
508 type: 'CacheFile' as 'CacheFile', 676 type: 'CacheFile' as 'CacheFile',
509 object: this.VideoFile.Video.url, 677 object: this.VideoFile.Video.url,
510 expires: this.expiresOn.toISOString(), 678 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
511 url: { 679 url: {
512 type: 'Link', 680 type: 'Link',
513 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, 681 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts
index 6471da840..7012a39ee 100644
--- a/server/tests/api/check-params/redundancy.ts
+++ b/server/tests/api/check-params/redundancy.ts
@@ -3,21 +3,25 @@
3import 'mocha' 3import 'mocha'
4 4
5import { 5import {
6 checkBadCountPagination,
7 checkBadSortPagination,
8 checkBadStartPagination,
6 cleanupTests, 9 cleanupTests,
7 createUser, 10 createUser,
8 doubleFollow, 11 doubleFollow,
9 flushAndRunMultipleServers, 12 flushAndRunMultipleServers, makeDeleteRequest,
10 flushTests, 13 makeGetRequest, makePostBodyRequest,
11 killallServers,
12 makePutBodyRequest, 14 makePutBodyRequest,
13 ServerInfo, 15 ServerInfo,
14 setAccessTokensToServers, 16 setAccessTokensToServers, uploadVideoAndGetId,
15 userLogin 17 userLogin, waitJobs
16} from '../../../../shared/extra-utils' 18} from '../../../../shared/extra-utils'
17 19
18describe('Test server redundancy API validators', function () { 20describe('Test server redundancy API validators', function () {
19 let servers: ServerInfo[] 21 let servers: ServerInfo[]
20 let userAccessToken = null 22 let userAccessToken = null
23 let videoIdLocal: number
24 let videoIdRemote: number
21 25
22 // --------------------------------------------------------------- 26 // ---------------------------------------------------------------
23 27
@@ -36,9 +40,134 @@ describe('Test server redundancy API validators', function () {
36 40
37 await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password }) 41 await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password })
38 userAccessToken = await userLogin(servers[0], user) 42 userAccessToken = await userLogin(servers[0], user)
43
44 videoIdLocal = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video' })).id
45 videoIdRemote = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video' })).id
46
47 await waitJobs(servers)
48 })
49
50 describe('When listing redundancies', function () {
51 const path = '/api/v1/server/redundancy/videos'
52
53 let url: string
54 let token: string
55
56 before(function () {
57 url = servers[0].url
58 token = servers[0].accessToken
59 })
60
61 it('Should fail with an invalid token', async function () {
62 await makeGetRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 })
63 })
64
65 it('Should fail if the user is not an administrator', async function () {
66 await makeGetRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 })
67 })
68
69 it('Should fail with a bad start pagination', async function () {
70 await checkBadStartPagination(url, path, servers[0].accessToken)
71 })
72
73 it('Should fail with a bad count pagination', async function () {
74 await checkBadCountPagination(url, path, servers[0].accessToken)
75 })
76
77 it('Should fail with an incorrect sort', async function () {
78 await checkBadSortPagination(url, path, servers[0].accessToken)
79 })
80
81 it('Should fail with a bad target', async function () {
82 await makeGetRequest({ url, path, token, query: { target: 'bad target' } })
83 })
84
85 it('Should fail without target', async function () {
86 await makeGetRequest({ url, path, token })
87 })
88
89 it('Should succeed with the correct params', async function () {
90 await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, statusCodeExpected: 200 })
91 })
92 })
93
94 describe('When manually adding a redundancy', function () {
95 const path = '/api/v1/server/redundancy/videos'
96
97 let url: string
98 let token: string
99
100 before(function () {
101 url = servers[0].url
102 token = servers[0].accessToken
103 })
104
105 it('Should fail with an invalid token', async function () {
106 await makePostBodyRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 })
107 })
108
109 it('Should fail if the user is not an administrator', async function () {
110 await makePostBodyRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 })
111 })
112
113 it('Should fail without a video id', async function () {
114 await makePostBodyRequest({ url, path, token })
115 })
116
117 it('Should fail with an incorrect video id', async function () {
118 await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } })
119 })
120
121 it('Should fail with a not found video id', async function () {
122 await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, statusCodeExpected: 404 })
123 })
124
125 it('Should fail with a local a video id', async function () {
126 await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } })
127 })
128
129 it('Should succeed with the correct params', async function () {
130 await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 204 })
131 })
132
133 it('Should fail if the video is already duplicated', async function () {
134 this.timeout(30000)
135
136 await waitJobs(servers)
137
138 await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 409 })
139 })
140 })
141
142 describe('When manually removing a redundancy', function () {
143 const path = '/api/v1/server/redundancy/videos/'
144
145 let url: string
146 let token: string
147
148 before(function () {
149 url = servers[0].url
150 token = servers[0].accessToken
151 })
152
153 it('Should fail with an invalid token', async function () {
154 await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', statusCodeExpected: 401 })
155 })
156
157 it('Should fail if the user is not an administrator', async function () {
158 await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, statusCodeExpected: 403 })
159 })
160
161 it('Should fail with an incorrect video id', async function () {
162 await makeDeleteRequest({ url, path: path + 'toto', token })
163 })
164
165 it('Should fail with a not found video redundancy', async function () {
166 await makeDeleteRequest({ url, path: path + '454545', token, statusCodeExpected: 404 })
167 })
39 }) 168 })
40 169
41 describe('When updating redundancy', function () { 170 describe('When updating server redundancy', function () {
42 const path = '/api/v1/server/redundancy' 171 const path = '/api/v1/server/redundancy'
43 172
44 it('Should fail with an invalid token', async function () { 173 it('Should fail with an invalid token', async function () {
diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts
index 8e69b95a6..5359055b0 100644
--- a/server/tests/api/redundancy/index.ts
+++ b/server/tests/api/redundancy/index.ts
@@ -1 +1,2 @@
1import './redundancy' 1import './redundancy'
2import './manage-redundancy'
diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts
new file mode 100644
index 000000000..6a8937f24
--- /dev/null
+++ b/server/tests/api/redundancy/manage-redundancy.ts
@@ -0,0 +1,373 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 cleanupTests,
7 doubleFollow,
8 flushAndRunMultipleServers,
9 getLocalIdByUUID,
10 ServerInfo,
11 setAccessTokensToServers,
12 uploadVideo,
13 uploadVideoAndGetId,
14 waitUntilLog
15} from '../../../../shared/extra-utils'
16import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
17import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy'
18import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
19
20const expect = chai.expect
21
22describe('Test manage videos redundancy', function () {
23 const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ]
24
25 let servers: ServerInfo[]
26 let video1Server2UUID: string
27 let video2Server2UUID: string
28 let redundanciesToRemove: number[] = []
29
30 before(async function () {
31 this.timeout(120000)
32
33 const config = {
34 transcoding: {
35 hls: {
36 enabled: true
37 }
38 },
39 redundancy: {
40 videos: {
41 check_interval: '1 second',
42 strategies: [
43 {
44 strategy: 'recently-added',
45 min_lifetime: '1 hour',
46 size: '10MB',
47 min_views: 0
48 }
49 ]
50 }
51 }
52 }
53 servers = await flushAndRunMultipleServers(3, config)
54
55 // Get the access tokens
56 await setAccessTokensToServers(servers)
57
58 {
59 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
60 video1Server2UUID = res.body.video.uuid
61 }
62
63 {
64 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
65 video2Server2UUID = res.body.video.uuid
66 }
67
68 await waitJobs(servers)
69
70 // Server 1 and server 2 follow each other
71 await doubleFollow(servers[ 0 ], servers[ 1 ])
72 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
73
74 await waitJobs(servers)
75 })
76
77 it('Should not have redundancies on server 3', async function () {
78 for (const target of targets) {
79 const res = await listVideoRedundancies({
80 url: servers[2].url,
81 accessToken: servers[2].accessToken,
82 target
83 })
84
85 expect(res.body.total).to.equal(0)
86 expect(res.body.data).to.have.lengthOf(0)
87 }
88 })
89
90 it('Should not have "remote-videos" redundancies on server 2', async function () {
91 this.timeout(120000)
92
93 await waitJobs(servers)
94 await waitUntilLog(servers[0], 'Duplicated ', 10)
95 await waitJobs(servers)
96
97 const res = await listVideoRedundancies({
98 url: servers[1].url,
99 accessToken: servers[1].accessToken,
100 target: 'remote-videos'
101 })
102
103 expect(res.body.total).to.equal(0)
104 expect(res.body.data).to.have.lengthOf(0)
105 })
106
107 it('Should have "my-videos" redundancies on server 2', async function () {
108 this.timeout(120000)
109
110 const res = await listVideoRedundancies({
111 url: servers[1].url,
112 accessToken: servers[1].accessToken,
113 target: 'my-videos'
114 })
115
116 expect(res.body.total).to.equal(2)
117
118 const videos = res.body.data as VideoRedundancy[]
119 expect(videos).to.have.lengthOf(2)
120
121 const videos1 = videos.find(v => v.uuid === video1Server2UUID)
122 const videos2 = videos.find(v => v.uuid === video2Server2UUID)
123
124 expect(videos1.name).to.equal('video 1 server 2')
125 expect(videos2.name).to.equal('video 2 server 2')
126
127 expect(videos1.redundancies.files).to.have.lengthOf(4)
128 expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1)
129
130 const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists)
131
132 for (const r of redundancies) {
133 expect(r.strategy).to.be.null
134 expect(r.fileUrl).to.exist
135 expect(r.createdAt).to.exist
136 expect(r.updatedAt).to.exist
137 expect(r.expiresOn).to.exist
138 }
139 })
140
141 it('Should not have "my-videos" redundancies on server 1', async function () {
142 const res = await listVideoRedundancies({
143 url: servers[0].url,
144 accessToken: servers[0].accessToken,
145 target: 'my-videos'
146 })
147
148 expect(res.body.total).to.equal(0)
149 expect(res.body.data).to.have.lengthOf(0)
150 })
151
152 it('Should have "remote-videos" redundancies on server 1', async function () {
153 this.timeout(120000)
154
155 const res = await listVideoRedundancies({
156 url: servers[0].url,
157 accessToken: servers[0].accessToken,
158 target: 'remote-videos'
159 })
160
161 expect(res.body.total).to.equal(2)
162
163 const videos = res.body.data as VideoRedundancy[]
164 expect(videos).to.have.lengthOf(2)
165
166 const videos1 = videos.find(v => v.uuid === video1Server2UUID)
167 const videos2 = videos.find(v => v.uuid === video2Server2UUID)
168
169 expect(videos1.name).to.equal('video 1 server 2')
170 expect(videos2.name).to.equal('video 2 server 2')
171
172 expect(videos1.redundancies.files).to.have.lengthOf(4)
173 expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1)
174
175 const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists)
176
177 for (const r of redundancies) {
178 expect(r.strategy).to.equal('recently-added')
179 expect(r.fileUrl).to.exist
180 expect(r.createdAt).to.exist
181 expect(r.updatedAt).to.exist
182 expect(r.expiresOn).to.exist
183 }
184 })
185
186 it('Should correctly paginate and sort results', async function () {
187 {
188 const res = await listVideoRedundancies({
189 url: servers[0].url,
190 accessToken: servers[0].accessToken,
191 target: 'remote-videos',
192 sort: 'name',
193 start: 0,
194 count: 2
195 })
196
197 const videos = res.body.data
198 expect(videos[ 0 ].name).to.equal('video 1 server 2')
199 expect(videos[ 1 ].name).to.equal('video 2 server 2')
200 }
201
202 {
203 const res = await listVideoRedundancies({
204 url: servers[0].url,
205 accessToken: servers[0].accessToken,
206 target: 'remote-videos',
207 sort: '-name',
208 start: 0,
209 count: 2
210 })
211
212 const videos = res.body.data
213 expect(videos[ 0 ].name).to.equal('video 2 server 2')
214 expect(videos[ 1 ].name).to.equal('video 1 server 2')
215 }
216
217 {
218 const res = await listVideoRedundancies({
219 url: servers[0].url,
220 accessToken: servers[0].accessToken,
221 target: 'remote-videos',
222 sort: '-name',
223 start: 1,
224 count: 1
225 })
226
227 const videos = res.body.data
228 expect(videos[ 0 ].name).to.equal('video 1 server 2')
229 }
230 })
231
232 it('Should manually add a redundancy and list it', async function () {
233 this.timeout(120000)
234
235 const uuid = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid
236 await waitJobs(servers)
237 const videoId = await getLocalIdByUUID(servers[0].url, uuid)
238
239 await addVideoRedundancy({
240 url: servers[0].url,
241 accessToken: servers[0].accessToken,
242 videoId
243 })
244
245 await waitJobs(servers)
246 await waitUntilLog(servers[0], 'Duplicated ', 15)
247 await waitJobs(servers)
248
249 {
250 const res = await listVideoRedundancies({
251 url: servers[0].url,
252 accessToken: servers[0].accessToken,
253 target: 'remote-videos',
254 sort: '-name',
255 start: 0,
256 count: 5
257 })
258
259 const videos = res.body.data
260 expect(videos[ 0 ].name).to.equal('video 3 server 2')
261
262 const video = videos[ 0 ]
263 expect(video.redundancies.files).to.have.lengthOf(4)
264 expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
265
266 const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
267
268 for (const r of redundancies) {
269 redundanciesToRemove.push(r.id)
270
271 expect(r.strategy).to.equal('manual')
272 expect(r.fileUrl).to.exist
273 expect(r.createdAt).to.exist
274 expect(r.updatedAt).to.exist
275 expect(r.expiresOn).to.be.null
276 }
277 }
278
279 const res = await listVideoRedundancies({
280 url: servers[1].url,
281 accessToken: servers[1].accessToken,
282 target: 'my-videos',
283 sort: '-name',
284 start: 0,
285 count: 5
286 })
287
288 const videos = res.body.data
289 expect(videos[ 0 ].name).to.equal('video 3 server 2')
290
291 const video = videos[ 0 ]
292 expect(video.redundancies.files).to.have.lengthOf(4)
293 expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
294
295 const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
296
297 for (const r of redundancies) {
298 expect(r.strategy).to.be.null
299 expect(r.fileUrl).to.exist
300 expect(r.createdAt).to.exist
301 expect(r.updatedAt).to.exist
302 expect(r.expiresOn).to.be.null
303 }
304 })
305
306 it('Should manually remove a redundancy and remove it from the list', async function () {
307 this.timeout(120000)
308
309 for (const redundancyId of redundanciesToRemove) {
310 await removeVideoRedundancy({
311 url: servers[ 0 ].url,
312 accessToken: servers[ 0 ].accessToken,
313 redundancyId
314 })
315 }
316
317 {
318 const res = await listVideoRedundancies({
319 url: servers[0].url,
320 accessToken: servers[0].accessToken,
321 target: 'remote-videos',
322 sort: '-name',
323 start: 0,
324 count: 5
325 })
326
327 const videos = res.body.data
328 expect(videos).to.have.lengthOf(2)
329
330 expect(videos[ 0 ].name).to.equal('video 2 server 2')
331
332 redundanciesToRemove = []
333 const video = videos[ 0 ]
334 expect(video.redundancies.files).to.have.lengthOf(4)
335 expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
336
337 const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
338
339 for (const r of redundancies) {
340 redundanciesToRemove.push(r.id)
341 }
342 }
343 })
344
345 it('Should remove another (auto) redundancy', async function () {
346 {
347 for (const redundancyId of redundanciesToRemove) {
348 await removeVideoRedundancy({
349 url: servers[ 0 ].url,
350 accessToken: servers[ 0 ].accessToken,
351 redundancyId
352 })
353 }
354
355 const res = await listVideoRedundancies({
356 url: servers[0].url,
357 accessToken: servers[0].accessToken,
358 target: 'remote-videos',
359 sort: '-name',
360 start: 0,
361 count: 5
362 })
363
364 const videos = res.body.data
365 expect(videos[ 0 ].name).to.equal('video 1 server 2')
366 expect(videos).to.have.lengthOf(1)
367 }
368 })
369
370 after(async function () {
371 await cleanupTests(servers)
372 })
373})
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 1cdf93aa1..f5bf130d5 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -5,7 +5,8 @@ import 'mocha'
5import { VideoDetails } from '../../../../shared/models/videos' 5import { VideoDetails } from '../../../../shared/models/videos'
6import { 6import {
7 checkSegmentHash, 7 checkSegmentHash,
8 checkVideoFilesWereRemoved, cleanupTests, 8 checkVideoFilesWereRemoved,
9 cleanupTests,
9 doubleFollow, 10 doubleFollow,
10 flushAndRunMultipleServers, 11 flushAndRunMultipleServers,
11 getFollowingListPaginationAndSort, 12 getFollowingListPaginationAndSort,
@@ -28,11 +29,16 @@ import {
28import { waitJobs } from '../../../../shared/extra-utils/server/jobs' 29import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
29 30
30import * as magnetUtil from 'magnet-uri' 31import * as magnetUtil from 'magnet-uri'
31import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy' 32import {
33 addVideoRedundancy,
34 listVideoRedundancies,
35 removeVideoRedundancy,
36 updateRedundancy
37} from '../../../../shared/extra-utils/server/redundancy'
32import { ActorFollow } from '../../../../shared/models/actors' 38import { ActorFollow } from '../../../../shared/models/actors'
33import { readdir } from 'fs-extra' 39import { readdir } from 'fs-extra'
34import { join } from 'path' 40import { join } from 'path'
35import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' 41import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
36import { getStats } from '../../../../shared/extra-utils/server/stats' 42import { getStats } from '../../../../shared/extra-utils/server/stats'
37import { ServerStats } from '../../../../shared/models/server/server-stats.model' 43import { ServerStats } from '../../../../shared/models/server/server-stats.model'
38 44
@@ -40,6 +46,7 @@ const expect = chai.expect
40 46
41let servers: ServerInfo[] = [] 47let servers: ServerInfo[] = []
42let video1Server2UUID: string 48let video1Server2UUID: string
49let video1Server2Id: number
43 50
44function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { 51function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
45 const parsed = magnetUtil.decode(file.magnetUri) 52 const parsed = magnetUtil.decode(file.magnetUri)
@@ -52,7 +59,19 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
52 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) 59 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
53} 60}
54 61
55async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { 62async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) {
63 const strategies: any[] = []
64
65 if (strategy !== null) {
66 strategies.push(
67 immutableAssign({
68 min_lifetime: '1 hour',
69 strategy: strategy,
70 size: '400KB'
71 }, additionalParams)
72 )
73 }
74
56 const config = { 75 const config = {
57 transcoding: { 76 transcoding: {
58 hls: { 77 hls: {
@@ -62,16 +81,11 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional
62 redundancy: { 81 redundancy: {
63 videos: { 82 videos: {
64 check_interval: '5 seconds', 83 check_interval: '5 seconds',
65 strategies: [ 84 strategies
66 immutableAssign({
67 min_lifetime: '1 hour',
68 strategy: strategy,
69 size: '400KB'
70 }, additionalParams)
71 ]
72 } 85 }
73 } 86 }
74 } 87 }
88
75 servers = await flushAndRunMultipleServers(3, config) 89 servers = await flushAndRunMultipleServers(3, config)
76 90
77 // Get the access tokens 91 // Get the access tokens
@@ -80,6 +94,7 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional
80 { 94 {
81 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) 95 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
82 video1Server2UUID = res.body.video.uuid 96 video1Server2UUID = res.body.video.uuid
97 video1Server2Id = res.body.video.id
83 98
84 await viewVideo(servers[ 1 ].url, video1Server2UUID) 99 await viewVideo(servers[ 1 ].url, video1Server2UUID)
85 } 100 }
@@ -216,29 +231,38 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
216 } 231 }
217} 232}
218 233
219async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { 234async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
235 let totalSize: number = null
236 let statsLength = 1
237
238 if (strategy !== 'manual') {
239 totalSize = 409600
240 statsLength = 2
241 }
242
220 const res = await getStats(servers[0].url) 243 const res = await getStats(servers[0].url)
221 const data: ServerStats = res.body 244 const data: ServerStats = res.body
222 245
223 expect(data.videosRedundancy).to.have.lengthOf(1) 246 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
224 const stat = data.videosRedundancy[0]
225 247
248 const stat = data.videosRedundancy[0]
226 expect(stat.strategy).to.equal(strategy) 249 expect(stat.strategy).to.equal(strategy)
227 expect(stat.totalSize).to.equal(409600) 250 expect(stat.totalSize).to.equal(totalSize)
251
252 return stat
253}
254
255async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) {
256 const stat = await checkStatsGlobal(strategy)
257
228 expect(stat.totalUsed).to.be.at.least(1).and.below(409601) 258 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
229 expect(stat.totalVideoFiles).to.equal(4) 259 expect(stat.totalVideoFiles).to.equal(4)
230 expect(stat.totalVideos).to.equal(1) 260 expect(stat.totalVideos).to.equal(1)
231} 261}
232 262
233async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { 263async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) {
234 const res = await getStats(servers[0].url) 264 const stat = await checkStatsGlobal(strategy)
235 const data: ServerStats = res.body
236
237 expect(data.videosRedundancy).to.have.lengthOf(1)
238 265
239 const stat = data.videosRedundancy[0]
240 expect(stat.strategy).to.equal(strategy)
241 expect(stat.totalSize).to.equal(409600)
242 expect(stat.totalUsed).to.equal(0) 266 expect(stat.totalUsed).to.equal(0)
243 expect(stat.totalVideoFiles).to.equal(0) 267 expect(stat.totalVideoFiles).to.equal(0)
244 expect(stat.totalVideos).to.equal(0) 268 expect(stat.totalVideos).to.equal(0)
@@ -446,6 +470,74 @@ describe('Test videos redundancy', function () {
446 }) 470 })
447 }) 471 })
448 472
473 describe('With manual strategy', function () {
474 before(function () {
475 this.timeout(120000)
476
477 return flushAndRunServers(null)
478 })
479
480 it('Should have 1 webseed on the first video', async function () {
481 await check1WebSeed()
482 await check0PlaylistRedundancies()
483 await checkStatsWith1Webseed('manual')
484 })
485
486 it('Should create a redundancy on first video', async function () {
487 await addVideoRedundancy({
488 url: servers[0].url,
489 accessToken: servers[0].accessToken,
490 videoId: video1Server2Id
491 })
492 })
493
494 it('Should have 2 webseeds on the first video', async function () {
495 this.timeout(80000)
496
497 await waitJobs(servers)
498 await waitUntilLog(servers[0], 'Duplicated ', 5)
499 await waitJobs(servers)
500
501 await check2Webseeds()
502 await check1PlaylistRedundancies()
503 await checkStatsWith2Webseed('manual')
504 })
505
506 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
507 this.timeout(80000)
508
509 const res = await listVideoRedundancies({
510 url: servers[0].url,
511 accessToken: servers[0].accessToken,
512 target: 'remote-videos'
513 })
514
515 const videos = res.body.data as VideoRedundancy[]
516 expect(videos).to.have.lengthOf(1)
517
518 const video = videos[0]
519 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
520 await removeVideoRedundancy({
521 url: servers[0].url,
522 accessToken: servers[0].accessToken,
523 redundancyId: r.id
524 })
525 }
526
527 await waitJobs(servers)
528 await wait(5000)
529
530 await check1WebSeed()
531 await check0PlaylistRedundancies()
532
533 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
534 })
535
536 after(async function () {
537 await cleanupTests(servers)
538 })
539 })
540
449 describe('Test expiration', function () { 541 describe('Test expiration', function () {
450 const strategy = 'recently-added' 542 const strategy = 'recently-added'
451 543
diff --git a/server/typings/models/video/video-file.ts b/server/typings/models/video/video-file.ts
index 352fe3d32..139b22b2c 100644
--- a/server/typings/models/video/video-file.ts
+++ b/server/typings/models/video/video-file.ts
@@ -1,7 +1,7 @@
1import { VideoFileModel } from '../../../models/video/video-file' 1import { VideoFileModel } from '../../../models/video/video-file'
2import { PickWith, PickWithOpt } from '../../utils' 2import { PickWith, PickWithOpt } from '../../utils'
3import { MVideo, MVideoUUID } from './video' 3import { MVideo, MVideoUUID } from './video'
4import { MVideoRedundancyFileUrl } from './video-redundancy' 4import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy'
5import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' 5import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist'
6 6
7type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M> 7type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M>
@@ -22,6 +22,9 @@ export type MVideoFileStreamingPlaylistVideo = MVideoFile &
22export type MVideoFileVideoUUID = MVideoFile & 22export type MVideoFileVideoUUID = MVideoFile &
23 Use<'Video', MVideoUUID> 23 Use<'Video', MVideoUUID>
24 24
25export type MVideoFileRedundanciesAll = MVideoFile &
26 PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancy[]>
27
25export type MVideoFileRedundanciesOpt = MVideoFile & 28export type MVideoFileRedundanciesOpt = MVideoFile &
26 PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]> 29 PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
27 30
diff --git a/server/typings/models/video/video-streaming-playlist.ts b/server/typings/models/video/video-streaming-playlist.ts
index 436c0c072..6fd489945 100644
--- a/server/typings/models/video/video-streaming-playlist.ts
+++ b/server/typings/models/video/video-streaming-playlist.ts
@@ -1,6 +1,6 @@
1import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' 1import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist'
2import { PickWith, PickWithOpt } from '../../utils' 2import { PickWith, PickWithOpt } from '../../utils'
3import { MVideoRedundancyFileUrl } from './video-redundancy' 3import { MVideoRedundancyFileUrl, MVideoRedundancy } from './video-redundancy'
4import { MVideo } from './video' 4import { MVideo } from './video'
5import { MVideoFile } from './video-file' 5import { MVideoFile } from './video-file'
6 6
@@ -20,6 +20,10 @@ export type MStreamingPlaylistFilesVideo = MStreamingPlaylist &
20 Use<'VideoFiles', MVideoFile[]> & 20 Use<'VideoFiles', MVideoFile[]> &
21 Use<'Video', MVideo> 21 Use<'Video', MVideo>
22 22
23export type MStreamingPlaylistRedundanciesAll = MStreamingPlaylist &
24 Use<'VideoFiles', MVideoFile[]> &
25 Use<'RedundancyVideos', MVideoRedundancy[]>
26
23export type MStreamingPlaylistRedundancies = MStreamingPlaylist & 27export type MStreamingPlaylistRedundancies = MStreamingPlaylist &
24 Use<'VideoFiles', MVideoFile[]> & 28 Use<'VideoFiles', MVideoFile[]> &
25 Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> 29 Use<'RedundancyVideos', MVideoRedundancyFileUrl[]>
diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts
index 7f69a91de..bcc5e5028 100644
--- a/server/typings/models/video/video.ts
+++ b/server/typings/models/video/video.ts
@@ -10,8 +10,13 @@ import {
10} from './video-channels' 10} from './video-channels'
11import { MTag } from './tag' 11import { MTag } from './tag'
12import { MVideoCaptionLanguage } from './video-caption' 12import { MVideoCaptionLanguage } from './video-caption'
13import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' 13import {
14import { MVideoFile, MVideoFileRedundanciesOpt } from './video-file' 14 MStreamingPlaylistFiles,
15 MStreamingPlaylistRedundancies,
16 MStreamingPlaylistRedundanciesAll,
17 MStreamingPlaylistRedundanciesOpt
18} from './video-streaming-playlist'
19import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file'
15import { MThumbnail } from './thumbnail' 20import { MThumbnail } from './thumbnail'
16import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' 21import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
17import { MScheduleVideoUpdate } from './schedule-video-update' 22import { MScheduleVideoUpdate } from './schedule-video-update'
@@ -158,6 +163,10 @@ export type MVideoForUser = MVideo &
158 Use<'VideoBlacklist', MVideoBlacklistLight> & 163 Use<'VideoBlacklist', MVideoBlacklistLight> &
159 Use<'Thumbnails', MThumbnail[]> 164 Use<'Thumbnails', MThumbnail[]>
160 165
166export type MVideoForRedundancyAPI = MVideo &
167 Use<'VideoFiles', MVideoFileRedundanciesAll[]> &
168 Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]>
169
161// ############################################################################ 170// ############################################################################
162 171
163// Format for API or AP object 172// Format for API or AP object
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 16:07:05 +0000