aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--config/default.yaml8
-rw-r--r--config/production.yaml.example8
-rw-r--r--config/test.yaml7
-rw-r--r--server.ts12
-rw-r--r--server/helpers/ffmpeg-utils.ts2
-rw-r--r--server/initializers/checker-after-init.ts115
-rw-r--r--server/initializers/checker-before-init.ts (renamed from server/initializers/checker.ts)110
-rw-r--r--server/initializers/constants.ts11
-rw-r--r--server/initializers/index.ts1
-rw-r--r--server/initializers/installer.ts2
-rw-r--r--server/lib/activitypub/audience.ts7
-rw-r--r--server/lib/activitypub/cache-file.ts22
-rw-r--r--server/lib/activitypub/process/process-create.ts8
-rw-r--r--server/lib/activitypub/process/process-undo.ts2
-rw-r--r--server/lib/activitypub/process/process-update.ts28
-rw-r--r--server/lib/activitypub/send/send-update.ts4
-rw-r--r--server/lib/activitypub/videos.ts50
-rw-r--r--server/lib/cache/index.ts1
-rw-r--r--server/lib/redundancy.ts3
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts86
-rw-r--r--server/models/activitypub/actor.ts31
-rw-r--r--server/models/redundancy/video-redundancy.ts64
-rw-r--r--server/models/video/video-file.ts6
-rw-r--r--server/tests/api/server/redundancy.ts264
-rw-r--r--server/tests/utils/server/servers.ts4
-rw-r--r--shared/models/redundancy/videos-redundancy.model.ts3
-rw-r--r--support/doc/redundancy.md46
27 files changed, 627 insertions, 278 deletions
diff --git a/config/default.yaml b/config/default.yaml
index fa1fb628a..0d7d948c2 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -75,14 +75,20 @@ redundancy:
75 strategies: 75 strategies:
76# - 76# -
77# size: '10GB' 77# size: '10GB'
78# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
79# min_lifetime: '48 hours'
78# strategy: 'most-views' # Cache videos that have the most views 80# strategy: 'most-views' # Cache videos that have the most views
79# - 81# -
80# size: '10GB' 82# size: '10GB'
83# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
84# min_lifetime: '48 hours'
81# strategy: 'trending' # Cache trending videos 85# strategy: 'trending' # Cache trending videos
82# - 86# -
83# size: '10GB' 87# size: '10GB'
88# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
89# min_lifetime: '48 hours'
84# strategy: 'recently-added' # Cache recently added videos 90# strategy: 'recently-added' # Cache recently added videos
85# minViews: 10 # Having at least x views 91# min_views: 10 # Having at least x views
86 92
87cache: 93cache:
88 previews: 94 previews:
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 4d8752206..f9da8e0dd 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -76,14 +76,20 @@ redundancy:
76 strategies: 76 strategies:
77# - 77# -
78# size: '10GB' 78# size: '10GB'
79# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
80# min_lifetime: '48 hours'
79# strategy: 'most-views' # Cache videos that have the most views 81# strategy: 'most-views' # Cache videos that have the most views
80# - 82# -
81# size: '10GB' 83# size: '10GB'
84# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
85# min_lifetime: '48 hours'
82# strategy: 'trending' # Cache trending videos 86# strategy: 'trending' # Cache trending videos
83# - 87# -
84# size: '10GB' 88# size: '10GB'
89# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
90# min_lifetime: '48 hours'
85# strategy: 'recently-added' # Cache recently added videos 91# strategy: 'recently-added' # Cache recently added videos
86# minViews: 10 # Having at least x views 92# min_views: 10 # Having at least x views
87 93
88############################################################################### 94###############################################################################
89# 95#
diff --git a/config/test.yaml b/config/test.yaml
index ad94b00cd..04c999966 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -23,18 +23,21 @@ log:
23 23
24redundancy: 24redundancy:
25 videos: 25 videos:
26 check_interval: '5 seconds' 26 check_interval: '10 minutes'
27 strategies: 27 strategies:
28 - 28 -
29 size: '10MB' 29 size: '10MB'
30 min_lifetime: '10 minutes'
30 strategy: 'most-views' 31 strategy: 'most-views'
31 - 32 -
32 size: '10MB' 33 size: '10MB'
34 min_lifetime: '10 minutes'
33 strategy: 'trending' 35 strategy: 'trending'
34 - 36 -
35 size: '10MB' 37 size: '10MB'
38 min_lifetime: '10 minutes'
36 strategy: 'recently-added' 39 strategy: 'recently-added'
37 minViews: 1 40 min_views: 1
38 41
39cache: 42cache:
40 previews: 43 previews:
diff --git a/server.ts b/server.ts
index 8bc5e5f32..59fb820b4 100644
--- a/server.ts
+++ b/server.ts
@@ -1,6 +1,4 @@
1// FIXME: https://github.com/nodejs/node/pull/16853 1// FIXME: https://github.com/nodejs/node/pull/16853
2import { VideosCaptionCache } from './server/lib/cache/videos-caption-cache'
3
4require('tls').DEFAULT_ECDH_CURVE = 'auto' 2require('tls').DEFAULT_ECDH_CURVE = 'auto'
5 3
6import { isTestInstance } from './server/helpers/core-utils' 4import { isTestInstance } from './server/helpers/core-utils'
@@ -17,7 +15,7 @@ import * as cors from 'cors'
17import * as cookieParser from 'cookie-parser' 15import * as cookieParser from 'cookie-parser'
18import * as helmet from 'helmet' 16import * as helmet from 'helmet'
19import * as useragent from 'useragent' 17import * as useragent from 'useragent'
20import * as anonymise from 'ip-anonymize' 18import * as anonymize from 'ip-anonymize'
21 19
22process.title = 'peertube' 20process.title = 'peertube'
23 21
@@ -25,7 +23,7 @@ process.title = 'peertube'
25const app = express() 23const app = express()
26 24
27// ----------- Core checker ----------- 25// ----------- Core checker -----------
28import { checkMissedConfig, checkFFmpeg, checkConfig, checkActivityPubUrls } from './server/initializers/checker' 26import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-before-init'
29 27
30// Do not use barrels because we don't want to load all modules here (we need to initialize database first) 28// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
31import { logger } from './server/helpers/logger' 29import { logger } from './server/helpers/logger'
@@ -43,6 +41,8 @@ checkFFmpeg(CONFIG)
43 process.exit(-1) 41 process.exit(-1)
44 }) 42 })
45 43
44import { checkConfig, checkActivityPubUrls } from './server/initializers/checker-after-init'
45
46const errorMessage = checkConfig() 46const errorMessage = checkConfig()
47if (errorMessage !== null) { 47if (errorMessage !== null) {
48 throw new Error(errorMessage) 48 throw new Error(errorMessage)
@@ -76,7 +76,7 @@ migrate()
76import { installApplication } from './server/initializers' 76import { installApplication } from './server/initializers'
77import { Emailer } from './server/lib/emailer' 77import { Emailer } from './server/lib/emailer'
78import { JobQueue } from './server/lib/job-queue' 78import { JobQueue } from './server/lib/job-queue'
79import { VideosPreviewCache } from './server/lib/cache' 79import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache'
80import { 80import {
81 activityPubRouter, 81 activityPubRouter,
82 apiRouter, 82 apiRouter,
@@ -111,7 +111,7 @@ if (isTestInstance()) {
111// For the logger 111// For the logger
112morgan.token('remote-addr', req => { 112morgan.token('remote-addr', req => {
113 return (req.get('DNT') === '1') ? 113 return (req.get('DNT') === '1') ?
114 anonymise(req.ip || (req.connection && req.connection.remoteAddress) || undefined, 114 anonymize(req.ip || (req.connection && req.connection.remoteAddress) || undefined,
115 16, // bitmask for IPv4 115 16, // bitmask for IPv4
116 16 // bitmask for IPv6 116 16 // bitmask for IPv6
117 ) : 117 ) :
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 7c45f3632..22bc25476 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -4,7 +4,7 @@ import { VideoResolution } from '../../shared/models/videos'
4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers' 4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
6import { logger } from './logger' 6import { logger } from './logger'
7import { checkFFmpegEncoders } from '../initializers/checker' 7import { checkFFmpegEncoders } from '../initializers/checker-before-init'
8import { remove } from 'fs-extra' 8import { remove } from 'fs-extra'
9 9
10function computeResolutionsToTranscode (videoFileHeight: number) { 10function computeResolutionsToTranscode (videoFileHeight: number) {
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
new file mode 100644
index 000000000..588526235
--- /dev/null
+++ b/server/initializers/checker-after-init.ts
@@ -0,0 +1,115 @@
1import * as config from 'config'
2import { promisify0, isProdInstance, parseDuration, isTestInstance } from '../helpers/core-utils'
3import { UserModel } from '../models/account/user'
4import { ApplicationModel } from '../models/application/application'
5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { parse } from 'url'
7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils'
10import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash'
13
14async function checkActivityPubUrls () {
15 const actor = await getServerActor()
16
17 const parsed = parse(actor.url)
18 if (CONFIG.WEBSERVER.HOST !== parsed.host) {
19 const NODE_ENV = config.util.getEnv('NODE_ENV')
20 const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
21
22 logger.warn(
23 'It seems PeerTube was started (and created some data) with another domain name. ' +
24 'This means you will not be able to federate! ' +
25 'Please use %s %s npm run update-host to fix this.',
26 NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
27 NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
28 )
29 }
30}
31
32// Some checks on configuration files
33// Return an error message, or null if everything is okay
34function checkConfig () {
35 const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY
36
37 // NSFW policy
38 {
39 const available = [ 'do_not_list', 'blur', 'display' ]
40 if (available.indexOf(defaultNSFWPolicy) === -1) {
41 return 'NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy
42 }
43 }
44
45 // Redundancies
46 const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
47 if (isArray(redundancyVideos)) {
48 const available = [ 'most-views', 'trending', 'recently-added' ]
49 for (const r of redundancyVideos) {
50 if (available.indexOf(r.strategy) === -1) {
51 return 'Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy
52 }
53
54 // Lifetime should not be < 10 hours
55 if (!isTestInstance() && r.minLifetime < 1000 * 3600 * 10) {
56 return 'Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy
57 }
58 }
59
60 const filtered = uniq(redundancyVideos.map(r => r.strategy))
61 if (filtered.length !== redundancyVideos.length) {
62 return 'Redundancy video entries should have unique strategies'
63 }
64
65 const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
66 if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
67 return 'Min views in recently added strategy is not a number'
68 }
69 }
70
71 if (isProdInstance()) {
72 const configStorage = config.get('storage')
73 for (const key of Object.keys(configStorage)) {
74 if (configStorage[key].startsWith('storage/')) {
75 logger.warn(
76 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
77 key
78 )
79 }
80 }
81 }
82
83 return null
84}
85
86// We get db by param to not import it in this file (import orders)
87async function clientsExist () {
88 const totalClients = await OAuthClientModel.countTotal()
89
90 return totalClients !== 0
91}
92
93// We get db by param to not import it in this file (import orders)
94async function usersExist () {
95 const totalUsers = await UserModel.countTotal()
96
97 return totalUsers !== 0
98}
99
100// We get db by param to not import it in this file (import orders)
101async function applicationExist () {
102 const totalApplication = await ApplicationModel.countTotal()
103
104 return totalApplication !== 0
105}
106
107// ---------------------------------------------------------------------------
108
109export {
110 checkConfig,
111 clientsExist,
112 usersExist,
113 applicationExist,
114 checkActivityPubUrls
115}
diff --git a/server/initializers/checker.ts b/server/initializers/checker-before-init.ts
index 5b068caa1..4f46d406a 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker-before-init.ts
@@ -1,78 +1,8 @@
1import * as config from 'config' 1import * as config from 'config'
2import { promisify0, isProdInstance } from '../helpers/core-utils' 2import { promisify0 } from '../helpers/core-utils'
3import { UserModel } from '../models/account/user'
4import { ApplicationModel } from '../models/application/application'
5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { parse } from 'url'
7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils'
10import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc' 3import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash'
13
14async function checkActivityPubUrls () {
15 const actor = await getServerActor()
16
17 const parsed = parse(actor.url)
18 if (CONFIG.WEBSERVER.HOST !== parsed.host) {
19 const NODE_ENV = config.util.getEnv('NODE_ENV')
20 const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
21
22 logger.warn(
23 'It seems PeerTube was started (and created some data) with another domain name. ' +
24 'This means you will not be able to federate! ' +
25 'Please use %s %s npm run update-host to fix this.',
26 NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
27 NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
28 )
29 }
30}
31 4
32// Some checks on configuration files 5// ONLY USE CORE MODULES IN THIS FILE!
33// Return an error message, or null if everything is okay
34function checkConfig () {
35 const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
36
37 // NSFW policy
38 if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
39 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
40 }
41
42 // Redundancies
43 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
44 if (isArray(redundancyVideos)) {
45 for (const r of redundancyVideos) {
46 if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
47 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
48 }
49 }
50
51 const filtered = uniq(redundancyVideos.map(r => r.strategy))
52 if (filtered.length !== redundancyVideos.length) {
53 return 'Redundancy video entries should have unique strategies'
54 }
55
56 const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
57 if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
58 return 'Min views in recently added strategy is not a number'
59 }
60 }
61
62 if (isProdInstance()) {
63 const configStorage = config.get('storage')
64 for (const key of Object.keys(configStorage)) {
65 if (configStorage[key].startsWith('storage/')) {
66 logger.warn(
67 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
68 key
69 )
70 }
71 }
72 }
73
74 return null
75}
76 6
77// Check the config files 7// Check the config files
78function checkMissedConfig () { 8function checkMissedConfig () {
@@ -109,6 +39,14 @@ function checkMissedConfig () {
109 } 39 }
110 } 40 }
111 41
42 const redundancyVideos = config.get<any>('redundancy.videos.strategies')
43 if (isArray(redundancyVideos)) {
44 for (const r of redundancyVideos) {
45 if (!r.size) miss.push('redundancy.videos.strategies.size')
46 if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime')
47 }
48 }
49
112 const missingAlternatives = requiredAlternatives.filter( 50 const missingAlternatives = requiredAlternatives.filter(
113 set => !set.find(alternative => !alternative.find(key => !config.has(key))) 51 set => !set.find(alternative => !alternative.find(key => !config.has(key)))
114 ) 52 )
@@ -163,36 +101,10 @@ async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
163 } 101 }
164} 102}
165 103
166// We get db by param to not import it in this file (import orders)
167async function clientsExist () {
168 const totalClients = await OAuthClientModel.countTotal()
169
170 return totalClients !== 0
171}
172
173// We get db by param to not import it in this file (import orders)
174async function usersExist () {
175 const totalUsers = await UserModel.countTotal()
176
177 return totalUsers !== 0
178}
179
180// We get db by param to not import it in this file (import orders)
181async function applicationExist () {
182 const totalApplication = await ApplicationModel.countTotal()
183
184 return totalApplication !== 0
185}
186
187// --------------------------------------------------------------------------- 104// ---------------------------------------------------------------------------
188 105
189export { 106export {
190 checkConfig,
191 checkFFmpeg, 107 checkFFmpeg,
192 checkFFmpegEncoders, 108 checkFFmpegEncoders,
193 checkMissedConfig, 109 checkMissedConfig
194 clientsExist,
195 usersExist,
196 applicationExist,
197 checkActivityPubUrls
198} 110}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 03424ffb8..947cbda28 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -601,7 +601,6 @@ const MEMOIZE_TTL = {
601 601
602const REDUNDANCY = { 602const REDUNDANCY = {
603 VIDEOS: { 603 VIDEOS: {
604 EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
605 RANDOMIZED_FACTOR: 5 604 RANDOMIZED_FACTOR: 5
606 } 605 }
607} 606}
@@ -750,10 +749,16 @@ function updateWebserverConfig () {
750 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) 749 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
751} 750}
752 751
753function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] { 752function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
754 if (!objs) return [] 753 if (!objs) return []
755 754
756 return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) })) 755 return objs.map(obj => {
756 return Object.assign(obj, {
757 minLifetime: parseDuration(obj.min_lifetime),
758 size: bytes.parse(obj.size),
759 minViews: obj.min_views
760 })
761 })
757} 762}
758 763
759function buildLanguages () { 764function buildLanguages () {
diff --git a/server/initializers/index.ts b/server/initializers/index.ts
index 332702774..fe9190a9c 100644
--- a/server/initializers/index.ts
+++ b/server/initializers/index.ts
@@ -1,6 +1,5 @@
1// Constants first, database in second! 1// Constants first, database in second!
2export * from './constants' 2export * from './constants'
3export * from './database' 3export * from './database'
4export * from './checker'
5export * from './installer' 4export * from './installer'
6export * from './migrator' 5export * from './migrator'
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index 818bb04a2..c952ad46c 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -5,7 +5,7 @@ import { createApplicationActor, createUserAccountAndChannel } from '../lib/user
5import { UserModel } from '../models/account/user' 5import { UserModel } from '../models/account/user'
6import { ApplicationModel } from '../models/application/application' 6import { ApplicationModel } from '../models/application/application'
7import { OAuthClientModel } from '../models/oauth/oauth-client' 7import { OAuthClientModel } from '../models/oauth/oauth-client'
8import { applicationExist, clientsExist, usersExist } from './checker' 8import { applicationExist, clientsExist, usersExist } from './checker-after-init'
9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' 9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants'
10import { sequelizeTypescript } from './database' 10import { sequelizeTypescript } from './database'
11import { remove, ensureDir } from 'fs-extra' 11import { remove, ensureDir } from 'fs-extra'
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index a86428461..10277eca7 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -50,7 +50,12 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): Acti
50 50
51async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) { 51async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) {
52 const actors = await VideoShareModel.loadActorsByShare(video.id, t) 52 const actors = await VideoShareModel.loadActorsByShare(video.id, t)
53 actors.push(video.VideoChannel.Account.Actor) 53
54 const videoActor = video.VideoChannel && video.VideoChannel.Account
55 ? video.VideoChannel.Account.Actor
56 : await ActorModel.loadAccountActorByVideoId(video.id, t)
57
58 actors.push(videoActor)
54 59
55 return actors 60 return actors
56} 61}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 87f8a4162..5286d8e6d 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,7 +1,7 @@
1import { CacheFileObject } from '../../../shared/index' 1import { CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { sequelizeTypescript } from '../../initializers'
4import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize'
5 5
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url 7 const url = cacheFileObject.url
@@ -22,25 +22,29 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
22 } 22 }
23} 23}
24 24
25function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 25function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }, t: Transaction) {
26 return sequelizeTypescript.transaction(async t => { 26 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
27 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
28 27
29 return VideoRedundancyModel.create(attributes, { transaction: t }) 28 return VideoRedundancyModel.create(attributes, { transaction: t })
30 })
31} 29}
32 30
33function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) { 31function updateCacheFile (
32 cacheFileObject: CacheFileObject,
33 redundancyModel: VideoRedundancyModel,
34 video: VideoModel,
35 byActor: { id?: number },
36 t: Transaction
37) {
34 if (redundancyModel.actorId !== byActor.id) { 38 if (redundancyModel.actorId !== byActor.id) {
35 throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') 39 throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
36 } 40 }
37 41
38 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) 42 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
39 43
40 redundancyModel.set('expires', attributes.expiresOn) 44 redundancyModel.set('expires', attributes.expiresOn)
41 redundancyModel.set('fileUrl', attributes.fileUrl) 45 redundancyModel.set('fileUrl', attributes.fileUrl)
42 46
43 return redundancyModel.save() 47 return redundancyModel.save({ transaction: t })
44} 48}
45 49
46export { 50export {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index cff8dcfc6..ceb5413ca 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -95,7 +95,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
95 if (video.isOwned()) { 95 if (video.isOwned()) {
96 // Don't resend the activity to the sender 96 // Don't resend the activity to the sender
97 const exceptions = [ byActor ] 97 const exceptions = [ byActor ]
98 await forwardActivity(activity, undefined, exceptions) 98 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
99 } 99 }
100} 100}
101 101
@@ -104,12 +104,14 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate)
104 104
105 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 105 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
106 106
107 await createCacheFile(cacheFile, video, byActor) 107 await sequelizeTypescript.transaction(async t => {
108 return createCacheFile(cacheFile, video, byActor, t)
109 })
108 110
109 if (video.isOwned()) { 111 if (video.isOwned()) {
110 // Don't resend the activity to the sender 112 // Don't resend the activity to the sender
111 const exceptions = [ byActor ] 113 const exceptions = [ byActor ]
112 await forwardActivity(activity, undefined, exceptions) 114 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
113 } 115 }
114} 116}
115 117
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 73ca0a17c..ff019cd8c 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -100,7 +100,7 @@ async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo
100 100
101 return sequelizeTypescript.transaction(async t => { 101 return sequelizeTypescript.transaction(async t => {
102 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 102 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
103 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) 103 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFileObject.id)
104 104
105 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') 105 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
106 106
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index ed3489ebf..e092a6729 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -12,6 +12,7 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-vali
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
14import { createCacheFile, updateCacheFile } from '../cache-file' 14import { createCacheFile, updateCacheFile } from '../cache-file'
15import { forwardVideoRelatedActivity } from '../send/utils'
15 16
16async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { 17async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
17 const objectType = activity.object.type 18 const objectType = activity.object.type
@@ -68,18 +69,29 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
68async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { 69async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
69 const cacheFileObject = activity.object as CacheFileObject 70 const cacheFileObject = activity.object as CacheFileObject
70 71
71 if (!isCacheFileObjectValid(cacheFileObject) === false) { 72 if (!isCacheFileObjectValid(cacheFileObject)) {
72 logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject }) 73 logger.debug('Cache file object sent by update is not valid.', { cacheFileObject })
73 return undefined 74 return undefined
74 } 75 }
75 76
76 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 77 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
77 if (!redundancyModel) { 78
78 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) 79 await sequelizeTypescript.transaction(async t => {
79 return createCacheFile(cacheFileObject, video, byActor) 80 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
80 } 81
82 if (!redundancyModel) {
83 await createCacheFile(cacheFileObject, video, byActor, t)
84 } else {
85 await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
86 }
87 })
88
89 if (video.isOwned()) {
90 // Don't resend the activity to the sender
91 const exceptions = [ byActor ]
81 92
82 return updateCacheFile(cacheFileObject, redundancyModel, byActor) 93 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
94 }
83} 95}
84 96
85async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { 97async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index ec46789b7..a68f03edf 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video'
7import { VideoChannelModel } from '../../../models/video/video-channel' 7import { VideoChannelModel } from '../../../models/video/video-channel'
8import { VideoShareModel } from '../../../models/video/video-share' 8import { VideoShareModel } from '../../../models/video/video-share'
9import { getUpdateActivityPubUrl } from '../url' 9import { getUpdateActivityPubUrl } from '../url'
10import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 10import { broadcastToFollowers, sendVideoRelatedActivity } from './utils'
11import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' 11import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience'
12import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
13import { VideoCaptionModel } from '../../../models/video/video-caption' 13import { VideoCaptionModel } from '../../../models/video/video-caption'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 48c0e0a5c..db72ef23c 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -176,7 +176,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
176 syncParam, 176 syncParam,
177 refreshViews 177 refreshViews
178 } 178 }
179 const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) 179 const p = refreshVideoIfNeeded(refreshOptions)
180 if (syncParam.refreshVideo === true) videoFromDatabase = await p 180 if (syncParam.refreshVideo === true) videoFromDatabase = await p
181 181
182 return { video: videoFromDatabase } 182 return { video: videoFromDatabase }
@@ -245,29 +245,37 @@ async function updateVideoFromAP (options: {
245 generateThumbnailFromUrl(options.video, options.videoObject.icon) 245 generateThumbnailFromUrl(options.video, options.videoObject.icon)
246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) 246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
247 247
248 // Remove old video files 248 {
249 const videoFileDestroyTasks: Bluebird<void>[] = [] 249 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
250 for (const videoFile of options.video.VideoFiles) { 250 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
251 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
252 }
253 await Promise.all(videoFileDestroyTasks)
254 251
255 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) 252 // Remove video files that do not exist anymore
256 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) 253 const destroyTasks = options.video.VideoFiles
257 await Promise.all(tasks) 254 .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
255 .map(f => f.destroy(sequelizeOptions))
256 await Promise.all(destroyTasks)
258 257
259 // Update Tags 258 // Update or add other one
260 const tags = options.videoObject.tag.map(tag => tag.name) 259 const upsertTasks = videoFileAttributes.map(a => VideoFileModel.upsert(a, sequelizeOptions))
261 const tagInstances = await TagModel.findOrCreateTags(tags, t) 260 await Promise.all(upsertTasks)
262 await options.video.$set('Tags', tagInstances, sequelizeOptions) 261 }
263 262
264 // Update captions 263 {
265 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) 264 // Update Tags
265 const tags = options.videoObject.tag.map(tag => tag.name)
266 const tagInstances = await TagModel.findOrCreateTags(tags, t)
267 await options.video.$set('Tags', tagInstances, sequelizeOptions)
268 }
266 269
267 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { 270 {
268 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) 271 // Update captions
269 }) 272 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
270 await Promise.all(videoCaptionsPromises) 273
274 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
275 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
276 })
277 await Promise.all(videoCaptionsPromises)
278 }
271 }) 279 })
272 280
273 logger.info('Remote video with uuid %s updated', options.videoObject.uuid) 281 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
@@ -382,7 +390,7 @@ async function refreshVideoIfNeeded (options: {
382 channel: channelActor.VideoChannel, 390 channel: channelActor.VideoChannel,
383 updateViews: options.refreshViews 391 updateViews: options.refreshViews
384 } 392 }
385 await updateVideoFromAP(updateOptions) 393 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
386 await syncVideoExternalAttributes(video, videoObject, options.syncParam) 394 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
387 } catch (err) { 395 } catch (err) {
388 logger.warn('Cannot refresh video.', { err }) 396 logger.warn('Cannot refresh video.', { err })
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts
index 7bf63790a..54eb983fa 100644
--- a/server/lib/cache/index.ts
+++ b/server/lib/cache/index.ts
@@ -1 +1,2 @@
1export * from './videos-preview-cache' 1export * from './videos-preview-cache'
2export * from './videos-caption-cache'
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 78221cc3d..16b122658 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -6,7 +6,8 @@ import { getServerActor } from '../helpers/utils'
6async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) { 6async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) {
7 const serverActor = await getServerActor() 7 const serverActor = await getServerActor()
8 8
9 await sendUndoCacheFile(serverActor, videoRedundancy, t) 9 // Local cache, send undo to remote instances
10 if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t)
10 11
11 await videoRedundancy.destroy({ transaction: t }) 12 await videoRedundancy.destroy({ transaction: t })
12} 13}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 998d2295a..97df3e4f5 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 7import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
@@ -12,6 +12,7 @@ import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { VideoModel } from '../../models/video/video' 12import { VideoModel } from '../../models/video/video'
13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
14import { isTestInstance } from '../../helpers/core-utils' 14import { isTestInstance } from '../../helpers/core-utils'
15import { removeVideoRedundancy } from '../redundancy'
15 16
16export class VideosRedundancyScheduler extends AbstractScheduler { 17export class VideosRedundancyScheduler extends AbstractScheduler {
17 18
@@ -30,7 +31,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
30 this.executing = true 31 this.executing = true
31 32
32 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 33 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
33 if (!isTestInstance()) logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) 34 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
34 35
35 try { 36 try {
36 const videoToDuplicate = await this.findVideoToDuplicate(obj) 37 const videoToDuplicate = await this.findVideoToDuplicate(obj)
@@ -39,20 +40,24 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
39 const videoFiles = videoToDuplicate.VideoFiles 40 const videoFiles = videoToDuplicate.VideoFiles
40 videoFiles.forEach(f => f.Video = videoToDuplicate) 41 videoFiles.forEach(f => f.Video = videoToDuplicate)
41 42
42 if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { 43 await this.purgeCacheIfNeeded(obj, videoFiles)
43 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 44
45 if (await this.isTooHeavy(obj, videoFiles)) {
46 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
44 continue 47 continue
45 } 48 }
46 49
47 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 50 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
48 51
49 await this.createVideoRedundancy(obj.strategy, videoFiles) 52 await this.createVideoRedundancy(obj, videoFiles)
50 } catch (err) { 53 } catch (err) {
51 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 54 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
52 } 55 }
53 } 56 }
54 57
55 await this.removeExpired() 58 await this.extendsLocalExpiration()
59
60 await this.purgeRemoteExpired()
56 61
57 this.executing = false 62 this.executing = false
58 } 63 }
@@ -61,16 +66,27 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
61 return this.instance || (this.instance = new this()) 66 return this.instance || (this.instance = new this())
62 } 67 }
63 68
64 private async removeExpired () { 69 private async extendsLocalExpiration () {
65 const expired = await VideoRedundancyModel.listAllExpired() 70 const expired = await VideoRedundancyModel.listLocalExpired()
71
72 for (const redundancyModel of expired) {
73 try {
74 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
75 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
76 } catch (err) {
77 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel))
78 }
79 }
80 }
66 81
67 for (const m of expired) { 82 private async purgeRemoteExpired () {
68 logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m)) 83 const expired = await VideoRedundancyModel.listRemoteExpired()
69 84
85 for (const redundancyModel of expired) {
70 try { 86 try {
71 await m.destroy() 87 await removeVideoRedundancy(redundancyModel)
72 } catch (err) { 88 } catch (err) {
73 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) 89 logger.error('Cannot remove redundancy %s from our redundancy system.', this.buildEntryLogId(redundancyModel))
74 } 90 }
75 } 91 }
76 } 92 }
@@ -90,18 +106,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
90 } 106 }
91 } 107 }
92 108
93 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 109 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
94 const serverActor = await getServerActor() 110 const serverActor = await getServerActor()
95 111
96 for (const file of filesToDuplicate) { 112 for (const file of filesToDuplicate) {
97 const existing = await VideoRedundancyModel.loadByFileId(file.id) 113 const existing = await VideoRedundancyModel.loadByFileId(file.id)
98 if (existing) { 114 if (existing) {
99 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy) 115 await this.extendsExpirationOf(existing, redundancy.minLifetime)
100 116
101 existing.expiresOn = this.buildNewExpiration()
102 await existing.save()
103
104 await sendUpdateCacheFile(serverActor, existing)
105 continue 117 continue
106 } 118 }
107 119
@@ -109,7 +121,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
109 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) 121 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
110 if (!video) continue 122 if (!video) continue
111 123
112 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) 124 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
113 125
114 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 126 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
115 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 127 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
@@ -120,10 +132,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
120 await rename(tmpPath, destPath) 132 await rename(tmpPath, destPath)
121 133
122 const createdModel = await VideoRedundancyModel.create({ 134 const createdModel = await VideoRedundancyModel.create({
123 expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS), 135 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
124 url: getVideoCacheFileActivityPubUrl(file), 136 url: getVideoCacheFileActivityPubUrl(file),
125 fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), 137 fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
126 strategy, 138 strategy: redundancy.strategy,
127 videoFileId: file.id, 139 videoFileId: file.id,
128 actorId: serverActor.id 140 actorId: serverActor.id
129 }) 141 })
@@ -133,16 +145,36 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
133 } 145 }
134 } 146 }
135 147
136 private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { 148 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
137 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) 149 logger.info('Extending expiration of %s.', redundancy.url)
150
151 const serverActor = await getServerActor()
152
153 redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs)
154 await redundancy.save()
155
156 await sendUpdateCacheFile(serverActor, redundancy)
157 }
158
159 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
160 while (this.isTooHeavy(redundancy, filesToDuplicate)) {
161 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
162 if (!toDelete) return
163
164 await removeVideoRedundancy(toDelete)
165 }
166 }
167
168 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
169 const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate)
138 170
139 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) 171 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
140 172
141 return totalDuplicated > maxSize 173 return totalDuplicated > maxSize
142 } 174 }
143 175
144 private buildNewExpiration () { 176 private buildNewExpiration (expiresAfterMs: number) {
145 return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS) 177 return new Date(Date.now() + expiresAfterMs)
146 } 178 }
147 179
148 private buildEntryLogId (object: VideoRedundancyModel) { 180 private buildEntryLogId (object: VideoRedundancyModel) {
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index f8bb59323..12b83916e 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -37,6 +37,7 @@ import { ServerModel } from '../server/server'
37import { throwIfNotValid } from '../utils' 37import { throwIfNotValid } from '../utils'
38import { VideoChannelModel } from '../video/video-channel' 38import { VideoChannelModel } from '../video/video-channel'
39import { ActorFollowModel } from './actor-follow' 39import { ActorFollowModel } from './actor-follow'
40import { VideoModel } from '../video/video'
40 41
41enum ScopeNames { 42enum ScopeNames {
42 FULL = 'FULL' 43 FULL = 'FULL'
@@ -266,6 +267,36 @@ export class ActorModel extends Model<ActorModel> {
266 return ActorModel.unscoped().findById(id) 267 return ActorModel.unscoped().findById(id)
267 } 268 }
268 269
270 static loadAccountActorByVideoId (videoId: number, transaction: Sequelize.Transaction) {
271 const query = {
272 include: [
273 {
274 attributes: [ 'id' ],
275 model: AccountModel.unscoped(),
276 required: true,
277 include: [
278 {
279 attributes: [ 'id' ],
280 model: VideoChannelModel.unscoped(),
281 required: true,
282 include: {
283 attributes: [ 'id' ],
284 model: VideoModel.unscoped(),
285 required: true,
286 where: {
287 id: videoId
288 }
289 }
290 }
291 ]
292 }
293 ],
294 transaction
295 }
296
297 return ActorModel.unscoped().findOne(query as any) // FIXME: typings
298 }
299
269 static isActorUrlExist (url: string) { 300 static isActorUrlExist (url: string) {
270 const query = { 301 const query = {
271 raw: true, 302 raw: true,
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index fb07287a8..970d2fe06 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -9,7 +9,6 @@ import {
9 Is, 9 Is,
10 Model, 10 Model,
11 Scopes, 11 Scopes,
12 Sequelize,
13 Table, 12 Table,
14 UpdatedAt 13 UpdatedAt
15} from 'sequelize-typescript' 14} from 'sequelize-typescript'
@@ -28,6 +27,7 @@ import { ServerModel } from '../server/server'
28import { sample } from 'lodash' 27import { sample } from 'lodash'
29import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
30import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import * as Sequelize from 'sequelize'
31 31
32export enum ScopeNames { 32export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO' 33 WITH_VIDEO = 'WITH_VIDEO'
@@ -116,11 +116,11 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
116 Actor: ActorModel 116 Actor: ActorModel
117 117
118 @AfterDestroy 118 @AfterDestroy
119 static removeFilesAndSendDelete (instance: VideoRedundancyModel) { 119 static removeFile (instance: VideoRedundancyModel) {
120 // Not us 120 // Not us
121 if (!instance.strategy) return 121 if (!instance.strategy) return
122 122
123 logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) 123 logger.info('Removing duplicated video file %s-%s.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
124 124
125 return instance.VideoFile.Video.removeFile(instance.VideoFile) 125 return instance.VideoFile.Video.removeFile(instance.VideoFile)
126 } 126 }
@@ -135,11 +135,12 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
135 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 135 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
136 } 136 }
137 137
138 static loadByUrl (url: string) { 138 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
139 const query = { 139 const query = {
140 where: { 140 where: {
141 url 141 url
142 } 142 },
143 transaction
143 } 144 }
144 145
145 return VideoRedundancyModel.findOne(query) 146 return VideoRedundancyModel.findOne(query)
@@ -157,7 +158,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
157 // On VideoModel! 158 // On VideoModel!
158 const query = { 159 const query = {
159 attributes: [ 'id', 'views' ], 160 attributes: [ 'id', 'views' ],
160 logging: !isTestInstance(),
161 limit: randomizedFactor, 161 limit: randomizedFactor,
162 order: getVideoSort('-views'), 162 order: getVideoSort('-views'),
163 include: [ 163 include: [
@@ -174,7 +174,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
174 const query = { 174 const query = {
175 attributes: [ 'id', 'views' ], 175 attributes: [ 'id', 'views' ],
176 subQuery: false, 176 subQuery: false,
177 logging: !isTestInstance(),
178 group: 'VideoModel.id', 177 group: 'VideoModel.id',
179 limit: randomizedFactor, 178 limit: randomizedFactor,
180 order: getVideoSort('-trending'), 179 order: getVideoSort('-trending'),
@@ -193,7 +192,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
193 // On VideoModel! 192 // On VideoModel!
194 const query = { 193 const query = {
195 attributes: [ 'id', 'publishedAt' ], 194 attributes: [ 'id', 'publishedAt' ],
196 logging: !isTestInstance(),
197 limit: randomizedFactor, 195 limit: randomizedFactor,
198 order: getVideoSort('-publishedAt'), 196 order: getVideoSort('-publishedAt'),
199 where: { 197 where: {
@@ -210,11 +208,29 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
210 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) 208 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
211 } 209 }
212 210
211 static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
212 const expiredDate = new Date()
213 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
214
215 const actor = await getServerActor()
216
217 const query = {
218 where: {
219 actorId: actor.id,
220 strategy,
221 createdAt: {
222 [ Sequelize.Op.lt ]: expiredDate
223 }
224 }
225 }
226
227 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
228 }
229
213 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { 230 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
214 const actor = await getServerActor() 231 const actor = await getServerActor()
215 232
216 const options = { 233 const options = {
217 logging: !isTestInstance(),
218 include: [ 234 include: [
219 { 235 {
220 attributes: [], 236 attributes: [],
@@ -228,21 +244,39 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
228 ] 244 ]
229 } 245 }
230 246
231 return VideoFileModel.sum('size', options) 247 return VideoFileModel.sum('size', options as any) // FIXME: typings
232 } 248 }
233 249
234 static listAllExpired () { 250 static async listLocalExpired () {
251 const actor = await getServerActor()
252
253 const query = {
254 where: {
255 actorId: actor.id,
256 expiresOn: {
257 [ Sequelize.Op.lt ]: new Date()
258 }
259 }
260 }
261
262 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
263 }
264
265 static async listRemoteExpired () {
266 const actor = await getServerActor()
267
235 const query = { 268 const query = {
236 logging: !isTestInstance(),
237 where: { 269 where: {
270 actorId: {
271 [Sequelize.Op.ne]: actor.id
272 },
238 expiresOn: { 273 expiresOn: {
239 [ Sequelize.Op.lt ]: new Date() 274 [ Sequelize.Op.lt ]: new Date()
240 } 275 }
241 } 276 }
242 } 277 }
243 278
244 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) 279 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
245 .findAll(query)
246 } 280 }
247 281
248 static async getStats (strategy: VideoRedundancyStrategy) { 282 static async getStats (strategy: VideoRedundancyStrategy) {
@@ -299,7 +333,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
299 333
300 const notIn = Sequelize.literal( 334 const notIn = Sequelize.literal(
301 '(' + 335 '(' +
302 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + 336 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
303 ')' 337 ')'
304 ) 338 )
305 339
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 0907ea569..0887a3738 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -106,4 +106,10 @@ export class VideoFileModel extends Model<VideoFileModel> {
106 return results.length === 1 106 return results.length === 1
107 }) 107 })
108 } 108 }
109
110 hasSameUniqueKeysThan (other: VideoFileModel) {
111 return this.fps === other.fps &&
112 this.resolution === other.resolution &&
113 this.videoId === other.videoId
114 }
109} 115}
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 6ce4b9dd1..a773e3de4 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -31,14 +31,13 @@ const expect = chai.expect
31 31
32let servers: ServerInfo[] = [] 32let servers: ServerInfo[] = []
33let video1Server2UUID: string 33let video1Server2UUID: string
34let video2Server2UUID: string
35 34
36function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { 35function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
37 const parsed = magnetUtil.decode(file.magnetUri) 36 const parsed = magnetUtil.decode(file.magnetUri)
38 37
39 for (const ws of baseWebseeds) { 38 for (const ws of baseWebseeds) {
40 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) 39 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
41 expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined 40 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
42 } 41 }
43} 42}
44 43
@@ -49,6 +48,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
49 check_interval: '5 seconds', 48 check_interval: '5 seconds',
50 strategies: [ 49 strategies: [
51 immutableAssign({ 50 immutableAssign({
51 min_lifetime: '1 hour',
52 strategy: strategy, 52 strategy: strategy,
53 size: '100KB' 53 size: '100KB'
54 }, additionalParams) 54 }, additionalParams)
@@ -68,11 +68,6 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
68 await viewVideo(servers[ 1 ].url, video1Server2UUID) 68 await viewVideo(servers[ 1 ].url, video1Server2UUID)
69 } 69 }
70 70
71 {
72 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
73 video2Server2UUID = res.body.video.uuid
74 }
75
76 await waitJobs(servers) 71 await waitJobs(servers)
77 72
78 // Server 1 and server 2 follow each other 73 // Server 1 and server 2 follow each other
@@ -85,68 +80,69 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
85 await waitJobs(servers) 80 await waitJobs(servers)
86} 81}
87 82
88async function check1WebSeed (strategy: VideoRedundancyStrategy) { 83async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
84 if (!videoUUID) videoUUID = video1Server2UUID
85
89 const webseeds = [ 86 const webseeds = [
90 'http://localhost:9002/static/webseed/' + video1Server2UUID 87 'http://localhost:9002/static/webseed/' + videoUUID
91 ] 88 ]
92 89
93 for (const server of servers) { 90 for (const server of servers) {
94 { 91 {
95 const res = await getVideo(server.url, video1Server2UUID) 92 const res = await getVideo(server.url, videoUUID)
96 93
97 const video: VideoDetails = res.body 94 const video: VideoDetails = res.body
98 video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) 95 for (const f of video.files) {
96 checkMagnetWebseeds(f, webseeds, server)
97 }
99 } 98 }
99 }
100}
100 101
101 { 102async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
102 const res = await getStats(server.url) 103 const res = await getStats(servers[0].url)
103 const data: ServerStats = res.body 104 const data: ServerStats = res.body
104 105
105 expect(data.videosRedundancy).to.have.lengthOf(1) 106 expect(data.videosRedundancy).to.have.lengthOf(1)
107 const stat = data.videosRedundancy[0]
106 108
107 const stat = data.videosRedundancy[0] 109 expect(stat.strategy).to.equal(strategy)
108 expect(stat.strategy).to.equal(strategy) 110 expect(stat.totalSize).to.equal(102400)
109 expect(stat.totalSize).to.equal(102400) 111 expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
110 expect(stat.totalUsed).to.equal(0) 112 expect(stat.totalVideoFiles).to.equal(4)
111 expect(stat.totalVideoFiles).to.equal(0) 113 expect(stat.totalVideos).to.equal(1)
112 expect(stat.totalVideos).to.equal(0)
113 }
114 }
115} 114}
116 115
117async function enableRedundancy () { 116async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
118 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) 117 const res = await getStats(servers[0].url)
118 const data: ServerStats = res.body
119 119
120 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') 120 expect(data.videosRedundancy).to.have.lengthOf(1)
121 const follows: ActorFollow[] = res.body.data
122 const server2 = follows.find(f => f.following.host === 'localhost:9002')
123 const server3 = follows.find(f => f.following.host === 'localhost:9003')
124
125 expect(server3).to.not.be.undefined
126 expect(server3.following.hostRedundancyAllowed).to.be.false
127 121
128 expect(server2).to.not.be.undefined 122 const stat = data.videosRedundancy[0]
129 expect(server2.following.hostRedundancyAllowed).to.be.true 123 expect(stat.strategy).to.equal(strategy)
124 expect(stat.totalSize).to.equal(102400)
125 expect(stat.totalUsed).to.equal(0)
126 expect(stat.totalVideoFiles).to.equal(0)
127 expect(stat.totalVideos).to.equal(0)
130} 128}
131 129
132async function check2Webseeds (strategy: VideoRedundancyStrategy) { 130async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
133 await waitJobs(servers) 131 if (!videoUUID) videoUUID = video1Server2UUID
134 await wait(15000)
135 await waitJobs(servers)
136 132
137 const webseeds = [ 133 const webseeds = [
138 'http://localhost:9001/static/webseed/' + video1Server2UUID, 134 'http://localhost:9001/static/webseed/' + videoUUID,
139 'http://localhost:9002/static/webseed/' + video1Server2UUID 135 'http://localhost:9002/static/webseed/' + videoUUID
140 ] 136 ]
141 137
142 for (const server of servers) { 138 for (const server of servers) {
143 { 139 {
144 const res = await getVideo(server.url, video1Server2UUID) 140 const res = await getVideo(server.url, videoUUID)
145 141
146 const video: VideoDetails = res.body 142 const video: VideoDetails = res.body
147 143
148 for (const file of video.files) { 144 for (const file of video.files) {
149 checkMagnetWebseeds(file, webseeds) 145 checkMagnetWebseeds(file, webseeds, server)
150 } 146 }
151 } 147 }
152 } 148 }
@@ -155,22 +151,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy) {
155 expect(files).to.have.lengthOf(4) 151 expect(files).to.have.lengthOf(4)
156 152
157 for (const resolution of [ 240, 360, 480, 720 ]) { 153 for (const resolution of [ 240, 360, 480, 720 ]) {
158 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined 154 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
159 } 155 }
156}
160 157
161 { 158async function enableRedundancyOnServer1 () {
162 const res = await getStats(servers[0].url) 159 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
163 const data: ServerStats = res.body
164 160
165 expect(data.videosRedundancy).to.have.lengthOf(1) 161 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
166 const stat = data.videosRedundancy[0] 162 const follows: ActorFollow[] = res.body.data
163 const server2 = follows.find(f => f.following.host === 'localhost:9002')
164 const server3 = follows.find(f => f.following.host === 'localhost:9003')
167 165
168 expect(stat.strategy).to.equal(strategy) 166 expect(server3).to.not.be.undefined
169 expect(stat.totalSize).to.equal(102400) 167 expect(server3.following.hostRedundancyAllowed).to.be.false
170 expect(stat.totalUsed).to.be.at.least(1).and.below(102401) 168
171 expect(stat.totalVideoFiles).to.equal(4) 169 expect(server2).to.not.be.undefined
172 expect(stat.totalVideos).to.equal(1) 170 expect(server2.following.hostRedundancyAllowed).to.be.true
173 }
174} 171}
175 172
176async function cleanServers () { 173async function cleanServers () {
@@ -188,18 +185,24 @@ describe('Test videos redundancy', function () {
188 return runServers(strategy) 185 return runServers(strategy)
189 }) 186 })
190 187
191 it('Should have 1 webseed on the first video', function () { 188 it('Should have 1 webseed on the first video', async function () {
192 return check1WebSeed(strategy) 189 await check1WebSeed(strategy)
190 await checkStatsWith1Webseed(strategy)
193 }) 191 })
194 192
195 it('Should enable redundancy on server 1', function () { 193 it('Should enable redundancy on server 1', function () {
196 return enableRedundancy() 194 return enableRedundancyOnServer1()
197 }) 195 })
198 196
199 it('Should have 2 webseed on the first video', function () { 197 it('Should have 2 webseed on the first video', async function () {
200 this.timeout(40000) 198 this.timeout(40000)
201 199
202 return check2Webseeds(strategy) 200 await waitJobs(servers)
201 await wait(15000)
202 await waitJobs(servers)
203
204 await check2Webseeds(strategy)
205 await checkStatsWith2Webseed(strategy)
203 }) 206 })
204 207
205 after(function () { 208 after(function () {
@@ -216,18 +219,24 @@ describe('Test videos redundancy', function () {
216 return runServers(strategy) 219 return runServers(strategy)
217 }) 220 })
218 221
219 it('Should have 1 webseed on the first video', function () { 222 it('Should have 1 webseed on the first video', async function () {
220 return check1WebSeed(strategy) 223 await check1WebSeed(strategy)
224 await checkStatsWith1Webseed(strategy)
221 }) 225 })
222 226
223 it('Should enable redundancy on server 1', function () { 227 it('Should enable redundancy on server 1', function () {
224 return enableRedundancy() 228 return enableRedundancyOnServer1()
225 }) 229 })
226 230
227 it('Should have 2 webseed on the first video', function () { 231 it('Should have 2 webseed on the first video', async function () {
228 this.timeout(40000) 232 this.timeout(40000)
229 233
230 return check2Webseeds(strategy) 234 await waitJobs(servers)
235 await wait(15000)
236 await waitJobs(servers)
237
238 await check2Webseeds(strategy)
239 await checkStatsWith2Webseed(strategy)
231 }) 240 })
232 241
233 after(function () { 242 after(function () {
@@ -241,15 +250,16 @@ describe('Test videos redundancy', function () {
241 before(function () { 250 before(function () {
242 this.timeout(120000) 251 this.timeout(120000)
243 252
244 return runServers(strategy, { minViews: 3 }) 253 return runServers(strategy, { min_views: 3 })
245 }) 254 })
246 255
247 it('Should have 1 webseed on the first video', function () { 256 it('Should have 1 webseed on the first video', async function () {
248 return check1WebSeed(strategy) 257 await check1WebSeed(strategy)
258 await checkStatsWith1Webseed(strategy)
249 }) 259 })
250 260
251 it('Should enable redundancy on server 1', function () { 261 it('Should enable redundancy on server 1', function () {
252 return enableRedundancy() 262 return enableRedundancyOnServer1()
253 }) 263 })
254 264
255 it('Should still have 1 webseed on the first video', async function () { 265 it('Should still have 1 webseed on the first video', async function () {
@@ -259,10 +269,11 @@ describe('Test videos redundancy', function () {
259 await wait(15000) 269 await wait(15000)
260 await waitJobs(servers) 270 await waitJobs(servers)
261 271
262 return check1WebSeed(strategy) 272 await check1WebSeed(strategy)
273 await checkStatsWith1Webseed(strategy)
263 }) 274 })
264 275
265 it('Should view 2 times the first video', async function () { 276 it('Should view 2 times the first video to have > min_views config', async function () {
266 this.timeout(40000) 277 this.timeout(40000)
267 278
268 await viewVideo(servers[ 0 ].url, video1Server2UUID) 279 await viewVideo(servers[ 0 ].url, video1Server2UUID)
@@ -272,10 +283,117 @@ describe('Test videos redundancy', function () {
272 await waitJobs(servers) 283 await waitJobs(servers)
273 }) 284 })
274 285
275 it('Should have 2 webseed on the first video', function () { 286 it('Should have 2 webseed on the first video', async function () {
276 this.timeout(40000) 287 this.timeout(40000)
277 288
278 return check2Webseeds(strategy) 289 await waitJobs(servers)
290 await wait(15000)
291 await waitJobs(servers)
292
293 await check2Webseeds(strategy)
294 await checkStatsWith2Webseed(strategy)
295 })
296
297 after(function () {
298 return cleanServers()
299 })
300 })
301
302 describe('Test expiration', function () {
303 const strategy = 'recently-added'
304
305 async function checkContains (servers: ServerInfo[], str: string) {
306 for (const server of servers) {
307 const res = await getVideo(server.url, video1Server2UUID)
308 const video: VideoDetails = res.body
309
310 for (const f of video.files) {
311 expect(f.magnetUri).to.contain(str)
312 }
313 }
314 }
315
316 async function checkNotContains (servers: ServerInfo[], str: string) {
317 for (const server of servers) {
318 const res = await getVideo(server.url, video1Server2UUID)
319 const video: VideoDetails = res.body
320
321 for (const f of video.files) {
322 expect(f.magnetUri).to.not.contain(str)
323 }
324 }
325 }
326
327 before(async function () {
328 this.timeout(120000)
329
330 await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
331
332 await enableRedundancyOnServer1()
333 })
334
335 it('Should still have 2 webseeds after 10 seconds', async function () {
336 this.timeout(40000)
337
338 await wait(10000)
339
340 try {
341 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
342 } catch {
343 // Maybe a server deleted a redundancy in the scheduler
344 await wait(2000)
345
346 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
347 }
348 })
349
350 it('Should stop server 1 and expire video redundancy', async function () {
351 this.timeout(40000)
352
353 killallServers([ servers[0] ])
354
355 await wait(10000)
356
357 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
358 })
359
360 after(function () {
361 return killallServers([ servers[1], servers[2] ])
362 })
363 })
364
365 describe('Test file replacement', function () {
366 let video2Server2UUID: string
367 const strategy = 'recently-added'
368
369 before(async function () {
370 this.timeout(120000)
371
372 await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
373
374 await enableRedundancyOnServer1()
375
376 await waitJobs(servers)
377 await wait(5000)
378 await waitJobs(servers)
379
380 await check2Webseeds(strategy)
381 await checkStatsWith2Webseed(strategy)
382
383 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
384 video2Server2UUID = res.body.video.uuid
385 })
386
387 it('Should cache video 2 webseed on the first video', async function () {
388 this.timeout(40000)
389 this.retries(3)
390
391 await waitJobs(servers)
392
393 await wait(7000)
394
395 await check1WebSeed(strategy, video1Server2UUID)
396 await check2Webseeds(strategy, video2Server2UUID)
279 }) 397 })
280 398
281 after(function () { 399 after(function () {
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 26ab4e1bb..fbfc83ca1 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -144,8 +144,8 @@ function runServer (serverNumber: number, configOverride?: Object) {
144 }) 144 })
145} 145}
146 146
147async function reRunServer (server: ServerInfo) { 147async function reRunServer (server: ServerInfo, configOverride?: any) {
148 const newServer = await runServer(server.serverNumber) 148 const newServer = await runServer(server.serverNumber, configOverride)
149 server.app = newServer.app 149 server.app = newServer.app
150 150
151 return server 151 return server
diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts
index 436394c1e..a8c2743c1 100644
--- a/shared/models/redundancy/videos-redundancy.model.ts
+++ b/shared/models/redundancy/videos-redundancy.model.ts
@@ -3,17 +3,20 @@ export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-adde
3export type MostViewsRedundancyStrategy = { 3export type MostViewsRedundancyStrategy = {
4 strategy: 'most-views' 4 strategy: 'most-views'
5 size: number 5 size: number
6 minLifetime: number
6} 7}
7 8
8export type TrendingRedundancyStrategy = { 9export type TrendingRedundancyStrategy = {
9 strategy: 'trending' 10 strategy: 'trending'
10 size: number 11 size: number
12 minLifetime: number
11} 13}
12 14
13export type RecentlyAddedStrategy = { 15export type RecentlyAddedStrategy = {
14 strategy: 'recently-added' 16 strategy: 'recently-added'
15 size: number 17 size: number
16 minViews: number 18 minViews: number
19 minLifetime: number
17} 20}
18 21
19export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy 22export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy
diff --git a/support/doc/redundancy.md b/support/doc/redundancy.md
new file mode 100644
index 000000000..62c6365a9
--- /dev/null
+++ b/support/doc/redundancy.md
@@ -0,0 +1,46 @@
1# Redundancy
2
3A PeerTube instance can cache other PeerTube videos to improve bandwidth of popular videos or small instances.
4
5## How it works
6
7The instance administrator can choose between multiple redundancy strategies (cache trending videos or recently uploaded videos etc), set their maximum size and the minimum duplication lifetime.
8Then, they choose the instances they want to cache in `Manage follows -> Following` admin table.
9
10Videos are kept in the cache for at least `min_lifetime`, and then evicted when the cache is full.
11
12When PeerTube chooses a video to duplicate, it imports all the resolution files (to avoid consistency issues) using their magnet URI and put them in the `storage.videos` directory.
13Then it sends a `Create -> CacheFile` ActivityPub message to other federated instances. This new instance is injected as [WebSeed](https://github.com/Chocobozzz/PeerTube/blob/develop/FAQ.md#what-is-webseed) in the magnet URI by instances that received this ActivityPub message.
14
15## Stats
16
17See the `/api/v1/server/stats` endpoint. For example:
18
19```
20{
21 ...
22 "videosRedundancy": [
23 {
24 "totalUsed": 0,
25 "totalVideos": 0,
26 "totalVideoFiles": 0,
27 "strategy": "most-views",
28 "totalSize": 104857600
29 },
30 {
31 "totalUsed": 0,
32 "totalVideos": 0,
33 "totalVideoFiles": 0,
34 "strategy": "trending",
35 "totalSize": 104857600
36 },
37 {
38 "totalUsed": 0,
39 "totalVideos": 0,
40 "totalVideoFiles": 0,
41 "strategy": "recently-added",
42 "totalSize": 104857600
43 }
44 ]
45}
46``` \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-18 01:08:14 +0000