diff options
author | Chocobozzz <me@florianbigard.com> | 2018-09-24 13:07:33 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-09-24 13:38:39 +0200 |
commit | e5565833f62b97f62ea75eba5b479963ae78b873 (patch) | |
tree | 835793ce464f9666b0ceae79f3d278cc4e007b32 | |
parent | d1a63fc7ac58a1db00d8ca4f43aadba02eb9b084 (diff) | |
download | PeerTube-e5565833f62b97f62ea75eba5b479963ae78b873.tar.gz PeerTube-e5565833f62b97f62ea75eba5b479963ae78b873.tar.zst PeerTube-e5565833f62b97f62ea75eba5b479963ae78b873.zip |
Improve redundancy: add 'min_lifetime' configuration
27 files changed, 627 insertions, 278 deletions
diff --git a/config/default.yaml b/config/default.yaml index fa1fb628a..0d7d948c2 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -75,14 +75,20 @@ redundancy: | |||
75 | strategies: | 75 | strategies: |
76 | # - | 76 | # - |
77 | # size: '10GB' | 77 | # size: '10GB' |
78 | # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) | ||
79 | # min_lifetime: '48 hours' | ||
78 | # strategy: 'most-views' # Cache videos that have the most views | 80 | # strategy: 'most-views' # Cache videos that have the most views |
79 | # - | 81 | # - |
80 | # size: '10GB' | 82 | # size: '10GB' |
83 | # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) | ||
84 | # min_lifetime: '48 hours' | ||
81 | # strategy: 'trending' # Cache trending videos | 85 | # strategy: 'trending' # Cache trending videos |
82 | # - | 86 | # - |
83 | # size: '10GB' | 87 | # size: '10GB' |
88 | # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) | ||
89 | # min_lifetime: '48 hours' | ||
84 | # strategy: 'recently-added' # Cache recently added videos | 90 | # strategy: 'recently-added' # Cache recently added videos |
85 | # minViews: 10 # Having at least x views | 91 | # min_views: 10 # Having at least x views |
86 | 92 | ||
87 | cache: | 93 | cache: |
88 | previews: | 94 | previews: |
diff --git a/config/production.yaml.example b/config/production.yaml.example index 4d8752206..f9da8e0dd 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example | |||
@@ -76,14 +76,20 @@ redundancy: | |||
76 | strategies: | 76 | strategies: |
77 | # - | 77 | # - |
78 | # size: '10GB' | 78 | # size: '10GB' |
79 | # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) | ||
80 | # min_lifetime: '48 hours' | ||
79 | # strategy: 'most-views' # Cache videos that have the most views | 81 | # strategy: 'most-views' # Cache videos that have the most views |
80 | # - | 82 | # - |
81 | # size: '10GB' | 83 | # size: '10GB' |
84 | # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) | ||
85 | # min_lifetime: '48 hours' | ||
82 | # strategy: 'trending' # Cache trending videos | 86 | # strategy: 'trending' # Cache trending videos |
83 | # - | 87 | # - |
84 | # size: '10GB' | 88 | # size: '10GB' |
89 | # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) | ||
90 | # min_lifetime: '48 hours' | ||
85 | # strategy: 'recently-added' # Cache recently added videos | 91 | # strategy: 'recently-added' # Cache recently added videos |
86 | # minViews: 10 # Having at least x views | 92 | # min_views: 10 # Having at least x views |
87 | 93 | ||
88 | ############################################################################### | 94 | ############################################################################### |
89 | # | 95 | # |
diff --git a/config/test.yaml b/config/test.yaml index ad94b00cd..04c999966 100644 --- a/config/test.yaml +++ b/config/test.yaml | |||
@@ -23,18 +23,21 @@ log: | |||
23 | 23 | ||
24 | redundancy: | 24 | redundancy: |
25 | videos: | 25 | videos: |
26 | check_interval: '5 seconds' | 26 | check_interval: '10 minutes' |
27 | strategies: | 27 | strategies: |
28 | - | 28 | - |
29 | size: '10MB' | 29 | size: '10MB' |
30 | min_lifetime: '10 minutes' | ||
30 | strategy: 'most-views' | 31 | strategy: 'most-views' |
31 | - | 32 | - |
32 | size: '10MB' | 33 | size: '10MB' |
34 | min_lifetime: '10 minutes' | ||
33 | strategy: 'trending' | 35 | strategy: 'trending' |
34 | - | 36 | - |
35 | size: '10MB' | 37 | size: '10MB' |
38 | min_lifetime: '10 minutes' | ||
36 | strategy: 'recently-added' | 39 | strategy: 'recently-added' |
37 | minViews: 1 | 40 | min_views: 1 |
38 | 41 | ||
39 | cache: | 42 | cache: |
40 | previews: | 43 | previews: |
@@ -1,6 +1,4 @@ | |||
1 | // FIXME: https://github.com/nodejs/node/pull/16853 | 1 | // FIXME: https://github.com/nodejs/node/pull/16853 |
2 | import { VideosCaptionCache } from './server/lib/cache/videos-caption-cache' | ||
3 | |||
4 | require('tls').DEFAULT_ECDH_CURVE = 'auto' | 2 | require('tls').DEFAULT_ECDH_CURVE = 'auto' |
5 | 3 | ||
6 | import { isTestInstance } from './server/helpers/core-utils' | 4 | import { isTestInstance } from './server/helpers/core-utils' |
@@ -17,7 +15,7 @@ import * as cors from 'cors' | |||
17 | import * as cookieParser from 'cookie-parser' | 15 | import * as cookieParser from 'cookie-parser' |
18 | import * as helmet from 'helmet' | 16 | import * as helmet from 'helmet' |
19 | import * as useragent from 'useragent' | 17 | import * as useragent from 'useragent' |
20 | import * as anonymise from 'ip-anonymize' | 18 | import * as anonymize from 'ip-anonymize' |
21 | 19 | ||
22 | process.title = 'peertube' | 20 | process.title = 'peertube' |
23 | 21 | ||
@@ -25,7 +23,7 @@ process.title = 'peertube' | |||
25 | const app = express() | 23 | const app = express() |
26 | 24 | ||
27 | // ----------- Core checker ----------- | 25 | // ----------- Core checker ----------- |
28 | import { checkMissedConfig, checkFFmpeg, checkConfig, checkActivityPubUrls } from './server/initializers/checker' | 26 | import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-before-init' |
29 | 27 | ||
30 | // Do not use barrels because we don't want to load all modules here (we need to initialize database first) | 28 | // Do not use barrels because we don't want to load all modules here (we need to initialize database first) |
31 | import { logger } from './server/helpers/logger' | 29 | import { logger } from './server/helpers/logger' |
@@ -43,6 +41,8 @@ checkFFmpeg(CONFIG) | |||
43 | process.exit(-1) | 41 | process.exit(-1) |
44 | }) | 42 | }) |
45 | 43 | ||
44 | import { checkConfig, checkActivityPubUrls } from './server/initializers/checker-after-init' | ||
45 | |||
46 | const errorMessage = checkConfig() | 46 | const errorMessage = checkConfig() |
47 | if (errorMessage !== null) { | 47 | if (errorMessage !== null) { |
48 | throw new Error(errorMessage) | 48 | throw new Error(errorMessage) |
@@ -76,7 +76,7 @@ migrate() | |||
76 | import { installApplication } from './server/initializers' | 76 | import { installApplication } from './server/initializers' |
77 | import { Emailer } from './server/lib/emailer' | 77 | import { Emailer } from './server/lib/emailer' |
78 | import { JobQueue } from './server/lib/job-queue' | 78 | import { JobQueue } from './server/lib/job-queue' |
79 | import { VideosPreviewCache } from './server/lib/cache' | 79 | import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache' |
80 | import { | 80 | import { |
81 | activityPubRouter, | 81 | activityPubRouter, |
82 | apiRouter, | 82 | apiRouter, |
@@ -111,7 +111,7 @@ if (isTestInstance()) { | |||
111 | // For the logger | 111 | // For the logger |
112 | morgan.token('remote-addr', req => { | 112 | morgan.token('remote-addr', req => { |
113 | return (req.get('DNT') === '1') ? | 113 | return (req.get('DNT') === '1') ? |
114 | anonymise(req.ip || (req.connection && req.connection.remoteAddress) || undefined, | 114 | anonymize(req.ip || (req.connection && req.connection.remoteAddress) || undefined, |
115 | 16, // bitmask for IPv4 | 115 | 16, // bitmask for IPv4 |
116 | 16 // bitmask for IPv6 | 116 | 16 // bitmask for IPv6 |
117 | ) : | 117 | ) : |
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 7c45f3632..22bc25476 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts | |||
@@ -4,7 +4,7 @@ import { VideoResolution } from '../../shared/models/videos' | |||
4 | import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers' | 4 | import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers' |
5 | import { processImage } from './image-utils' | 5 | import { processImage } from './image-utils' |
6 | import { logger } from './logger' | 6 | import { logger } from './logger' |
7 | import { checkFFmpegEncoders } from '../initializers/checker' | 7 | import { checkFFmpegEncoders } from '../initializers/checker-before-init' |
8 | import { remove } from 'fs-extra' | 8 | import { remove } from 'fs-extra' |
9 | 9 | ||
10 | function computeResolutionsToTranscode (videoFileHeight: number) { | 10 | function computeResolutionsToTranscode (videoFileHeight: number) { |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts new file mode 100644 index 000000000..588526235 --- /dev/null +++ b/server/initializers/checker-after-init.ts | |||
@@ -0,0 +1,115 @@ | |||
1 | import * as config from 'config' | ||
2 | import { promisify0, isProdInstance, parseDuration, isTestInstance } from '../helpers/core-utils' | ||
3 | import { UserModel } from '../models/account/user' | ||
4 | import { ApplicationModel } from '../models/application/application' | ||
5 | import { OAuthClientModel } from '../models/oauth/oauth-client' | ||
6 | import { parse } from 'url' | ||
7 | import { CONFIG } from './constants' | ||
8 | import { logger } from '../helpers/logger' | ||
9 | import { getServerActor } from '../helpers/utils' | ||
10 | import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy' | ||
11 | import { isArray } from '../helpers/custom-validators/misc' | ||
12 | import { uniq } from 'lodash' | ||
13 | |||
14 | async function checkActivityPubUrls () { | ||
15 | const actor = await getServerActor() | ||
16 | |||
17 | const parsed = parse(actor.url) | ||
18 | if (CONFIG.WEBSERVER.HOST !== parsed.host) { | ||
19 | const NODE_ENV = config.util.getEnv('NODE_ENV') | ||
20 | const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR') | ||
21 | |||
22 | logger.warn( | ||
23 | 'It seems PeerTube was started (and created some data) with another domain name. ' + | ||
24 | 'This means you will not be able to federate! ' + | ||
25 | 'Please use %s %s npm run update-host to fix this.', | ||
26 | NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '', | ||
27 | NODE_ENV ? `NODE_ENV=${NODE_ENV}` : '' | ||
28 | ) | ||
29 | } | ||
30 | } | ||
31 | |||
32 | // Some checks on configuration files | ||
33 | // Return an error message, or null if everything is okay | ||
34 | function checkConfig () { | ||
35 | const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY | ||
36 | |||
37 | // NSFW policy | ||
38 | { | ||
39 | const available = [ 'do_not_list', 'blur', 'display' ] | ||
40 | if (available.indexOf(defaultNSFWPolicy) === -1) { | ||
41 | return 'NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy | ||
42 | } | ||
43 | } | ||
44 | |||
45 | // Redundancies | ||
46 | const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES | ||
47 | if (isArray(redundancyVideos)) { | ||
48 | const available = [ 'most-views', 'trending', 'recently-added' ] | ||
49 | for (const r of redundancyVideos) { | ||
50 | if (available.indexOf(r.strategy) === -1) { | ||
51 | return 'Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy | ||
52 | } | ||
53 | |||
54 | // Lifetime should not be < 10 hours | ||
55 | if (!isTestInstance() && r.minLifetime < 1000 * 3600 * 10) { | ||
56 | return 'Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy | ||
57 | } | ||
58 | } | ||
59 | |||
60 | const filtered = uniq(redundancyVideos.map(r => r.strategy)) | ||
61 | if (filtered.length !== redundancyVideos.length) { | ||
62 | return 'Redundancy video entries should have unique strategies' | ||
63 | } | ||
64 | |||
65 | const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy | ||
66 | if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { | ||
67 | return 'Min views in recently added strategy is not a number' | ||
68 | } | ||
69 | } | ||
70 | |||
71 | if (isProdInstance()) { | ||
72 | const configStorage = config.get('storage') | ||
73 | for (const key of Object.keys(configStorage)) { | ||
74 | if (configStorage[key].startsWith('storage/')) { | ||
75 | logger.warn( | ||
76 | 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.', | ||
77 | key | ||
78 | ) | ||
79 | } | ||
80 | } | ||
81 | } | ||
82 | |||
83 | return null | ||
84 | } | ||
85 | |||
86 | // We get db by param to not import it in this file (import orders) | ||
87 | async function clientsExist () { | ||
88 | const totalClients = await OAuthClientModel.countTotal() | ||
89 | |||
90 | return totalClients !== 0 | ||
91 | } | ||
92 | |||
93 | // We get db by param to not import it in this file (import orders) | ||
94 | async function usersExist () { | ||
95 | const totalUsers = await UserModel.countTotal() | ||
96 | |||
97 | return totalUsers !== 0 | ||
98 | } | ||
99 | |||
100 | // We get db by param to not import it in this file (import orders) | ||
101 | async function applicationExist () { | ||
102 | const totalApplication = await ApplicationModel.countTotal() | ||
103 | |||
104 | return totalApplication !== 0 | ||
105 | } | ||
106 | |||
107 | // --------------------------------------------------------------------------- | ||
108 | |||
109 | export { | ||
110 | checkConfig, | ||
111 | clientsExist, | ||
112 | usersExist, | ||
113 | applicationExist, | ||
114 | checkActivityPubUrls | ||
115 | } | ||
diff --git a/server/initializers/checker.ts b/server/initializers/checker-before-init.ts index 5b068caa1..4f46d406a 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -1,78 +1,8 @@ | |||
1 | import * as config from 'config' | 1 | import * as config from 'config' |
2 | import { promisify0, isProdInstance } from '../helpers/core-utils' | 2 | import { promisify0 } from '../helpers/core-utils' |
3 | import { UserModel } from '../models/account/user' | ||
4 | import { ApplicationModel } from '../models/application/application' | ||
5 | import { OAuthClientModel } from '../models/oauth/oauth-client' | ||
6 | import { parse } from 'url' | ||
7 | import { CONFIG } from './constants' | ||
8 | import { logger } from '../helpers/logger' | ||
9 | import { getServerActor } from '../helpers/utils' | ||
10 | import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy' | ||
11 | import { isArray } from '../helpers/custom-validators/misc' | 3 | import { isArray } from '../helpers/custom-validators/misc' |
12 | import { uniq } from 'lodash' | ||
13 | |||
14 | async function checkActivityPubUrls () { | ||
15 | const actor = await getServerActor() | ||
16 | |||
17 | const parsed = parse(actor.url) | ||
18 | if (CONFIG.WEBSERVER.HOST !== parsed.host) { | ||
19 | const NODE_ENV = config.util.getEnv('NODE_ENV') | ||
20 | const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR') | ||
21 | |||
22 | logger.warn( | ||
23 | 'It seems PeerTube was started (and created some data) with another domain name. ' + | ||
24 | 'This means you will not be able to federate! ' + | ||
25 | 'Please use %s %s npm run update-host to fix this.', | ||
26 | NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '', | ||
27 | NODE_ENV ? `NODE_ENV=${NODE_ENV}` : '' | ||
28 | ) | ||
29 | } | ||
30 | } | ||
31 | 4 | ||
32 | // Some checks on configuration files | 5 | // ONLY USE CORE MODULES IN THIS FILE! |
33 | // Return an error message, or null if everything is okay | ||
34 | function checkConfig () { | ||
35 | const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy') | ||
36 | |||
37 | // NSFW policy | ||
38 | if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { | ||
39 | return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy | ||
40 | } | ||
41 | |||
42 | // Redundancies | ||
43 | const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies') | ||
44 | if (isArray(redundancyVideos)) { | ||
45 | for (const r of redundancyVideos) { | ||
46 | if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) { | ||
47 | return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy | ||
48 | } | ||
49 | } | ||
50 | |||
51 | const filtered = uniq(redundancyVideos.map(r => r.strategy)) | ||
52 | if (filtered.length !== redundancyVideos.length) { | ||
53 | return 'Redundancy video entries should have unique strategies' | ||
54 | } | ||
55 | |||
56 | const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy | ||
57 | if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { | ||
58 | return 'Min views in recently added strategy is not a number' | ||
59 | } | ||
60 | } | ||
61 | |||
62 | if (isProdInstance()) { | ||
63 | const configStorage = config.get('storage') | ||
64 | for (const key of Object.keys(configStorage)) { | ||
65 | if (configStorage[key].startsWith('storage/')) { | ||
66 | logger.warn( | ||
67 | 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.', | ||
68 | key | ||
69 | ) | ||
70 | } | ||
71 | } | ||
72 | } | ||
73 | |||
74 | return null | ||
75 | } | ||
76 | 6 | ||
77 | // Check the config files | 7 | // Check the config files |
78 | function checkMissedConfig () { | 8 | function checkMissedConfig () { |
@@ -109,6 +39,14 @@ function checkMissedConfig () { | |||
109 | } | 39 | } |
110 | } | 40 | } |
111 | 41 | ||
42 | const redundancyVideos = config.get<any>('redundancy.videos.strategies') | ||
43 | if (isArray(redundancyVideos)) { | ||
44 | for (const r of redundancyVideos) { | ||
45 | if (!r.size) miss.push('redundancy.videos.strategies.size') | ||
46 | if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime') | ||
47 | } | ||
48 | } | ||
49 | |||
112 | const missingAlternatives = requiredAlternatives.filter( | 50 | const missingAlternatives = requiredAlternatives.filter( |
113 | set => !set.find(alternative => !alternative.find(key => !config.has(key))) | 51 | set => !set.find(alternative => !alternative.find(key => !config.has(key))) |
114 | ) | 52 | ) |
@@ -163,36 +101,10 @@ async function checkFFmpegEncoders (): Promise<Map<string, boolean>> { | |||
163 | } | 101 | } |
164 | } | 102 | } |
165 | 103 | ||
166 | // We get db by param to not import it in this file (import orders) | ||
167 | async function clientsExist () { | ||
168 | const totalClients = await OAuthClientModel.countTotal() | ||
169 | |||
170 | return totalClients !== 0 | ||
171 | } | ||
172 | |||
173 | // We get db by param to not import it in this file (import orders) | ||
174 | async function usersExist () { | ||
175 | const totalUsers = await UserModel.countTotal() | ||
176 | |||
177 | return totalUsers !== 0 | ||
178 | } | ||
179 | |||
180 | // We get db by param to not import it in this file (import orders) | ||
181 | async function applicationExist () { | ||
182 | const totalApplication = await ApplicationModel.countTotal() | ||
183 | |||
184 | return totalApplication !== 0 | ||
185 | } | ||
186 | |||
187 | // --------------------------------------------------------------------------- | 104 | // --------------------------------------------------------------------------- |
188 | 105 | ||
189 | export { | 106 | export { |
190 | checkConfig, | ||
191 | checkFFmpeg, | 107 | checkFFmpeg, |
192 | checkFFmpegEncoders, | 108 | checkFFmpegEncoders, |
193 | checkMissedConfig, | 109 | checkMissedConfig |
194 | clientsExist, | ||
195 | usersExist, | ||
196 | applicationExist, | ||
197 | checkActivityPubUrls | ||
198 | } | 110 | } |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 03424ffb8..947cbda28 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -601,7 +601,6 @@ const MEMOIZE_TTL = { | |||
601 | 601 | ||
602 | const REDUNDANCY = { | 602 | const REDUNDANCY = { |
603 | VIDEOS: { | 603 | VIDEOS: { |
604 | EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days | ||
605 | RANDOMIZED_FACTOR: 5 | 604 | RANDOMIZED_FACTOR: 5 |
606 | } | 605 | } |
607 | } | 606 | } |
@@ -750,10 +749,16 @@ function updateWebserverConfig () { | |||
750 | CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) | 749 | CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) |
751 | } | 750 | } |
752 | 751 | ||
753 | function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] { | 752 | function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { |
754 | if (!objs) return [] | 753 | if (!objs) return [] |
755 | 754 | ||
756 | return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) })) | 755 | return objs.map(obj => { |
756 | return Object.assign(obj, { | ||
757 | minLifetime: parseDuration(obj.min_lifetime), | ||
758 | size: bytes.parse(obj.size), | ||
759 | minViews: obj.min_views | ||
760 | }) | ||
761 | }) | ||
757 | } | 762 | } |
758 | 763 | ||
759 | function buildLanguages () { | 764 | function buildLanguages () { |
diff --git a/server/initializers/index.ts b/server/initializers/index.ts index 332702774..fe9190a9c 100644 --- a/server/initializers/index.ts +++ b/server/initializers/index.ts | |||
@@ -1,6 +1,5 @@ | |||
1 | // Constants first, database in second! | 1 | // Constants first, database in second! |
2 | export * from './constants' | 2 | export * from './constants' |
3 | export * from './database' | 3 | export * from './database' |
4 | export * from './checker' | ||
5 | export * from './installer' | 4 | export * from './installer' |
6 | export * from './migrator' | 5 | export * from './migrator' |
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 818bb04a2..c952ad46c 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts | |||
@@ -5,7 +5,7 @@ import { createApplicationActor, createUserAccountAndChannel } from '../lib/user | |||
5 | import { UserModel } from '../models/account/user' | 5 | import { UserModel } from '../models/account/user' |
6 | import { ApplicationModel } from '../models/application/application' | 6 | import { ApplicationModel } from '../models/application/application' |
7 | import { OAuthClientModel } from '../models/oauth/oauth-client' | 7 | import { OAuthClientModel } from '../models/oauth/oauth-client' |
8 | import { applicationExist, clientsExist, usersExist } from './checker' | 8 | import { applicationExist, clientsExist, usersExist } from './checker-after-init' |
9 | import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' | 9 | import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' |
10 | import { sequelizeTypescript } from './database' | 10 | import { sequelizeTypescript } from './database' |
11 | import { remove, ensureDir } from 'fs-extra' | 11 | import { remove, ensureDir } from 'fs-extra' |
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts index a86428461..10277eca7 100644 --- a/server/lib/activitypub/audience.ts +++ b/server/lib/activitypub/audience.ts | |||
@@ -50,7 +50,12 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): Acti | |||
50 | 50 | ||
51 | async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) { | 51 | async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) { |
52 | const actors = await VideoShareModel.loadActorsByShare(video.id, t) | 52 | const actors = await VideoShareModel.loadActorsByShare(video.id, t) |
53 | actors.push(video.VideoChannel.Account.Actor) | 53 | |
54 | const videoActor = video.VideoChannel && video.VideoChannel.Account | ||
55 | ? video.VideoChannel.Account.Actor | ||
56 | : await ActorModel.loadAccountActorByVideoId(video.id, t) | ||
57 | |||
58 | actors.push(videoActor) | ||
54 | 59 | ||
55 | return actors | 60 | return actors |
56 | } | 61 | } |
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 87f8a4162..5286d8e6d 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | 1 | import { CacheFileObject } from '../../../shared/index' |
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { sequelizeTypescript } from '../../initializers' | ||
4 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 3 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
4 | import { Transaction } from 'sequelize' | ||
5 | 5 | ||
6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
7 | const url = cacheFileObject.url | 7 | const url = cacheFileObject.url |
@@ -22,25 +22,29 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
22 | } | 22 | } |
23 | } | 23 | } |
24 | 24 | ||
25 | function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { | 25 | function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }, t: Transaction) { |
26 | return sequelizeTypescript.transaction(async t => { | 26 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) |
27 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | ||
28 | 27 | ||
29 | return VideoRedundancyModel.create(attributes, { transaction: t }) | 28 | return VideoRedundancyModel.create(attributes, { transaction: t }) |
30 | }) | ||
31 | } | 29 | } |
32 | 30 | ||
33 | function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) { | 31 | function updateCacheFile ( |
32 | cacheFileObject: CacheFileObject, | ||
33 | redundancyModel: VideoRedundancyModel, | ||
34 | video: VideoModel, | ||
35 | byActor: { id?: number }, | ||
36 | t: Transaction | ||
37 | ) { | ||
34 | if (redundancyModel.actorId !== byActor.id) { | 38 | if (redundancyModel.actorId !== byActor.id) { |
35 | throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') | 39 | throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') |
36 | } | 40 | } |
37 | 41 | ||
38 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) | 42 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) |
39 | 43 | ||
40 | redundancyModel.set('expires', attributes.expiresOn) | 44 | redundancyModel.set('expires', attributes.expiresOn) |
41 | redundancyModel.set('fileUrl', attributes.fileUrl) | 45 | redundancyModel.set('fileUrl', attributes.fileUrl) |
42 | 46 | ||
43 | return redundancyModel.save() | 47 | return redundancyModel.save({ transaction: t }) |
44 | } | 48 | } |
45 | 49 | ||
46 | export { | 50 | export { |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index cff8dcfc6..ceb5413ca 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -95,7 +95,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) | |||
95 | if (video.isOwned()) { | 95 | if (video.isOwned()) { |
96 | // Don't resend the activity to the sender | 96 | // Don't resend the activity to the sender |
97 | const exceptions = [ byActor ] | 97 | const exceptions = [ byActor ] |
98 | await forwardActivity(activity, undefined, exceptions) | 98 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) |
99 | } | 99 | } |
100 | } | 100 | } |
101 | 101 | ||
@@ -104,12 +104,14 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) | |||
104 | 104 | ||
105 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) | 105 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) |
106 | 106 | ||
107 | await createCacheFile(cacheFile, video, byActor) | 107 | await sequelizeTypescript.transaction(async t => { |
108 | return createCacheFile(cacheFile, video, byActor, t) | ||
109 | }) | ||
108 | 110 | ||
109 | if (video.isOwned()) { | 111 | if (video.isOwned()) { |
110 | // Don't resend the activity to the sender | 112 | // Don't resend the activity to the sender |
111 | const exceptions = [ byActor ] | 113 | const exceptions = [ byActor ] |
112 | await forwardActivity(activity, undefined, exceptions) | 114 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) |
113 | } | 115 | } |
114 | } | 116 | } |
115 | 117 | ||
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 73ca0a17c..ff019cd8c 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -100,7 +100,7 @@ async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo | |||
100 | 100 | ||
101 | return sequelizeTypescript.transaction(async t => { | 101 | return sequelizeTypescript.transaction(async t => { |
102 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 102 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) |
103 | if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) | 103 | if (!cacheFile) throw new Error('Unknown video cache ' + cacheFileObject.id) |
104 | 104 | ||
105 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') | 105 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') |
106 | 106 | ||
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index ed3489ebf..e092a6729 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -12,6 +12,7 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-vali | |||
12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | 12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' |
13 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 13 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
14 | import { createCacheFile, updateCacheFile } from '../cache-file' | 14 | import { createCacheFile, updateCacheFile } from '../cache-file' |
15 | import { forwardVideoRelatedActivity } from '../send/utils' | ||
15 | 16 | ||
16 | async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { | 17 | async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { |
17 | const objectType = activity.object.type | 18 | const objectType = activity.object.type |
@@ -68,18 +69,29 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
68 | async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { | 69 | async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { |
69 | const cacheFileObject = activity.object as CacheFileObject | 70 | const cacheFileObject = activity.object as CacheFileObject |
70 | 71 | ||
71 | if (!isCacheFileObjectValid(cacheFileObject) === false) { | 72 | if (!isCacheFileObjectValid(cacheFileObject)) { |
72 | logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject }) | 73 | logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) |
73 | return undefined | 74 | return undefined |
74 | } | 75 | } |
75 | 76 | ||
76 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 77 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) |
77 | if (!redundancyModel) { | 78 | |
78 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) | 79 | await sequelizeTypescript.transaction(async t => { |
79 | return createCacheFile(cacheFileObject, video, byActor) | 80 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) |
80 | } | 81 | |
82 | if (!redundancyModel) { | ||
83 | await createCacheFile(cacheFileObject, video, byActor, t) | ||
84 | } else { | ||
85 | await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) | ||
86 | } | ||
87 | }) | ||
88 | |||
89 | if (video.isOwned()) { | ||
90 | // Don't resend the activity to the sender | ||
91 | const exceptions = [ byActor ] | ||
81 | 92 | ||
82 | return updateCacheFile(cacheFileObject, redundancyModel, byActor) | 93 | await forwardVideoRelatedActivity(activity, undefined, exceptions, video) |
94 | } | ||
83 | } | 95 | } |
84 | 96 | ||
85 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { | 97 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index ec46789b7..a68f03edf 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video' | |||
7 | import { VideoChannelModel } from '../../../models/video/video-channel' | 7 | import { VideoChannelModel } from '../../../models/video/video-channel' |
8 | import { VideoShareModel } from '../../../models/video/video-share' | 8 | import { VideoShareModel } from '../../../models/video/video-share' |
9 | import { getUpdateActivityPubUrl } from '../url' | 9 | import { getUpdateActivityPubUrl } from '../url' |
10 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' | 10 | import { broadcastToFollowers, sendVideoRelatedActivity } from './utils' |
11 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' | 11 | import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience' |
12 | import { logger } from '../../../helpers/logger' | 12 | import { logger } from '../../../helpers/logger' |
13 | import { VideoCaptionModel } from '../../../models/video/video-caption' | 13 | import { VideoCaptionModel } from '../../../models/video/video-caption' |
14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 48c0e0a5c..db72ef23c 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -176,7 +176,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { | |||
176 | syncParam, | 176 | syncParam, |
177 | refreshViews | 177 | refreshViews |
178 | } | 178 | } |
179 | const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) | 179 | const p = refreshVideoIfNeeded(refreshOptions) |
180 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | 180 | if (syncParam.refreshVideo === true) videoFromDatabase = await p |
181 | 181 | ||
182 | return { video: videoFromDatabase } | 182 | return { video: videoFromDatabase } |
@@ -245,29 +245,37 @@ async function updateVideoFromAP (options: { | |||
245 | generateThumbnailFromUrl(options.video, options.videoObject.icon) | 245 | generateThumbnailFromUrl(options.video, options.videoObject.icon) |
246 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) | 246 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) |
247 | 247 | ||
248 | // Remove old video files | 248 | { |
249 | const videoFileDestroyTasks: Bluebird<void>[] = [] | 249 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) |
250 | for (const videoFile of options.video.VideoFiles) { | 250 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) |
251 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) | ||
252 | } | ||
253 | await Promise.all(videoFileDestroyTasks) | ||
254 | 251 | ||
255 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) | 252 | // Remove video files that do not exist anymore |
256 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) | 253 | const destroyTasks = options.video.VideoFiles |
257 | await Promise.all(tasks) | 254 | .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f))) |
255 | .map(f => f.destroy(sequelizeOptions)) | ||
256 | await Promise.all(destroyTasks) | ||
258 | 257 | ||
259 | // Update Tags | 258 | // Update or add other one |
260 | const tags = options.videoObject.tag.map(tag => tag.name) | 259 | const upsertTasks = videoFileAttributes.map(a => VideoFileModel.upsert(a, sequelizeOptions)) |
261 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 260 | await Promise.all(upsertTasks) |
262 | await options.video.$set('Tags', tagInstances, sequelizeOptions) | 261 | } |
263 | 262 | ||
264 | // Update captions | 263 | { |
265 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) | 264 | // Update Tags |
265 | const tags = options.videoObject.tag.map(tag => tag.name) | ||
266 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
267 | await options.video.$set('Tags', tagInstances, sequelizeOptions) | ||
268 | } | ||
266 | 269 | ||
267 | const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { | 270 | { |
268 | return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) | 271 | // Update captions |
269 | }) | 272 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) |
270 | await Promise.all(videoCaptionsPromises) | 273 | |
274 | const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { | ||
275 | return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) | ||
276 | }) | ||
277 | await Promise.all(videoCaptionsPromises) | ||
278 | } | ||
271 | }) | 279 | }) |
272 | 280 | ||
273 | logger.info('Remote video with uuid %s updated', options.videoObject.uuid) | 281 | logger.info('Remote video with uuid %s updated', options.videoObject.uuid) |
@@ -382,7 +390,7 @@ async function refreshVideoIfNeeded (options: { | |||
382 | channel: channelActor.VideoChannel, | 390 | channel: channelActor.VideoChannel, |
383 | updateViews: options.refreshViews | 391 | updateViews: options.refreshViews |
384 | } | 392 | } |
385 | await updateVideoFromAP(updateOptions) | 393 | await retryTransactionWrapper(updateVideoFromAP, updateOptions) |
386 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | 394 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) |
387 | } catch (err) { | 395 | } catch (err) { |
388 | logger.warn('Cannot refresh video.', { err }) | 396 | logger.warn('Cannot refresh video.', { err }) |
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts index 7bf63790a..54eb983fa 100644 --- a/server/lib/cache/index.ts +++ b/server/lib/cache/index.ts | |||
@@ -1 +1,2 @@ | |||
1 | export * from './videos-preview-cache' | 1 | export * from './videos-preview-cache' |
2 | export * from './videos-caption-cache' | ||
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 78221cc3d..16b122658 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts | |||
@@ -6,7 +6,8 @@ import { getServerActor } from '../helpers/utils' | |||
6 | async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) { | 6 | async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) { |
7 | const serverActor = await getServerActor() | 7 | const serverActor = await getServerActor() |
8 | 8 | ||
9 | await sendUndoCacheFile(serverActor, videoRedundancy, t) | 9 | // Local cache, send undo to remote instances |
10 | if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t) | ||
10 | 11 | ||
11 | await videoRedundancy.destroy({ transaction: t }) | 12 | await videoRedundancy.destroy({ transaction: t }) |
12 | } | 13 | } |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 998d2295a..97df3e4f5 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' | 2 | import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | import { VideoFileModel } from '../../models/video/video-file' | 6 | import { VideoFileModel } from '../../models/video/video-file' |
7 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | 7 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' |
@@ -12,6 +12,7 @@ import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | |||
12 | import { VideoModel } from '../../models/video/video' | 12 | import { VideoModel } from '../../models/video/video' |
13 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 13 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' |
14 | import { isTestInstance } from '../../helpers/core-utils' | 14 | import { isTestInstance } from '../../helpers/core-utils' |
15 | import { removeVideoRedundancy } from '../redundancy' | ||
15 | 16 | ||
16 | export class VideosRedundancyScheduler extends AbstractScheduler { | 17 | export class VideosRedundancyScheduler extends AbstractScheduler { |
17 | 18 | ||
@@ -30,7 +31,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
30 | this.executing = true | 31 | this.executing = true |
31 | 32 | ||
32 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 33 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
33 | if (!isTestInstance()) logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) | 34 | logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) |
34 | 35 | ||
35 | try { | 36 | try { |
36 | const videoToDuplicate = await this.findVideoToDuplicate(obj) | 37 | const videoToDuplicate = await this.findVideoToDuplicate(obj) |
@@ -39,20 +40,24 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
39 | const videoFiles = videoToDuplicate.VideoFiles | 40 | const videoFiles = videoToDuplicate.VideoFiles |
40 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 41 | videoFiles.forEach(f => f.Video = videoToDuplicate) |
41 | 42 | ||
42 | if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { | 43 | await this.purgeCacheIfNeeded(obj, videoFiles) |
43 | if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 44 | |
45 | if (await this.isTooHeavy(obj, videoFiles)) { | ||
46 | logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | ||
44 | continue | 47 | continue |
45 | } | 48 | } |
46 | 49 | ||
47 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) | 50 | logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) |
48 | 51 | ||
49 | await this.createVideoRedundancy(obj.strategy, videoFiles) | 52 | await this.createVideoRedundancy(obj, videoFiles) |
50 | } catch (err) { | 53 | } catch (err) { |
51 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) | 54 | logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) |
52 | } | 55 | } |
53 | } | 56 | } |
54 | 57 | ||
55 | await this.removeExpired() | 58 | await this.extendsLocalExpiration() |
59 | |||
60 | await this.purgeRemoteExpired() | ||
56 | 61 | ||
57 | this.executing = false | 62 | this.executing = false |
58 | } | 63 | } |
@@ -61,16 +66,27 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
61 | return this.instance || (this.instance = new this()) | 66 | return this.instance || (this.instance = new this()) |
62 | } | 67 | } |
63 | 68 | ||
64 | private async removeExpired () { | 69 | private async extendsLocalExpiration () { |
65 | const expired = await VideoRedundancyModel.listAllExpired() | 70 | const expired = await VideoRedundancyModel.listLocalExpired() |
71 | |||
72 | for (const redundancyModel of expired) { | ||
73 | try { | ||
74 | const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | ||
75 | await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) | ||
76 | } catch (err) { | ||
77 | logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) | ||
78 | } | ||
79 | } | ||
80 | } | ||
66 | 81 | ||
67 | for (const m of expired) { | 82 | private async purgeRemoteExpired () { |
68 | logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m)) | 83 | const expired = await VideoRedundancyModel.listRemoteExpired() |
69 | 84 | ||
85 | for (const redundancyModel of expired) { | ||
70 | try { | 86 | try { |
71 | await m.destroy() | 87 | await removeVideoRedundancy(redundancyModel) |
72 | } catch (err) { | 88 | } catch (err) { |
73 | logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) | 89 | logger.error('Cannot remove redundancy %s from our redundancy system.', this.buildEntryLogId(redundancyModel)) |
74 | } | 90 | } |
75 | } | 91 | } |
76 | } | 92 | } |
@@ -90,18 +106,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
90 | } | 106 | } |
91 | } | 107 | } |
92 | 108 | ||
93 | private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { | 109 | private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { |
94 | const serverActor = await getServerActor() | 110 | const serverActor = await getServerActor() |
95 | 111 | ||
96 | for (const file of filesToDuplicate) { | 112 | for (const file of filesToDuplicate) { |
97 | const existing = await VideoRedundancyModel.loadByFileId(file.id) | 113 | const existing = await VideoRedundancyModel.loadByFileId(file.id) |
98 | if (existing) { | 114 | if (existing) { |
99 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy) | 115 | await this.extendsExpirationOf(existing, redundancy.minLifetime) |
100 | 116 | ||
101 | existing.expiresOn = this.buildNewExpiration() | ||
102 | await existing.save() | ||
103 | |||
104 | await sendUpdateCacheFile(serverActor, existing) | ||
105 | continue | 117 | continue |
106 | } | 118 | } |
107 | 119 | ||
@@ -109,7 +121,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
109 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) | 121 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) |
110 | if (!video) continue | 122 | if (!video) continue |
111 | 123 | ||
112 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) | 124 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
113 | 125 | ||
114 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 126 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
115 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) | 127 | const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) |
@@ -120,10 +132,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
120 | await rename(tmpPath, destPath) | 132 | await rename(tmpPath, destPath) |
121 | 133 | ||
122 | const createdModel = await VideoRedundancyModel.create({ | 134 | const createdModel = await VideoRedundancyModel.create({ |
123 | expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS), | 135 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), |
124 | url: getVideoCacheFileActivityPubUrl(file), | 136 | url: getVideoCacheFileActivityPubUrl(file), |
125 | fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), | 137 | fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), |
126 | strategy, | 138 | strategy: redundancy.strategy, |
127 | videoFileId: file.id, | 139 | videoFileId: file.id, |
128 | actorId: serverActor.id | 140 | actorId: serverActor.id |
129 | }) | 141 | }) |
@@ -133,16 +145,36 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
133 | } | 145 | } |
134 | } | 146 | } |
135 | 147 | ||
136 | private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { | 148 | private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { |
137 | const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) | 149 | logger.info('Extending expiration of %s.', redundancy.url) |
150 | |||
151 | const serverActor = await getServerActor() | ||
152 | |||
153 | redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs) | ||
154 | await redundancy.save() | ||
155 | |||
156 | await sendUpdateCacheFile(serverActor, redundancy) | ||
157 | } | ||
158 | |||
159 | private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | ||
160 | while (this.isTooHeavy(redundancy, filesToDuplicate)) { | ||
161 | const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) | ||
162 | if (!toDelete) return | ||
163 | |||
164 | await removeVideoRedundancy(toDelete) | ||
165 | } | ||
166 | } | ||
167 | |||
168 | private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { | ||
169 | const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate) | ||
138 | 170 | ||
139 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) | 171 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) |
140 | 172 | ||
141 | return totalDuplicated > maxSize | 173 | return totalDuplicated > maxSize |
142 | } | 174 | } |
143 | 175 | ||
144 | private buildNewExpiration () { | 176 | private buildNewExpiration (expiresAfterMs: number) { |
145 | return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS) | 177 | return new Date(Date.now() + expiresAfterMs) |
146 | } | 178 | } |
147 | 179 | ||
148 | private buildEntryLogId (object: VideoRedundancyModel) { | 180 | private buildEntryLogId (object: VideoRedundancyModel) { |
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index f8bb59323..12b83916e 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts | |||
@@ -37,6 +37,7 @@ import { ServerModel } from '../server/server' | |||
37 | import { throwIfNotValid } from '../utils' | 37 | import { throwIfNotValid } from '../utils' |
38 | import { VideoChannelModel } from '../video/video-channel' | 38 | import { VideoChannelModel } from '../video/video-channel' |
39 | import { ActorFollowModel } from './actor-follow' | 39 | import { ActorFollowModel } from './actor-follow' |
40 | import { VideoModel } from '../video/video' | ||
40 | 41 | ||
41 | enum ScopeNames { | 42 | enum ScopeNames { |
42 | FULL = 'FULL' | 43 | FULL = 'FULL' |
@@ -266,6 +267,36 @@ export class ActorModel extends Model<ActorModel> { | |||
266 | return ActorModel.unscoped().findById(id) | 267 | return ActorModel.unscoped().findById(id) |
267 | } | 268 | } |
268 | 269 | ||
270 | static loadAccountActorByVideoId (videoId: number, transaction: Sequelize.Transaction) { | ||
271 | const query = { | ||
272 | include: [ | ||
273 | { | ||
274 | attributes: [ 'id' ], | ||
275 | model: AccountModel.unscoped(), | ||
276 | required: true, | ||
277 | include: [ | ||
278 | { | ||
279 | attributes: [ 'id' ], | ||
280 | model: VideoChannelModel.unscoped(), | ||
281 | required: true, | ||
282 | include: { | ||
283 | attributes: [ 'id' ], | ||
284 | model: VideoModel.unscoped(), | ||
285 | required: true, | ||
286 | where: { | ||
287 | id: videoId | ||
288 | } | ||
289 | } | ||
290 | } | ||
291 | ] | ||
292 | } | ||
293 | ], | ||
294 | transaction | ||
295 | } | ||
296 | |||
297 | return ActorModel.unscoped().findOne(query as any) // FIXME: typings | ||
298 | } | ||
299 | |||
269 | static isActorUrlExist (url: string) { | 300 | static isActorUrlExist (url: string) { |
270 | const query = { | 301 | const query = { |
271 | raw: true, | 302 | raw: true, |
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index fb07287a8..970d2fe06 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts | |||
@@ -9,7 +9,6 @@ import { | |||
9 | Is, | 9 | Is, |
10 | Model, | 10 | Model, |
11 | Scopes, | 11 | Scopes, |
12 | Sequelize, | ||
13 | Table, | 12 | Table, |
14 | UpdatedAt | 13 | UpdatedAt |
15 | } from 'sequelize-typescript' | 14 | } from 'sequelize-typescript' |
@@ -28,6 +27,7 @@ import { ServerModel } from '../server/server' | |||
28 | import { sample } from 'lodash' | 27 | import { sample } from 'lodash' |
29 | import { isTestInstance } from '../../helpers/core-utils' | 28 | import { isTestInstance } from '../../helpers/core-utils' |
30 | import * as Bluebird from 'bluebird' | 29 | import * as Bluebird from 'bluebird' |
30 | import * as Sequelize from 'sequelize' | ||
31 | 31 | ||
32 | export enum ScopeNames { | 32 | export enum ScopeNames { |
33 | WITH_VIDEO = 'WITH_VIDEO' | 33 | WITH_VIDEO = 'WITH_VIDEO' |
@@ -116,11 +116,11 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
116 | Actor: ActorModel | 116 | Actor: ActorModel |
117 | 117 | ||
118 | @AfterDestroy | 118 | @AfterDestroy |
119 | static removeFilesAndSendDelete (instance: VideoRedundancyModel) { | 119 | static removeFile (instance: VideoRedundancyModel) { |
120 | // Not us | 120 | // Not us |
121 | if (!instance.strategy) return | 121 | if (!instance.strategy) return |
122 | 122 | ||
123 | logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) | 123 | logger.info('Removing duplicated video file %s-%s.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) |
124 | 124 | ||
125 | return instance.VideoFile.Video.removeFile(instance.VideoFile) | 125 | return instance.VideoFile.Video.removeFile(instance.VideoFile) |
126 | } | 126 | } |
@@ -135,11 +135,12 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
135 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | 135 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) |
136 | } | 136 | } |
137 | 137 | ||
138 | static loadByUrl (url: string) { | 138 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { |
139 | const query = { | 139 | const query = { |
140 | where: { | 140 | where: { |
141 | url | 141 | url |
142 | } | 142 | }, |
143 | transaction | ||
143 | } | 144 | } |
144 | 145 | ||
145 | return VideoRedundancyModel.findOne(query) | 146 | return VideoRedundancyModel.findOne(query) |
@@ -157,7 +158,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
157 | // On VideoModel! | 158 | // On VideoModel! |
158 | const query = { | 159 | const query = { |
159 | attributes: [ 'id', 'views' ], | 160 | attributes: [ 'id', 'views' ], |
160 | logging: !isTestInstance(), | ||
161 | limit: randomizedFactor, | 161 | limit: randomizedFactor, |
162 | order: getVideoSort('-views'), | 162 | order: getVideoSort('-views'), |
163 | include: [ | 163 | include: [ |
@@ -174,7 +174,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
174 | const query = { | 174 | const query = { |
175 | attributes: [ 'id', 'views' ], | 175 | attributes: [ 'id', 'views' ], |
176 | subQuery: false, | 176 | subQuery: false, |
177 | logging: !isTestInstance(), | ||
178 | group: 'VideoModel.id', | 177 | group: 'VideoModel.id', |
179 | limit: randomizedFactor, | 178 | limit: randomizedFactor, |
180 | order: getVideoSort('-trending'), | 179 | order: getVideoSort('-trending'), |
@@ -193,7 +192,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
193 | // On VideoModel! | 192 | // On VideoModel! |
194 | const query = { | 193 | const query = { |
195 | attributes: [ 'id', 'publishedAt' ], | 194 | attributes: [ 'id', 'publishedAt' ], |
196 | logging: !isTestInstance(), | ||
197 | limit: randomizedFactor, | 195 | limit: randomizedFactor, |
198 | order: getVideoSort('-publishedAt'), | 196 | order: getVideoSort('-publishedAt'), |
199 | where: { | 197 | where: { |
@@ -210,11 +208,29 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
210 | return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) | 208 | return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) |
211 | } | 209 | } |
212 | 210 | ||
211 | static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) { | ||
212 | const expiredDate = new Date() | ||
213 | expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) | ||
214 | |||
215 | const actor = await getServerActor() | ||
216 | |||
217 | const query = { | ||
218 | where: { | ||
219 | actorId: actor.id, | ||
220 | strategy, | ||
221 | createdAt: { | ||
222 | [ Sequelize.Op.lt ]: expiredDate | ||
223 | } | ||
224 | } | ||
225 | } | ||
226 | |||
227 | return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) | ||
228 | } | ||
229 | |||
213 | static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { | 230 | static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { |
214 | const actor = await getServerActor() | 231 | const actor = await getServerActor() |
215 | 232 | ||
216 | const options = { | 233 | const options = { |
217 | logging: !isTestInstance(), | ||
218 | include: [ | 234 | include: [ |
219 | { | 235 | { |
220 | attributes: [], | 236 | attributes: [], |
@@ -228,21 +244,39 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
228 | ] | 244 | ] |
229 | } | 245 | } |
230 | 246 | ||
231 | return VideoFileModel.sum('size', options) | 247 | return VideoFileModel.sum('size', options as any) // FIXME: typings |
232 | } | 248 | } |
233 | 249 | ||
234 | static listAllExpired () { | 250 | static async listLocalExpired () { |
251 | const actor = await getServerActor() | ||
252 | |||
253 | const query = { | ||
254 | where: { | ||
255 | actorId: actor.id, | ||
256 | expiresOn: { | ||
257 | [ Sequelize.Op.lt ]: new Date() | ||
258 | } | ||
259 | } | ||
260 | } | ||
261 | |||
262 | return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) | ||
263 | } | ||
264 | |||
265 | static async listRemoteExpired () { | ||
266 | const actor = await getServerActor() | ||
267 | |||
235 | const query = { | 268 | const query = { |
236 | logging: !isTestInstance(), | ||
237 | where: { | 269 | where: { |
270 | actorId: { | ||
271 | [Sequelize.Op.ne]: actor.id | ||
272 | }, | ||
238 | expiresOn: { | 273 | expiresOn: { |
239 | [ Sequelize.Op.lt ]: new Date() | 274 | [ Sequelize.Op.lt ]: new Date() |
240 | } | 275 | } |
241 | } | 276 | } |
242 | } | 277 | } |
243 | 278 | ||
244 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) | 279 | return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) |
245 | .findAll(query) | ||
246 | } | 280 | } |
247 | 281 | ||
248 | static async getStats (strategy: VideoRedundancyStrategy) { | 282 | static async getStats (strategy: VideoRedundancyStrategy) { |
@@ -299,7 +333,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
299 | 333 | ||
300 | const notIn = Sequelize.literal( | 334 | const notIn = Sequelize.literal( |
301 | '(' + | 335 | '(' + |
302 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + | 336 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + |
303 | ')' | 337 | ')' |
304 | ) | 338 | ) |
305 | 339 | ||
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 0907ea569..0887a3738 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -106,4 +106,10 @@ export class VideoFileModel extends Model<VideoFileModel> { | |||
106 | return results.length === 1 | 106 | return results.length === 1 |
107 | }) | 107 | }) |
108 | } | 108 | } |
109 | |||
110 | hasSameUniqueKeysThan (other: VideoFileModel) { | ||
111 | return this.fps === other.fps && | ||
112 | this.resolution === other.resolution && | ||
113 | this.videoId === other.videoId | ||
114 | } | ||
109 | } | 115 | } |
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 6ce4b9dd1..a773e3de4 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts | |||
@@ -31,14 +31,13 @@ const expect = chai.expect | |||
31 | 31 | ||
32 | let servers: ServerInfo[] = [] | 32 | let servers: ServerInfo[] = [] |
33 | let video1Server2UUID: string | 33 | let video1Server2UUID: string |
34 | let video2Server2UUID: string | ||
35 | 34 | ||
36 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { | 35 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { |
37 | const parsed = magnetUtil.decode(file.magnetUri) | 36 | const parsed = magnetUtil.decode(file.magnetUri) |
38 | 37 | ||
39 | for (const ws of baseWebseeds) { | 38 | for (const ws of baseWebseeds) { |
40 | const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) | 39 | const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) |
41 | expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined | 40 | expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined |
42 | } | 41 | } |
43 | } | 42 | } |
44 | 43 | ||
@@ -49,6 +48,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: | |||
49 | check_interval: '5 seconds', | 48 | check_interval: '5 seconds', |
50 | strategies: [ | 49 | strategies: [ |
51 | immutableAssign({ | 50 | immutableAssign({ |
51 | min_lifetime: '1 hour', | ||
52 | strategy: strategy, | 52 | strategy: strategy, |
53 | size: '100KB' | 53 | size: '100KB' |
54 | }, additionalParams) | 54 | }, additionalParams) |
@@ -68,11 +68,6 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: | |||
68 | await viewVideo(servers[ 1 ].url, video1Server2UUID) | 68 | await viewVideo(servers[ 1 ].url, video1Server2UUID) |
69 | } | 69 | } |
70 | 70 | ||
71 | { | ||
72 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | ||
73 | video2Server2UUID = res.body.video.uuid | ||
74 | } | ||
75 | |||
76 | await waitJobs(servers) | 71 | await waitJobs(servers) |
77 | 72 | ||
78 | // Server 1 and server 2 follow each other | 73 | // Server 1 and server 2 follow each other |
@@ -85,68 +80,69 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: | |||
85 | await waitJobs(servers) | 80 | await waitJobs(servers) |
86 | } | 81 | } |
87 | 82 | ||
88 | async function check1WebSeed (strategy: VideoRedundancyStrategy) { | 83 | async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { |
84 | if (!videoUUID) videoUUID = video1Server2UUID | ||
85 | |||
89 | const webseeds = [ | 86 | const webseeds = [ |
90 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | 87 | 'http://localhost:9002/static/webseed/' + videoUUID |
91 | ] | 88 | ] |
92 | 89 | ||
93 | for (const server of servers) { | 90 | for (const server of servers) { |
94 | { | 91 | { |
95 | const res = await getVideo(server.url, video1Server2UUID) | 92 | const res = await getVideo(server.url, videoUUID) |
96 | 93 | ||
97 | const video: VideoDetails = res.body | 94 | const video: VideoDetails = res.body |
98 | video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) | 95 | for (const f of video.files) { |
96 | checkMagnetWebseeds(f, webseeds, server) | ||
97 | } | ||
99 | } | 98 | } |
99 | } | ||
100 | } | ||
100 | 101 | ||
101 | { | 102 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { |
102 | const res = await getStats(server.url) | 103 | const res = await getStats(servers[0].url) |
103 | const data: ServerStats = res.body | 104 | const data: ServerStats = res.body |
104 | 105 | ||
105 | expect(data.videosRedundancy).to.have.lengthOf(1) | 106 | expect(data.videosRedundancy).to.have.lengthOf(1) |
107 | const stat = data.videosRedundancy[0] | ||
106 | 108 | ||
107 | const stat = data.videosRedundancy[0] | 109 | expect(stat.strategy).to.equal(strategy) |
108 | expect(stat.strategy).to.equal(strategy) | 110 | expect(stat.totalSize).to.equal(102400) |
109 | expect(stat.totalSize).to.equal(102400) | 111 | expect(stat.totalUsed).to.be.at.least(1).and.below(102401) |
110 | expect(stat.totalUsed).to.equal(0) | 112 | expect(stat.totalVideoFiles).to.equal(4) |
111 | expect(stat.totalVideoFiles).to.equal(0) | 113 | expect(stat.totalVideos).to.equal(1) |
112 | expect(stat.totalVideos).to.equal(0) | ||
113 | } | ||
114 | } | ||
115 | } | 114 | } |
116 | 115 | ||
117 | async function enableRedundancy () { | 116 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { |
118 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | 117 | const res = await getStats(servers[0].url) |
118 | const data: ServerStats = res.body | ||
119 | 119 | ||
120 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') | 120 | expect(data.videosRedundancy).to.have.lengthOf(1) |
121 | const follows: ActorFollow[] = res.body.data | ||
122 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | ||
123 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | ||
124 | |||
125 | expect(server3).to.not.be.undefined | ||
126 | expect(server3.following.hostRedundancyAllowed).to.be.false | ||
127 | 121 | ||
128 | expect(server2).to.not.be.undefined | 122 | const stat = data.videosRedundancy[0] |
129 | expect(server2.following.hostRedundancyAllowed).to.be.true | 123 | expect(stat.strategy).to.equal(strategy) |
124 | expect(stat.totalSize).to.equal(102400) | ||
125 | expect(stat.totalUsed).to.equal(0) | ||
126 | expect(stat.totalVideoFiles).to.equal(0) | ||
127 | expect(stat.totalVideos).to.equal(0) | ||
130 | } | 128 | } |
131 | 129 | ||
132 | async function check2Webseeds (strategy: VideoRedundancyStrategy) { | 130 | async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { |
133 | await waitJobs(servers) | 131 | if (!videoUUID) videoUUID = video1Server2UUID |
134 | await wait(15000) | ||
135 | await waitJobs(servers) | ||
136 | 132 | ||
137 | const webseeds = [ | 133 | const webseeds = [ |
138 | 'http://localhost:9001/static/webseed/' + video1Server2UUID, | 134 | 'http://localhost:9001/static/webseed/' + videoUUID, |
139 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | 135 | 'http://localhost:9002/static/webseed/' + videoUUID |
140 | ] | 136 | ] |
141 | 137 | ||
142 | for (const server of servers) { | 138 | for (const server of servers) { |
143 | { | 139 | { |
144 | const res = await getVideo(server.url, video1Server2UUID) | 140 | const res = await getVideo(server.url, videoUUID) |
145 | 141 | ||
146 | const video: VideoDetails = res.body | 142 | const video: VideoDetails = res.body |
147 | 143 | ||
148 | for (const file of video.files) { | 144 | for (const file of video.files) { |
149 | checkMagnetWebseeds(file, webseeds) | 145 | checkMagnetWebseeds(file, webseeds, server) |
150 | } | 146 | } |
151 | } | 147 | } |
152 | } | 148 | } |
@@ -155,22 +151,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy) { | |||
155 | expect(files).to.have.lengthOf(4) | 151 | expect(files).to.have.lengthOf(4) |
156 | 152 | ||
157 | for (const resolution of [ 240, 360, 480, 720 ]) { | 153 | for (const resolution of [ 240, 360, 480, 720 ]) { |
158 | expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined | 154 | expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined |
159 | } | 155 | } |
156 | } | ||
160 | 157 | ||
161 | { | 158 | async function enableRedundancyOnServer1 () { |
162 | const res = await getStats(servers[0].url) | 159 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) |
163 | const data: ServerStats = res.body | ||
164 | 160 | ||
165 | expect(data.videosRedundancy).to.have.lengthOf(1) | 161 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') |
166 | const stat = data.videosRedundancy[0] | 162 | const follows: ActorFollow[] = res.body.data |
163 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | ||
164 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | ||
167 | 165 | ||
168 | expect(stat.strategy).to.equal(strategy) | 166 | expect(server3).to.not.be.undefined |
169 | expect(stat.totalSize).to.equal(102400) | 167 | expect(server3.following.hostRedundancyAllowed).to.be.false |
170 | expect(stat.totalUsed).to.be.at.least(1).and.below(102401) | 168 | |
171 | expect(stat.totalVideoFiles).to.equal(4) | 169 | expect(server2).to.not.be.undefined |
172 | expect(stat.totalVideos).to.equal(1) | 170 | expect(server2.following.hostRedundancyAllowed).to.be.true |
173 | } | ||
174 | } | 171 | } |
175 | 172 | ||
176 | async function cleanServers () { | 173 | async function cleanServers () { |
@@ -188,18 +185,24 @@ describe('Test videos redundancy', function () { | |||
188 | return runServers(strategy) | 185 | return runServers(strategy) |
189 | }) | 186 | }) |
190 | 187 | ||
191 | it('Should have 1 webseed on the first video', function () { | 188 | it('Should have 1 webseed on the first video', async function () { |
192 | return check1WebSeed(strategy) | 189 | await check1WebSeed(strategy) |
190 | await checkStatsWith1Webseed(strategy) | ||
193 | }) | 191 | }) |
194 | 192 | ||
195 | it('Should enable redundancy on server 1', function () { | 193 | it('Should enable redundancy on server 1', function () { |
196 | return enableRedundancy() | 194 | return enableRedundancyOnServer1() |
197 | }) | 195 | }) |
198 | 196 | ||
199 | it('Should have 2 webseed on the first video', function () { | 197 | it('Should have 2 webseed on the first video', async function () { |
200 | this.timeout(40000) | 198 | this.timeout(40000) |
201 | 199 | ||
202 | return check2Webseeds(strategy) | 200 | await waitJobs(servers) |
201 | await wait(15000) | ||
202 | await waitJobs(servers) | ||
203 | |||
204 | await check2Webseeds(strategy) | ||
205 | await checkStatsWith2Webseed(strategy) | ||
203 | }) | 206 | }) |
204 | 207 | ||
205 | after(function () { | 208 | after(function () { |
@@ -216,18 +219,24 @@ describe('Test videos redundancy', function () { | |||
216 | return runServers(strategy) | 219 | return runServers(strategy) |
217 | }) | 220 | }) |
218 | 221 | ||
219 | it('Should have 1 webseed on the first video', function () { | 222 | it('Should have 1 webseed on the first video', async function () { |
220 | return check1WebSeed(strategy) | 223 | await check1WebSeed(strategy) |
224 | await checkStatsWith1Webseed(strategy) | ||
221 | }) | 225 | }) |
222 | 226 | ||
223 | it('Should enable redundancy on server 1', function () { | 227 | it('Should enable redundancy on server 1', function () { |
224 | return enableRedundancy() | 228 | return enableRedundancyOnServer1() |
225 | }) | 229 | }) |
226 | 230 | ||
227 | it('Should have 2 webseed on the first video', function () { | 231 | it('Should have 2 webseed on the first video', async function () { |
228 | this.timeout(40000) | 232 | this.timeout(40000) |
229 | 233 | ||
230 | return check2Webseeds(strategy) | 234 | await waitJobs(servers) |
235 | await wait(15000) | ||
236 | await waitJobs(servers) | ||
237 | |||
238 | await check2Webseeds(strategy) | ||
239 | await checkStatsWith2Webseed(strategy) | ||
231 | }) | 240 | }) |
232 | 241 | ||
233 | after(function () { | 242 | after(function () { |
@@ -241,15 +250,16 @@ describe('Test videos redundancy', function () { | |||
241 | before(function () { | 250 | before(function () { |
242 | this.timeout(120000) | 251 | this.timeout(120000) |
243 | 252 | ||
244 | return runServers(strategy, { minViews: 3 }) | 253 | return runServers(strategy, { min_views: 3 }) |
245 | }) | 254 | }) |
246 | 255 | ||
247 | it('Should have 1 webseed on the first video', function () { | 256 | it('Should have 1 webseed on the first video', async function () { |
248 | return check1WebSeed(strategy) | 257 | await check1WebSeed(strategy) |
258 | await checkStatsWith1Webseed(strategy) | ||
249 | }) | 259 | }) |
250 | 260 | ||
251 | it('Should enable redundancy on server 1', function () { | 261 | it('Should enable redundancy on server 1', function () { |
252 | return enableRedundancy() | 262 | return enableRedundancyOnServer1() |
253 | }) | 263 | }) |
254 | 264 | ||
255 | it('Should still have 1 webseed on the first video', async function () { | 265 | it('Should still have 1 webseed on the first video', async function () { |
@@ -259,10 +269,11 @@ describe('Test videos redundancy', function () { | |||
259 | await wait(15000) | 269 | await wait(15000) |
260 | await waitJobs(servers) | 270 | await waitJobs(servers) |
261 | 271 | ||
262 | return check1WebSeed(strategy) | 272 | await check1WebSeed(strategy) |
273 | await checkStatsWith1Webseed(strategy) | ||
263 | }) | 274 | }) |
264 | 275 | ||
265 | it('Should view 2 times the first video', async function () { | 276 | it('Should view 2 times the first video to have > min_views config', async function () { |
266 | this.timeout(40000) | 277 | this.timeout(40000) |
267 | 278 | ||
268 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | 279 | await viewVideo(servers[ 0 ].url, video1Server2UUID) |
@@ -272,10 +283,117 @@ describe('Test videos redundancy', function () { | |||
272 | await waitJobs(servers) | 283 | await waitJobs(servers) |
273 | }) | 284 | }) |
274 | 285 | ||
275 | it('Should have 2 webseed on the first video', function () { | 286 | it('Should have 2 webseed on the first video', async function () { |
276 | this.timeout(40000) | 287 | this.timeout(40000) |
277 | 288 | ||
278 | return check2Webseeds(strategy) | 289 | await waitJobs(servers) |
290 | await wait(15000) | ||
291 | await waitJobs(servers) | ||
292 | |||
293 | await check2Webseeds(strategy) | ||
294 | await checkStatsWith2Webseed(strategy) | ||
295 | }) | ||
296 | |||
297 | after(function () { | ||
298 | return cleanServers() | ||
299 | }) | ||
300 | }) | ||
301 | |||
302 | describe('Test expiration', function () { | ||
303 | const strategy = 'recently-added' | ||
304 | |||
305 | async function checkContains (servers: ServerInfo[], str: string) { | ||
306 | for (const server of servers) { | ||
307 | const res = await getVideo(server.url, video1Server2UUID) | ||
308 | const video: VideoDetails = res.body | ||
309 | |||
310 | for (const f of video.files) { | ||
311 | expect(f.magnetUri).to.contain(str) | ||
312 | } | ||
313 | } | ||
314 | } | ||
315 | |||
316 | async function checkNotContains (servers: ServerInfo[], str: string) { | ||
317 | for (const server of servers) { | ||
318 | const res = await getVideo(server.url, video1Server2UUID) | ||
319 | const video: VideoDetails = res.body | ||
320 | |||
321 | for (const f of video.files) { | ||
322 | expect(f.magnetUri).to.not.contain(str) | ||
323 | } | ||
324 | } | ||
325 | } | ||
326 | |||
327 | before(async function () { | ||
328 | this.timeout(120000) | ||
329 | |||
330 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | ||
331 | |||
332 | await enableRedundancyOnServer1() | ||
333 | }) | ||
334 | |||
335 | it('Should still have 2 webseeds after 10 seconds', async function () { | ||
336 | this.timeout(40000) | ||
337 | |||
338 | await wait(10000) | ||
339 | |||
340 | try { | ||
341 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | ||
342 | } catch { | ||
343 | // Maybe a server deleted a redundancy in the scheduler | ||
344 | await wait(2000) | ||
345 | |||
346 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | ||
347 | } | ||
348 | }) | ||
349 | |||
350 | it('Should stop server 1 and expire video redundancy', async function () { | ||
351 | this.timeout(40000) | ||
352 | |||
353 | killallServers([ servers[0] ]) | ||
354 | |||
355 | await wait(10000) | ||
356 | |||
357 | await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') | ||
358 | }) | ||
359 | |||
360 | after(function () { | ||
361 | return killallServers([ servers[1], servers[2] ]) | ||
362 | }) | ||
363 | }) | ||
364 | |||
365 | describe('Test file replacement', function () { | ||
366 | let video2Server2UUID: string | ||
367 | const strategy = 'recently-added' | ||
368 | |||
369 | before(async function () { | ||
370 | this.timeout(120000) | ||
371 | |||
372 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | ||
373 | |||
374 | await enableRedundancyOnServer1() | ||
375 | |||
376 | await waitJobs(servers) | ||
377 | await wait(5000) | ||
378 | await waitJobs(servers) | ||
379 | |||
380 | await check2Webseeds(strategy) | ||
381 | await checkStatsWith2Webseed(strategy) | ||
382 | |||
383 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | ||
384 | video2Server2UUID = res.body.video.uuid | ||
385 | }) | ||
386 | |||
387 | it('Should cache video 2 webseed on the first video', async function () { | ||
388 | this.timeout(40000) | ||
389 | this.retries(3) | ||
390 | |||
391 | await waitJobs(servers) | ||
392 | |||
393 | await wait(7000) | ||
394 | |||
395 | await check1WebSeed(strategy, video1Server2UUID) | ||
396 | await check2Webseeds(strategy, video2Server2UUID) | ||
279 | }) | 397 | }) |
280 | 398 | ||
281 | after(function () { | 399 | after(function () { |
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts index 26ab4e1bb..fbfc83ca1 100644 --- a/server/tests/utils/server/servers.ts +++ b/server/tests/utils/server/servers.ts | |||
@@ -144,8 +144,8 @@ function runServer (serverNumber: number, configOverride?: Object) { | |||
144 | }) | 144 | }) |
145 | } | 145 | } |
146 | 146 | ||
147 | async function reRunServer (server: ServerInfo) { | 147 | async function reRunServer (server: ServerInfo, configOverride?: any) { |
148 | const newServer = await runServer(server.serverNumber) | 148 | const newServer = await runServer(server.serverNumber, configOverride) |
149 | server.app = newServer.app | 149 | server.app = newServer.app |
150 | 150 | ||
151 | return server | 151 | return server |
diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts index 436394c1e..a8c2743c1 100644 --- a/shared/models/redundancy/videos-redundancy.model.ts +++ b/shared/models/redundancy/videos-redundancy.model.ts | |||
@@ -3,17 +3,20 @@ export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-adde | |||
3 | export type MostViewsRedundancyStrategy = { | 3 | export type MostViewsRedundancyStrategy = { |
4 | strategy: 'most-views' | 4 | strategy: 'most-views' |
5 | size: number | 5 | size: number |
6 | minLifetime: number | ||
6 | } | 7 | } |
7 | 8 | ||
8 | export type TrendingRedundancyStrategy = { | 9 | export type TrendingRedundancyStrategy = { |
9 | strategy: 'trending' | 10 | strategy: 'trending' |
10 | size: number | 11 | size: number |
12 | minLifetime: number | ||
11 | } | 13 | } |
12 | 14 | ||
13 | export type RecentlyAddedStrategy = { | 15 | export type RecentlyAddedStrategy = { |
14 | strategy: 'recently-added' | 16 | strategy: 'recently-added' |
15 | size: number | 17 | size: number |
16 | minViews: number | 18 | minViews: number |
19 | minLifetime: number | ||
17 | } | 20 | } |
18 | 21 | ||
19 | export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy | 22 | export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy |
diff --git a/support/doc/redundancy.md b/support/doc/redundancy.md new file mode 100644 index 000000000..62c6365a9 --- /dev/null +++ b/support/doc/redundancy.md | |||
@@ -0,0 +1,46 @@ | |||
1 | # Redundancy | ||
2 | |||
3 | A PeerTube instance can cache other PeerTube videos to improve bandwidth of popular videos or small instances. | ||
4 | |||
5 | ## How it works | ||
6 | |||
7 | The instance administrator can choose between multiple redundancy strategies (cache trending videos or recently uploaded videos etc), set their maximum size and the minimum duplication lifetime. | ||
8 | Then, they choose the instances they want to cache in `Manage follows -> Following` admin table. | ||
9 | |||
10 | Videos are kept in the cache for at least `min_lifetime`, and then evicted when the cache is full. | ||
11 | |||
12 | When PeerTube chooses a video to duplicate, it imports all the resolution files (to avoid consistency issues) using their magnet URI and put them in the `storage.videos` directory. | ||
13 | Then it sends a `Create -> CacheFile` ActivityPub message to other federated instances. This new instance is injected as [WebSeed](https://github.com/Chocobozzz/PeerTube/blob/develop/FAQ.md#what-is-webseed) in the magnet URI by instances that received this ActivityPub message. | ||
14 | |||
15 | ## Stats | ||
16 | |||
17 | See the `/api/v1/server/stats` endpoint. For example: | ||
18 | |||
19 | ``` | ||
20 | { | ||
21 | ... | ||
22 | "videosRedundancy": [ | ||
23 | { | ||
24 | "totalUsed": 0, | ||
25 | "totalVideos": 0, | ||
26 | "totalVideoFiles": 0, | ||
27 | "strategy": "most-views", | ||
28 | "totalSize": 104857600 | ||
29 | }, | ||
30 | { | ||
31 | "totalUsed": 0, | ||
32 | "totalVideos": 0, | ||
33 | "totalVideoFiles": 0, | ||
34 | "strategy": "trending", | ||
35 | "totalSize": 104857600 | ||
36 | }, | ||
37 | { | ||
38 | "totalUsed": 0, | ||
39 | "totalVideos": 0, | ||
40 | "totalVideoFiles": 0, | ||
41 | "strategy": "recently-added", | ||
42 | "totalSize": 104857600 | ||
43 | } | ||
44 | ] | ||
45 | } | ||
46 | ``` \ No newline at end of file | ||