aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--config/default.yaml8
-rw-r--r--config/production.yaml.example8
-rw-r--r--server/initializers/checker-after-init.ts8
-rw-r--r--server/initializers/checker-before-init.ts3
-rw-r--r--server/initializers/config.ts6
-rw-r--r--server/lib/activitypub/process/process-create.ts3
-rw-r--r--server/lib/activitypub/process/process-update.ts3
-rw-r--r--server/lib/redundancy.ts27
-rw-r--r--server/models/activitypub/actor-follow.ts13
-rw-r--r--server/tests/api/redundancy/index.ts1
-rw-r--r--server/tests/api/redundancy/redundancy-constraints.ts200
-rw-r--r--shared/models/redundancy/video-redundancy-config-filter.type.ts1
12 files changed, 279 insertions, 2 deletions
diff --git a/config/default.yaml b/config/default.yaml
index 1a8b19136..0b096cf8d 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -126,6 +126,14 @@ redundancy:
126# strategy: 'recently-added' # Cache recently added videos 126# strategy: 'recently-added' # Cache recently added videos
127# min_views: 10 # Having at least x views 127# min_views: 10 # Having at least x views
128 128
129# Other instances that duplicate your content
130remote_redundancy:
131 videos:
132 # 'nobody': Do not accept remote redundancies
133 # 'anybody': Accept remote redundancies from anybody
134 # 'followings': Accept redundancies from instance followings
135 accept_from: 'anybody'
136
129csp: 137csp:
130 enabled: false 138 enabled: false
131 report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk! 139 report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk!
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 30cd2ffe0..b6f7d1913 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -127,6 +127,14 @@ redundancy:
127# strategy: 'recently-added' # Cache recently added videos 127# strategy: 'recently-added' # Cache recently added videos
128# min_views: 10 # Having at least x views 128# min_views: 10 # Having at least x views
129 129
130# Other instances that duplicate your content
131remote_redundancy:
132 videos:
133 # 'nobody': Do not accept remote redundancies
134 # 'anybody': Accept remote redundancies from anybody
135 # 'followings': Accept redundancies from instance followings
136 accept_from: 'anybody'
137
130csp: 138csp:
131 enabled: false 139 enabled: false
132 report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk! 140 report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk!
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index bc4aae957..a57d552df 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -11,6 +11,7 @@ import { RecentlyAddedStrategy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc' 11import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash' 12import { uniq } from 'lodash'
13import { WEBSERVER } from './constants' 13import { WEBSERVER } from './constants'
14import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
14 15
15async function checkActivityPubUrls () { 16async function checkActivityPubUrls () {
16 const actor = await getServerActor() 17 const actor = await getServerActor()
@@ -87,6 +88,13 @@ function checkConfig () {
87 return 'Videos redundancy should be an array (you must uncomment lines containing - too)' 88 return 'Videos redundancy should be an array (you must uncomment lines containing - too)'
88 } 89 }
89 90
91 // Remote redundancies
92 const acceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM
93 const acceptFromValues = new Set<VideoRedundancyConfigFilter>([ 'nobody', 'anybody', 'followings' ])
94 if (acceptFromValues.has(acceptFrom) === false) {
95 return 'remote_redundancy.videos.accept_from has an incorrect value'
96 }
97
90 // Check storage directory locations 98 // Check storage directory locations
91 if (isProdInstance()) { 99 if (isProdInstance()) {
92 const configStorage = config.get('storage') 100 const configStorage = config.get('storage')
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index a75f2cec2..064d89a4d 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -31,7 +31,8 @@ function checkMissedConfig () {
31 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', 31 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
32 'history.videos.max_age', 'views.videos.remote.max_age', 32 'history.videos.max_age', 'views.videos.remote.max_age',
33 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', 33 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
34 'theme.default' 34 'theme.default',
35 'remote_redundancy.videos.accept_from'
35 ] 36 ]
36 const requiredAlternatives = [ 37 const requiredAlternatives = [
37 [ // set 38 [ // set
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 3c07624e8..2c4d26a9e 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -5,6 +5,7 @@ import { VideosRedundancyStrategy } from '../../shared/models'
5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' 5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
7import * as bytes from 'bytes' 7import * as bytes from 'bytes'
8import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type'
8 9
9// Use a variable to reload the configuration if we need 10// Use a variable to reload the configuration if we need
10let config: IConfig = require('config') 11let config: IConfig = require('config')
@@ -117,6 +118,11 @@ const CONFIG = {
117 STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies')) 118 STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
118 } 119 }
119 }, 120 },
121 REMOTE_REDUNDANCY: {
122 VIDEOS: {
123 ACCEPT_FROM: config.get<VideoRedundancyConfigFilter>('remote_redundancy.videos.accept_from')
124 }
125 },
120 CSP: { 126 CSP: {
121 ENABLED: config.get<boolean>('csp.enabled'), 127 ENABLED: config.get<boolean>('csp.enabled'),
122 REPORT_ONLY: config.get<boolean>('csp.report_only'), 128 REPORT_ONLY: config.get<boolean>('csp.report_only'),
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index bee853721..d375e29e3 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -12,6 +12,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl
12import { createOrUpdateVideoPlaylist } from '../playlist' 12import { createOrUpdateVideoPlaylist } from '../playlist'
13import { APProcessorOptions } from '../../../typings/activitypub-processor.model' 13import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
14import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../typings/models' 14import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../typings/models'
15import { isRedundancyAccepted } from '@server/lib/redundancy'
15 16
16async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { 17async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
17 const { activity, byActor } = options 18 const { activity, byActor } = options
@@ -60,6 +61,8 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) {
60} 61}
61 62
62async function processCreateCacheFile (activity: ActivityCreate, byActor: MActorSignature) { 63async function processCreateCacheFile (activity: ActivityCreate, byActor: MActorSignature) {
64 if (await isRedundancyAccepted(activity, byActor) !== true) return
65
63 const cacheFile = activity.object as CacheFileObject 66 const cacheFile = activity.object as CacheFileObject
64 67
65 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 68 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index a47d605d8..9579512b7 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -16,6 +16,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl
16import { createOrUpdateVideoPlaylist } from '../playlist' 16import { createOrUpdateVideoPlaylist } from '../playlist'
17import { APProcessorOptions } from '../../../typings/activitypub-processor.model' 17import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
18import { MActorSignature, MAccountIdActor } from '../../../typings/models' 18import { MActorSignature, MAccountIdActor } from '../../../typings/models'
19import { isRedundancyAccepted } from '@server/lib/redundancy'
19 20
20async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { 21async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
21 const { activity, byActor } = options 22 const { activity, byActor } = options
@@ -78,6 +79,8 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd
78} 79}
79 80
80async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { 81async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) {
82 if (await isRedundancyAccepted(activity, byActor) !== true) return
83
81 const cacheFileObject = activity.object as CacheFileObject 84 const cacheFileObject = activity.object as CacheFileObject
82 85
83 if (!isCacheFileObjectValid(cacheFileObject)) { 86 if (!isCacheFileObjectValid(cacheFileObject)) {
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 78d84e02e..aa0e37478 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -2,7 +2,11 @@ import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
2import { sendUndoCacheFile } from './activitypub/send' 2import { sendUndoCacheFile } from './activitypub/send'
3import { Transaction } from 'sequelize' 3import { Transaction } from 'sequelize'
4import { getServerActor } from '../helpers/utils' 4import { getServerActor } from '../helpers/utils'
5import { MVideoRedundancyVideo } from '@server/typings/models' 5import { MActorSignature, MVideoRedundancyVideo } from '@server/typings/models'
6import { CONFIG } from '@server/initializers/config'
7import { logger } from '@server/helpers/logger'
8import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
9import { Activity } from '@shared/models'
6 10
7async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { 11async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) {
8 const serverActor = await getServerActor() 12 const serverActor = await getServerActor()
@@ -21,9 +25,30 @@ async function removeRedundanciesOfServer (serverId: number) {
21 } 25 }
22} 26}
23 27
28async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) {
29 const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM
30 if (configAcceptFrom === 'nobody') {
31 logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id)
32 return false
33 }
34
35 if (configAcceptFrom === 'followings') {
36 const serverActor = await getServerActor()
37 const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id)
38
39 if (allowed !== true) {
40 logger.info('Do not accept remote redundancy %s because actor %s is not followed by our instance.', activity.id, byActor.url)
41 return false
42 }
43 }
44
45 return true
46}
47
24// --------------------------------------------------------------------------- 48// ---------------------------------------------------------------------------
25 49
26export { 50export {
51 isRedundancyAccepted,
27 removeRedundanciesOfServer, 52 removeRedundanciesOfServer,
28 removeVideoRedundancy 53 removeVideoRedundancy
29} 54}
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index 27643704e..5a8e450a5 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -36,6 +36,7 @@ import {
36 MActorFollowSubscriptions 36 MActorFollowSubscriptions
37} from '@server/typings/models' 37} from '@server/typings/models'
38import { ActivityPubActorType } from '@shared/models' 38import { ActivityPubActorType } from '@shared/models'
39import { VideoModel } from '@server/models/video/video'
39 40
40@Table({ 41@Table({
41 tableName: 'actorFollow', 42 tableName: 'actorFollow',
@@ -151,6 +152,18 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
151 if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved) 152 if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved)
152 } 153 }
153 154
155 static isFollowedBy (actorId: number, followerActorId: number) {
156 const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
157 const options = {
158 type: QueryTypes.SELECT as QueryTypes.SELECT,
159 bind: { actorId, followerActorId },
160 raw: true
161 }
162
163 return VideoModel.sequelize.query(query, options)
164 .then(results => results.length === 1)
165 }
166
154 static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Bluebird<MActorFollowActorsDefault> { 167 static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Bluebird<MActorFollowActorsDefault> {
155 const query = { 168 const query = {
156 where: { 169 where: {
diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts
index 5359055b0..37dc3f88c 100644
--- a/server/tests/api/redundancy/index.ts
+++ b/server/tests/api/redundancy/index.ts
@@ -1,2 +1,3 @@
1import './redundancy-constraints'
1import './redundancy' 2import './redundancy'
2import './manage-redundancy' 3import './manage-redundancy'
diff --git a/server/tests/api/redundancy/redundancy-constraints.ts b/server/tests/api/redundancy/redundancy-constraints.ts
new file mode 100644
index 000000000..4fd8f065c
--- /dev/null
+++ b/server/tests/api/redundancy/redundancy-constraints.ts
@@ -0,0 +1,200 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 cleanupTests,
7 flushAndRunServer,
8 follow,
9 killallServers,
10 reRunServer,
11 ServerInfo,
12 setAccessTokensToServers,
13 uploadVideo,
14 waitUntilLog
15} from '../../../../shared/extra-utils'
16import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
17import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy'
18
19const expect = chai.expect
20
21describe('Test redundancy constraints', function () {
22 let remoteServer: ServerInfo
23 let localServer: ServerInfo
24 let servers: ServerInfo[]
25
26 async function getTotalRedundanciesLocalServer () {
27 const res = await listVideoRedundancies({
28 url: localServer.url,
29 accessToken: localServer.accessToken,
30 target: 'my-videos'
31 })
32
33 return res.body.total
34 }
35
36 async function getTotalRedundanciesRemoteServer () {
37 const res = await listVideoRedundancies({
38 url: remoteServer.url,
39 accessToken: remoteServer.accessToken,
40 target: 'remote-videos'
41 })
42
43 return res.body.total
44 }
45
46 before(async function () {
47 this.timeout(120000)
48
49 {
50 const config = {
51 redundancy: {
52 videos: {
53 check_interval: '1 second',
54 strategies: [
55 {
56 strategy: 'recently-added',
57 min_lifetime: '1 hour',
58 size: '100MB',
59 min_views: 0
60 }
61 ]
62 }
63 }
64 }
65 remoteServer = await flushAndRunServer(1, config)
66 }
67
68 {
69 const config = {
70 remote_redundancy: {
71 videos: {
72 accept_from: 'nobody'
73 }
74 }
75 }
76 localServer = await flushAndRunServer(2, config)
77 }
78
79 servers = [ remoteServer, localServer ]
80
81 // Get the access tokens
82 await setAccessTokensToServers(servers)
83
84 await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 1 server 2' })
85
86 await waitJobs(servers)
87
88 // Server 1 and server 2 follow each other
89 await follow(remoteServer.url, [ localServer.url ], remoteServer.accessToken)
90 await waitJobs(servers)
91 await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true)
92
93 await waitJobs(servers)
94 })
95
96 it('Should have redundancy on server 1 but not on server 2 with a nobody filter', async function () {
97 this.timeout(120000)
98
99 await waitJobs(servers)
100 await waitUntilLog(remoteServer, 'Duplicated ', 5)
101 await waitJobs(servers)
102
103 {
104 const total = await getTotalRedundanciesRemoteServer()
105 expect(total).to.equal(1)
106 }
107
108 {
109 const total = await getTotalRedundanciesLocalServer()
110 expect(total).to.equal(0)
111 }
112 })
113
114 it('Should have redundancy on server 1 and on server 2 with an anybody filter', async function () {
115 this.timeout(120000)
116
117 const config = {
118 remote_redundancy: {
119 videos: {
120 accept_from: 'anybody'
121 }
122 }
123 }
124 await killallServers([ localServer ])
125 await reRunServer(localServer, config)
126
127 await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 2 server 2' })
128
129 await waitJobs(servers)
130 await waitUntilLog(remoteServer, 'Duplicated ', 10)
131 await waitJobs(servers)
132
133 {
134 const total = await getTotalRedundanciesRemoteServer()
135 expect(total).to.equal(2)
136 }
137
138 {
139 const total = await getTotalRedundanciesLocalServer()
140 expect(total).to.equal(1)
141 }
142 })
143
144 it('Should have redundancy on server 1 but not on server 2 with a followings filter', async function () {
145 this.timeout(120000)
146
147 const config = {
148 remote_redundancy: {
149 videos: {
150 accept_from: 'followings'
151 }
152 }
153 }
154 await killallServers([ localServer ])
155 await reRunServer(localServer, config)
156
157 await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 3 server 2' })
158
159 await waitJobs(servers)
160 await waitUntilLog(remoteServer, 'Duplicated ', 15)
161 await waitJobs(servers)
162
163 {
164 const total = await getTotalRedundanciesRemoteServer()
165 expect(total).to.equal(3)
166 }
167
168 {
169 const total = await getTotalRedundanciesLocalServer()
170 expect(total).to.equal(1)
171 }
172 })
173
174 it('Should have redundancy on server 1 and on server 2 with followings filter now server 2 follows server 1', async function () {
175 this.timeout(120000)
176
177 await follow(localServer.url, [ remoteServer.url ], localServer.accessToken)
178 await waitJobs(servers)
179
180 await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 4 server 2' })
181
182 await waitJobs(servers)
183 await waitUntilLog(remoteServer, 'Duplicated ', 20)
184 await waitJobs(servers)
185
186 {
187 const total = await getTotalRedundanciesRemoteServer()
188 expect(total).to.equal(4)
189 }
190
191 {
192 const total = await getTotalRedundanciesLocalServer()
193 expect(total).to.equal(2)
194 }
195 })
196
197 after(async function () {
198 await cleanupTests(servers)
199 })
200})
diff --git a/shared/models/redundancy/video-redundancy-config-filter.type.ts b/shared/models/redundancy/video-redundancy-config-filter.type.ts
new file mode 100644
index 000000000..bb1ae701c
--- /dev/null
+++ b/shared/models/redundancy/video-redundancy-config-filter.type.ts
@@ -0,0 +1 @@
export type VideoRedundancyConfigFilter = 'nobody' | 'anybody' | 'followings'