From 8c9e7875269a990ed3000e1d4995d808e4ff50f7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Apr 2020 15:27:41 +0200 Subject: [PATCH] Add ability to accept or not remote redundancies --- config/default.yaml | 8 + config/production.yaml.example | 8 + server/initializers/checker-after-init.ts | 8 + server/initializers/checker-before-init.ts | 3 +- server/initializers/config.ts | 6 + .../lib/activitypub/process/process-create.ts | 3 + .../lib/activitypub/process/process-update.ts | 3 + server/lib/redundancy.ts | 27 ++- server/models/activitypub/actor-follow.ts | 13 ++ server/tests/api/redundancy/index.ts | 1 + .../api/redundancy/redundancy-constraints.ts | 200 ++++++++++++++++++ .../video-redundancy-config-filter.type.ts | 1 + 12 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 server/tests/api/redundancy/redundancy-constraints.ts create mode 100644 shared/models/redundancy/video-redundancy-config-filter.type.ts diff --git a/config/default.yaml b/config/default.yaml index 1a8b19136..0b096cf8d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -126,6 +126,14 @@ redundancy: # strategy: 'recently-added' # Cache recently added videos # min_views: 10 # Having at least x views +# Other instances that duplicate your content +remote_redundancy: + videos: + # 'nobody': Do not accept remote redundancies + # 'anybody': Accept remote redundancies from anybody + # 'followings': Accept redundancies from instance followings + accept_from: 'anybody' + csp: enabled: false report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk! diff --git a/config/production.yaml.example b/config/production.yaml.example index 30cd2ffe0..b6f7d1913 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -127,6 +127,14 @@ redundancy: # strategy: 'recently-added' # Cache recently added videos # min_views: 10 # Having at least x views +# Other instances that duplicate your content +remote_redundancy: + videos: + # 'nobody': Do not accept remote redundancies + # 'anybody': Accept remote redundancies from anybody + # 'followings': Accept redundancies from instance followings + accept_from: 'anybody' + csp: enabled: false report_only: true # CSP directives are still being tested, so disable the report only mode at your own risk! diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index bc4aae957..a57d552df 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -11,6 +11,7 @@ import { RecentlyAddedStrategy } from '../../shared/models/redundancy' import { isArray } from '../helpers/custom-validators/misc' import { uniq } from 'lodash' import { WEBSERVER } from './constants' +import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' async function checkActivityPubUrls () { const actor = await getServerActor() @@ -87,6 +88,13 @@ function checkConfig () { return 'Videos redundancy should be an array (you must uncomment lines containing - too)' } + // Remote redundancies + const acceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM + const acceptFromValues = new Set([ 'nobody', 'anybody', 'followings' ]) + if (acceptFromValues.has(acceptFrom) === false) { + return 'remote_redundancy.videos.accept_from has an incorrect value' + } + // Check storage directory locations if (isProdInstance()) { const configStorage = config.get('storage') diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index a75f2cec2..064d89a4d 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -31,7 +31,8 @@ function checkMissedConfig () { 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', 'history.videos.max_age', 'views.videos.remote.max_age', 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', - 'theme.default' + 'theme.default', + 'remote_redundancy.videos.accept_from' ] const requiredAlternatives = [ [ // set diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 3c07624e8..2c4d26a9e 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -5,6 +5,7 @@ import { VideosRedundancyStrategy } from '../../shared/models' import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import * as bytes from 'bytes' +import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' // Use a variable to reload the configuration if we need let config: IConfig = require('config') @@ -117,6 +118,11 @@ const CONFIG = { STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) } }, + REMOTE_REDUNDANCY: { + VIDEOS: { + ACCEPT_FROM: config.get('remote_redundancy.videos.accept_from') + } + }, CSP: { ENABLED: config.get('csp.enabled'), REPORT_ONLY: config.get('csp.report_only'), diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index bee853721..d375e29e3 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -12,6 +12,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl import { createOrUpdateVideoPlaylist } from '../playlist' import { APProcessorOptions } from '../../../typings/activitypub-processor.model' import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../typings/models' +import { isRedundancyAccepted } from '@server/lib/redundancy' async function processCreateActivity (options: APProcessorOptions) { const { activity, byActor } = options @@ -60,6 +61,8 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) { } async function processCreateCacheFile (activity: ActivityCreate, byActor: MActorSignature) { + if (await isRedundancyAccepted(activity, byActor) !== true) return + const cacheFile = activity.object as CacheFileObject const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index a47d605d8..9579512b7 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -16,6 +16,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl import { createOrUpdateVideoPlaylist } from '../playlist' import { APProcessorOptions } from '../../../typings/activitypub-processor.model' import { MActorSignature, MAccountIdActor } from '../../../typings/models' +import { isRedundancyAccepted } from '@server/lib/redundancy' async function processUpdateActivity (options: APProcessorOptions) { const { activity, byActor } = options @@ -78,6 +79,8 @@ async function processUpdateVideo (actor: MActorSignature, activity: ActivityUpd } async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { + if (await isRedundancyAccepted(activity, byActor) !== true) return + const cacheFileObject = activity.object as CacheFileObject if (!isCacheFileObjectValid(cacheFileObject)) { diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 78d84e02e..aa0e37478 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts @@ -2,7 +2,11 @@ import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' import { sendUndoCacheFile } from './activitypub/send' import { Transaction } from 'sequelize' import { getServerActor } from '../helpers/utils' -import { MVideoRedundancyVideo } from '@server/typings/models' +import { MActorSignature, MVideoRedundancyVideo } from '@server/typings/models' +import { CONFIG } from '@server/initializers/config' +import { logger } from '@server/helpers/logger' +import { ActorFollowModel } from '@server/models/activitypub/actor-follow' +import { Activity } from '@shared/models' async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { const serverActor = await getServerActor() @@ -21,9 +25,30 @@ async function removeRedundanciesOfServer (serverId: number) { } } +async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) { + const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM + if (configAcceptFrom === 'nobody') { + logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id) + return false + } + + if (configAcceptFrom === 'followings') { + const serverActor = await getServerActor() + const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id) + + if (allowed !== true) { + logger.info('Do not accept remote redundancy %s because actor %s is not followed by our instance.', activity.id, byActor.url) + return false + } + } + + return true +} + // --------------------------------------------------------------------------- export { + isRedundancyAccepted, removeRedundanciesOfServer, removeVideoRedundancy } diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 27643704e..5a8e450a5 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -36,6 +36,7 @@ import { MActorFollowSubscriptions } from '@server/typings/models' import { ActivityPubActorType } from '@shared/models' +import { VideoModel } from '@server/models/video/video' @Table({ tableName: 'actorFollow', @@ -151,6 +152,18 @@ export class ActorFollowModel extends Model { if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved) } + static isFollowedBy (actorId: number, followerActorId: number) { + const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1' + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + bind: { actorId, followerActorId }, + raw: true + } + + return VideoModel.sequelize.query(query, options) + .then(results => results.length === 1) + } + static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Bluebird { const query = { where: { diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts index 5359055b0..37dc3f88c 100644 --- a/server/tests/api/redundancy/index.ts +++ b/server/tests/api/redundancy/index.ts @@ -1,2 +1,3 @@ +import './redundancy-constraints' import './redundancy' import './manage-redundancy' diff --git a/server/tests/api/redundancy/redundancy-constraints.ts b/server/tests/api/redundancy/redundancy-constraints.ts new file mode 100644 index 000000000..4fd8f065c --- /dev/null +++ b/server/tests/api/redundancy/redundancy-constraints.ts @@ -0,0 +1,200 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import * as chai from 'chai' +import 'mocha' +import { + cleanupTests, + flushAndRunServer, + follow, + killallServers, + reRunServer, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + waitUntilLog +} from '../../../../shared/extra-utils' +import { waitJobs } from '../../../../shared/extra-utils/server/jobs' +import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy' + +const expect = chai.expect + +describe('Test redundancy constraints', function () { + let remoteServer: ServerInfo + let localServer: ServerInfo + let servers: ServerInfo[] + + async function getTotalRedundanciesLocalServer () { + const res = await listVideoRedundancies({ + url: localServer.url, + accessToken: localServer.accessToken, + target: 'my-videos' + }) + + return res.body.total + } + + async function getTotalRedundanciesRemoteServer () { + const res = await listVideoRedundancies({ + url: remoteServer.url, + accessToken: remoteServer.accessToken, + target: 'remote-videos' + }) + + return res.body.total + } + + before(async function () { + this.timeout(120000) + + { + const config = { + redundancy: { + videos: { + check_interval: '1 second', + strategies: [ + { + strategy: 'recently-added', + min_lifetime: '1 hour', + size: '100MB', + min_views: 0 + } + ] + } + } + } + remoteServer = await flushAndRunServer(1, config) + } + + { + const config = { + remote_redundancy: { + videos: { + accept_from: 'nobody' + } + } + } + localServer = await flushAndRunServer(2, config) + } + + servers = [ remoteServer, localServer ] + + // Get the access tokens + await setAccessTokensToServers(servers) + + await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 1 server 2' }) + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await follow(remoteServer.url, [ localServer.url ], remoteServer.accessToken) + await waitJobs(servers) + await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true) + + await waitJobs(servers) + }) + + it('Should have redundancy on server 1 but not on server 2 with a nobody filter', async function () { + this.timeout(120000) + + await waitJobs(servers) + await waitUntilLog(remoteServer, 'Duplicated ', 5) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(1) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(0) + } + }) + + it('Should have redundancy on server 1 and on server 2 with an anybody filter', async function () { + this.timeout(120000) + + const config = { + remote_redundancy: { + videos: { + accept_from: 'anybody' + } + } + } + await killallServers([ localServer ]) + await reRunServer(localServer, config) + + await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 2 server 2' }) + + await waitJobs(servers) + await waitUntilLog(remoteServer, 'Duplicated ', 10) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(2) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(1) + } + }) + + it('Should have redundancy on server 1 but not on server 2 with a followings filter', async function () { + this.timeout(120000) + + const config = { + remote_redundancy: { + videos: { + accept_from: 'followings' + } + } + } + await killallServers([ localServer ]) + await reRunServer(localServer, config) + + await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 3 server 2' }) + + await waitJobs(servers) + await waitUntilLog(remoteServer, 'Duplicated ', 15) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(3) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(1) + } + }) + + it('Should have redundancy on server 1 and on server 2 with followings filter now server 2 follows server 1', async function () { + this.timeout(120000) + + await follow(localServer.url, [ remoteServer.url ], localServer.accessToken) + await waitJobs(servers) + + await uploadVideo(localServer.url, localServer.accessToken, { name: 'video 4 server 2' }) + + await waitJobs(servers) + await waitUntilLog(remoteServer, 'Duplicated ', 20) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(4) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(2) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/shared/models/redundancy/video-redundancy-config-filter.type.ts b/shared/models/redundancy/video-redundancy-config-filter.type.ts new file mode 100644 index 000000000..bb1ae701c --- /dev/null +++ b/shared/models/redundancy/video-redundancy-config-filter.type.ts @@ -0,0 +1 @@ +export type VideoRedundancyConfigFilter = 'nobody' | 'anybody' | 'followings' -- 2.41.0