aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-24 13:07:33 +0200
committerChocobozzz <me@florianbigard.com>2018-09-24 13:38:39 +0200
commite5565833f62b97f62ea75eba5b479963ae78b873 (patch)
tree835793ce464f9666b0ceae79f3d278cc4e007b32 /server
parentd1a63fc7ac58a1db00d8ca4f43aadba02eb9b084 (diff)
downloadPeerTube-e5565833f62b97f62ea75eba5b479963ae78b873.tar.gz
PeerTube-e5565833f62b97f62ea75eba5b479963ae78b873.tar.zst
PeerTube-e5565833f62b97f62ea75eba5b479963ae78b873.zip
Improve redundancy: add 'min_lifetime' configuration
Diffstat (limited to 'server')
-rw-r--r--server/helpers/ffmpeg-utils.ts2
-rw-r--r--server/initializers/checker-after-init.ts115
-rw-r--r--server/initializers/checker-before-init.ts (renamed from server/initializers/checker.ts)110
-rw-r--r--server/initializers/constants.ts11
-rw-r--r--server/initializers/index.ts1
-rw-r--r--server/initializers/installer.ts2
-rw-r--r--server/lib/activitypub/audience.ts7
-rw-r--r--server/lib/activitypub/cache-file.ts22
-rw-r--r--server/lib/activitypub/process/process-create.ts8
-rw-r--r--server/lib/activitypub/process/process-undo.ts2
-rw-r--r--server/lib/activitypub/process/process-update.ts28
-rw-r--r--server/lib/activitypub/send/send-update.ts4
-rw-r--r--server/lib/activitypub/videos.ts50
-rw-r--r--server/lib/cache/index.ts1
-rw-r--r--server/lib/redundancy.ts3
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts86
-rw-r--r--server/models/activitypub/actor.ts31
-rw-r--r--server/models/redundancy/video-redundancy.ts64
-rw-r--r--server/models/video/video-file.ts6
-rw-r--r--server/tests/api/server/redundancy.ts264
-rw-r--r--server/tests/utils/server/servers.ts4
21 files changed, 553 insertions, 268 deletions
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 7c45f3632..22bc25476 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -4,7 +4,7 @@ import { VideoResolution } from '../../shared/models/videos'
4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers' 4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
6import { logger } from './logger' 6import { logger } from './logger'
7import { checkFFmpegEncoders } from '../initializers/checker' 7import { checkFFmpegEncoders } from '../initializers/checker-before-init'
8import { remove } from 'fs-extra' 8import { remove } from 'fs-extra'
9 9
10function computeResolutionsToTranscode (videoFileHeight: number) { 10function computeResolutionsToTranscode (videoFileHeight: number) {
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
new file mode 100644
index 000000000..588526235
--- /dev/null
+++ b/server/initializers/checker-after-init.ts
@@ -0,0 +1,115 @@
1import * as config from 'config'
2import { promisify0, isProdInstance, parseDuration, isTestInstance } from '../helpers/core-utils'
3import { UserModel } from '../models/account/user'
4import { ApplicationModel } from '../models/application/application'
5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { parse } from 'url'
7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils'
10import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash'
13
14async function checkActivityPubUrls () {
15 const actor = await getServerActor()
16
17 const parsed = parse(actor.url)
18 if (CONFIG.WEBSERVER.HOST !== parsed.host) {
19 const NODE_ENV = config.util.getEnv('NODE_ENV')
20 const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
21
22 logger.warn(
23 'It seems PeerTube was started (and created some data) with another domain name. ' +
24 'This means you will not be able to federate! ' +
25 'Please use %s %s npm run update-host to fix this.',
26 NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
27 NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
28 )
29 }
30}
31
32// Some checks on configuration files
33// Return an error message, or null if everything is okay
34function checkConfig () {
35 const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY
36
37 // NSFW policy
38 {
39 const available = [ 'do_not_list', 'blur', 'display' ]
40 if (available.indexOf(defaultNSFWPolicy) === -1) {
41 return 'NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy
42 }
43 }
44
45 // Redundancies
46 const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
47 if (isArray(redundancyVideos)) {
48 const available = [ 'most-views', 'trending', 'recently-added' ]
49 for (const r of redundancyVideos) {
50 if (available.indexOf(r.strategy) === -1) {
51 return 'Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy
52 }
53
54 // Lifetime should not be < 10 hours
55 if (!isTestInstance() && r.minLifetime < 1000 * 3600 * 10) {
56 return 'Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy
57 }
58 }
59
60 const filtered = uniq(redundancyVideos.map(r => r.strategy))
61 if (filtered.length !== redundancyVideos.length) {
62 return 'Redundancy video entries should have unique strategies'
63 }
64
65 const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
66 if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
67 return 'Min views in recently added strategy is not a number'
68 }
69 }
70
71 if (isProdInstance()) {
72 const configStorage = config.get('storage')
73 for (const key of Object.keys(configStorage)) {
74 if (configStorage[key].startsWith('storage/')) {
75 logger.warn(
76 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
77 key
78 )
79 }
80 }
81 }
82
83 return null
84}
85
86// We get db by param to not import it in this file (import orders)
87async function clientsExist () {
88 const totalClients = await OAuthClientModel.countTotal()
89
90 return totalClients !== 0
91}
92
93// We get db by param to not import it in this file (import orders)
94async function usersExist () {
95 const totalUsers = await UserModel.countTotal()
96
97 return totalUsers !== 0
98}
99
100// We get db by param to not import it in this file (import orders)
101async function applicationExist () {
102 const totalApplication = await ApplicationModel.countTotal()
103
104 return totalApplication !== 0
105}
106
107// ---------------------------------------------------------------------------
108
109export {
110 checkConfig,
111 clientsExist,
112 usersExist,
113 applicationExist,
114 checkActivityPubUrls
115}
diff --git a/server/initializers/checker.ts b/server/initializers/checker-before-init.ts
index 5b068caa1..4f46d406a 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker-before-init.ts
@@ -1,78 +1,8 @@
1import * as config from 'config' 1import * as config from 'config'
2import { promisify0, isProdInstance } from '../helpers/core-utils' 2import { promisify0 } from '../helpers/core-utils'
3import { UserModel } from '../models/account/user'
4import { ApplicationModel } from '../models/application/application'
5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { parse } from 'url'
7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils'
10import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc' 3import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash'
13
14async function checkActivityPubUrls () {
15 const actor = await getServerActor()
16
17 const parsed = parse(actor.url)
18 if (CONFIG.WEBSERVER.HOST !== parsed.host) {
19 const NODE_ENV = config.util.getEnv('NODE_ENV')
20 const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
21
22 logger.warn(
23 'It seems PeerTube was started (and created some data) with another domain name. ' +
24 'This means you will not be able to federate! ' +
25 'Please use %s %s npm run update-host to fix this.',
26 NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
27 NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
28 )
29 }
30}
31 4
32// Some checks on configuration files 5// ONLY USE CORE MODULES IN THIS FILE!
33// Return an error message, or null if everything is okay
34function checkConfig () {
35 const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
36
37 // NSFW policy
38 if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
39 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
40 }
41
42 // Redundancies
43 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
44 if (isArray(redundancyVideos)) {
45 for (const r of redundancyVideos) {
46 if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
47 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
48 }
49 }
50
51 const filtered = uniq(redundancyVideos.map(r => r.strategy))
52 if (filtered.length !== redundancyVideos.length) {
53 return 'Redundancy video entries should have unique strategies'
54 }
55
56 const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
57 if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
58 return 'Min views in recently added strategy is not a number'
59 }
60 }
61
62 if (isProdInstance()) {
63 const configStorage = config.get('storage')
64 for (const key of Object.keys(configStorage)) {
65 if (configStorage[key].startsWith('storage/')) {
66 logger.warn(
67 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
68 key
69 )
70 }
71 }
72 }
73
74 return null
75}
76 6
77// Check the config files 7// Check the config files
78function checkMissedConfig () { 8function checkMissedConfig () {
@@ -109,6 +39,14 @@ function checkMissedConfig () {
109 } 39 }
110 } 40 }
111 41
42 const redundancyVideos = config.get<any>('redundancy.videos.strategies')
43 if (isArray(redundancyVideos)) {
44 for (const r of redundancyVideos) {
45 if (!r.size) miss.push('redundancy.videos.strategies.size')
46 if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime')
47 }
48 }
49
112 const missingAlternatives = requiredAlternatives.filter( 50 const missingAlternatives = requiredAlternatives.filter(
113 set => !set.find(alternative => !alternative.find(key => !config.has(key))) 51 set => !set.find(alternative => !alternative.find(key => !config.has(key)))
114 ) 52 )
@@ -163,36 +101,10 @@ async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
163 } 101 }
164} 102}
165 103
166// We get db by param to not import it in this file (import orders)
167async function clientsExist () {
168 const totalClients = await OAuthClientModel.countTotal()
169
170 return totalClients !== 0
171}
172
173// We get db by param to not import it in this file (import orders)
174async function usersExist () {
175 const totalUsers = await UserModel.countTotal()
176
177 return totalUsers !== 0
178}
179
180// We get db by param to not import it in this file (import orders)
181async function applicationExist () {
182 const totalApplication = await ApplicationModel.countTotal()
183
184 return totalApplication !== 0
185}
186
187// --------------------------------------------------------------------------- 104// ---------------------------------------------------------------------------
188 105
189export { 106export {
190 checkConfig,
191 checkFFmpeg, 107 checkFFmpeg,
192 checkFFmpegEncoders, 108 checkFFmpegEncoders,
193 checkMissedConfig, 109 checkMissedConfig
194 clientsExist,
195 usersExist,
196 applicationExist,
197 checkActivityPubUrls
198} 110}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 03424ffb8..947cbda28 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -601,7 +601,6 @@ const MEMOIZE_TTL = {
601 601
602const REDUNDANCY = { 602const REDUNDANCY = {
603 VIDEOS: { 603 VIDEOS: {
604 EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
605 RANDOMIZED_FACTOR: 5 604 RANDOMIZED_FACTOR: 5
606 } 605 }
607} 606}
@@ -750,10 +749,16 @@ function updateWebserverConfig () {
750 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) 749 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
751} 750}
752 751
753function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] { 752function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
754 if (!objs) return [] 753 if (!objs) return []
755 754
756 return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) })) 755 return objs.map(obj => {
756 return Object.assign(obj, {
757 minLifetime: parseDuration(obj.min_lifetime),
758 size: bytes.parse(obj.size),
759 minViews: obj.min_views
760 })
761 })
757} 762}
758 763
759function buildLanguages () { 764function buildLanguages () {
diff --git a/server/initializers/index.ts b/server/initializers/index.ts
index 332702774..fe9190a9c 100644
--- a/server/initializers/index.ts
+++ b/server/initializers/index.ts
@@ -1,6 +1,5 @@
1// Constants first, database in second! 1// Constants first, database in second!
2export * from './constants' 2export * from './constants'
3export * from './database' 3export * from './database'
4export * from './checker'
5export * from './installer' 4export * from './installer'
6export * from './migrator' 5export * from './migrator'
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index 818bb04a2..c952ad46c 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -5,7 +5,7 @@ import { createApplicationActor, createUserAccountAndChannel } from '../lib/user
5import { UserModel } from '../models/account/user' 5import { UserModel } from '../models/account/user'
6import { ApplicationModel } from '../models/application/application' 6import { ApplicationModel } from '../models/application/application'
7import { OAuthClientModel } from '../models/oauth/oauth-client' 7import { OAuthClientModel } from '../models/oauth/oauth-client'
8import { applicationExist, clientsExist, usersExist } from './checker' 8import { applicationExist, clientsExist, usersExist } from './checker-after-init'
9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' 9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants'
10import { sequelizeTypescript } from './database' 10import { sequelizeTypescript } from './database'
11import { remove, ensureDir } from 'fs-extra' 11import { remove, ensureDir } from 'fs-extra'
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index a86428461..10277eca7 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -50,7 +50,12 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): Acti
50 50
51async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) { 51async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) {
52 const actors = await VideoShareModel.loadActorsByShare(video.id, t) 52 const actors = await VideoShareModel.loadActorsByShare(video.id, t)
53 actors.push(video.VideoChannel.Account.Actor) 53
54 const videoActor = video.VideoChannel && video.VideoChannel.Account
55 ? video.VideoChannel.Account.Actor
56 : await ActorModel.loadAccountActorByVideoId(video.id, t)
57
58 actors.push(videoActor)
54 59
55 return actors 60 return actors
56} 61}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 87f8a4162..5286d8e6d 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,7 +1,7 @@
1import { CacheFileObject } from '../../../shared/index' 1import { CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { sequelizeTypescript } from '../../initializers'
4import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize'
5 5
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url 7 const url = cacheFileObject.url
@@ -22,25 +22,29 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
22 } 22 }
23} 23}
24 24
25function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 25function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }, t: Transaction) {
26 return sequelizeTypescript.transaction(async t => { 26 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
27 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
28 27
29 return VideoRedundancyModel.create(attributes, { transaction: t }) 28 return VideoRedundancyModel.create(attributes, { transaction: t })
30 })
31} 29}
32 30
33function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) { 31function updateCacheFile (
32 cacheFileObject: CacheFileObject,
33 redundancyModel: VideoRedundancyModel,
34 video: VideoModel,
35 byActor: { id?: number },
36 t: Transaction
37) {
34 if (redundancyModel.actorId !== byActor.id) { 38 if (redundancyModel.actorId !== byActor.id) {
35 throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') 39 throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
36 } 40 }
37 41
38 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) 42 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
39 43
40 redundancyModel.set('expires', attributes.expiresOn) 44 redundancyModel.set('expires', attributes.expiresOn)
41 redundancyModel.set('fileUrl', attributes.fileUrl) 45 redundancyModel.set('fileUrl', attributes.fileUrl)
42 46
43 return redundancyModel.save() 47 return redundancyModel.save({ transaction: t })
44} 48}
45 49
46export { 50export {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index cff8dcfc6..ceb5413ca 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -95,7 +95,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
95 if (video.isOwned()) { 95 if (video.isOwned()) {
96 // Don't resend the activity to the sender 96 // Don't resend the activity to the sender
97 const exceptions = [ byActor ] 97 const exceptions = [ byActor ]
98 await forwardActivity(activity, undefined, exceptions) 98 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
99 } 99 }
100} 100}
101 101
@@ -104,12 +104,14 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate)
104 104
105 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 105 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
106 106
107 await createCacheFile(cacheFile, video, byActor) 107 await sequelizeTypescript.transaction(async t => {
108 return createCacheFile(cacheFile, video, byActor, t)
109 })
108 110
109 if (video.isOwned()) { 111 if (video.isOwned()) {
110 // Don't resend the activity to the sender 112 // Don't resend the activity to the sender
111 const exceptions = [ byActor ] 113 const exceptions = [ byActor ]
112 await forwardActivity(activity, undefined, exceptions) 114 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
113 } 115 }
114} 116}
115 117
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 73ca0a17c..ff019cd8c 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -100,7 +100,7 @@ async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo
100 100
101 return sequelizeTypescript.transaction(async t => { 101 return sequelizeTypescript.transaction(async t => {
102 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 102 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
103 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) 103 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFileObject.id)
104 104
105 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') 105 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
106 106
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index ed3489ebf..e092a6729 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -12,6 +12,7 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-vali
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
14import { createCacheFile, updateCacheFile } from '../cache-file' 14import { createCacheFile, updateCacheFile } from '../cache-file'
15import { forwardVideoRelatedActivity } from '../send/utils'
15 16
16async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { 17async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
17 const objectType = activity.object.type 18 const objectType = activity.object.type
@@ -68,18 +69,29 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
68async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { 69async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
69 const cacheFileObject = activity.object as CacheFileObject 70 const cacheFileObject = activity.object as CacheFileObject
70 71
71 if (!isCacheFileObjectValid(cacheFileObject) === false) { 72 if (!isCacheFileObjectValid(cacheFileObject)) {
72 logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject }) 73 logger.debug('Cache file object sent by update is not valid.', { cacheFileObject })
73 return undefined 74 return undefined
74 } 75 }
75 76
76 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 77 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
77 if (!redundancyModel) { 78
78 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) 79 await sequelizeTypescript.transaction(async t => {
79 return createCacheFile(cacheFileObject, video, byActor) 80 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
80 } 81
82 if (!redundancyModel) {
83 await createCacheFile(cacheFileObject, video, byActor, t)
84 } else {
85 await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
86 }
87 })
88
89 if (video.isOwned()) {
90 // Don't resend the activity to the sender
91 const exceptions = [ byActor ]
81 92
82 return updateCacheFile(cacheFileObject, redundancyModel, byActor) 93 await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
94 }
83} 95}
84 96
85async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { 97async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index ec46789b7..a68f03edf 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video'
7import { VideoChannelModel } from '../../../models/video/video-channel' 7import { VideoChannelModel } from '../../../models/video/video-channel'
8import { VideoShareModel } from '../../../models/video/video-share' 8import { VideoShareModel } from '../../../models/video/video-share'
9import { getUpdateActivityPubUrl } from '../url' 9import { getUpdateActivityPubUrl } from '../url'
10import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' 10import { broadcastToFollowers, sendVideoRelatedActivity } from './utils'
11import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' 11import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience'
12import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
13import { VideoCaptionModel } from '../../../models/video/video-caption' 13import { VideoCaptionModel } from '../../../models/video/video-caption'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 48c0e0a5c..db72ef23c 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -176,7 +176,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
176 syncParam, 176 syncParam,
177 refreshViews 177 refreshViews
178 } 178 }
179 const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) 179 const p = refreshVideoIfNeeded(refreshOptions)
180 if (syncParam.refreshVideo === true) videoFromDatabase = await p 180 if (syncParam.refreshVideo === true) videoFromDatabase = await p
181 181
182 return { video: videoFromDatabase } 182 return { video: videoFromDatabase }
@@ -245,29 +245,37 @@ async function updateVideoFromAP (options: {
245 generateThumbnailFromUrl(options.video, options.videoObject.icon) 245 generateThumbnailFromUrl(options.video, options.videoObject.icon)
246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) 246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
247 247
248 // Remove old video files 248 {
249 const videoFileDestroyTasks: Bluebird<void>[] = [] 249 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
250 for (const videoFile of options.video.VideoFiles) { 250 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
251 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
252 }
253 await Promise.all(videoFileDestroyTasks)
254 251
255 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) 252 // Remove video files that do not exist anymore
256 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) 253 const destroyTasks = options.video.VideoFiles
257 await Promise.all(tasks) 254 .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
255 .map(f => f.destroy(sequelizeOptions))
256 await Promise.all(destroyTasks)
258 257
259 // Update Tags 258 // Update or add other one
260 const tags = options.videoObject.tag.map(tag => tag.name) 259 const upsertTasks = videoFileAttributes.map(a => VideoFileModel.upsert(a, sequelizeOptions))
261 const tagInstances = await TagModel.findOrCreateTags(tags, t) 260 await Promise.all(upsertTasks)
262 await options.video.$set('Tags', tagInstances, sequelizeOptions) 261 }
263 262
264 // Update captions 263 {
265 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) 264 // Update Tags
265 const tags = options.videoObject.tag.map(tag => tag.name)
266 const tagInstances = await TagModel.findOrCreateTags(tags, t)
267 await options.video.$set('Tags', tagInstances, sequelizeOptions)
268 }
266 269
267 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { 270 {
268 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) 271 // Update captions
269 }) 272 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
270 await Promise.all(videoCaptionsPromises) 273
274 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
275 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
276 })
277 await Promise.all(videoCaptionsPromises)
278 }
271 }) 279 })
272 280
273 logger.info('Remote video with uuid %s updated', options.videoObject.uuid) 281 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
@@ -382,7 +390,7 @@ async function refreshVideoIfNeeded (options: {
382 channel: channelActor.VideoChannel, 390 channel: channelActor.VideoChannel,
383 updateViews: options.refreshViews 391 updateViews: options.refreshViews
384 } 392 }
385 await updateVideoFromAP(updateOptions) 393 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
386 await syncVideoExternalAttributes(video, videoObject, options.syncParam) 394 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
387 } catch (err) { 395 } catch (err) {
388 logger.warn('Cannot refresh video.', { err }) 396 logger.warn('Cannot refresh video.', { err })
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts
index 7bf63790a..54eb983fa 100644
--- a/server/lib/cache/index.ts
+++ b/server/lib/cache/index.ts
@@ -1 +1,2 @@
1export * from './videos-preview-cache' 1export * from './videos-preview-cache'
2export * from './videos-caption-cache'
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 78221cc3d..16b122658 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -6,7 +6,8 @@ import { getServerActor } from '../helpers/utils'
6async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) { 6async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) {
7 const serverActor = await getServerActor() 7 const serverActor = await getServerActor()
8 8
9 await sendUndoCacheFile(serverActor, videoRedundancy, t) 9 // Local cache, send undo to remote instances
10 if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t)
10 11
11 await videoRedundancy.destroy({ transaction: t }) 12 await videoRedundancy.destroy({ transaction: t })
12} 13}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 998d2295a..97df3e4f5 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 7import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
@@ -12,6 +12,7 @@ import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { VideoModel } from '../../models/video/video' 12import { VideoModel } from '../../models/video/video'
13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
14import { isTestInstance } from '../../helpers/core-utils' 14import { isTestInstance } from '../../helpers/core-utils'
15import { removeVideoRedundancy } from '../redundancy'
15 16
16export class VideosRedundancyScheduler extends AbstractScheduler { 17export class VideosRedundancyScheduler extends AbstractScheduler {
17 18
@@ -30,7 +31,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
30 this.executing = true 31 this.executing = true
31 32
32 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 33 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
33 if (!isTestInstance()) logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) 34 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
34 35
35 try { 36 try {
36 const videoToDuplicate = await this.findVideoToDuplicate(obj) 37 const videoToDuplicate = await this.findVideoToDuplicate(obj)
@@ -39,20 +40,24 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
39 const videoFiles = videoToDuplicate.VideoFiles 40 const videoFiles = videoToDuplicate.VideoFiles
40 videoFiles.forEach(f => f.Video = videoToDuplicate) 41 videoFiles.forEach(f => f.Video = videoToDuplicate)
41 42
42 if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { 43 await this.purgeCacheIfNeeded(obj, videoFiles)
43 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 44
45 if (await this.isTooHeavy(obj, videoFiles)) {
46 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
44 continue 47 continue
45 } 48 }
46 49
47 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 50 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
48 51
49 await this.createVideoRedundancy(obj.strategy, videoFiles) 52 await this.createVideoRedundancy(obj, videoFiles)
50 } catch (err) { 53 } catch (err) {
51 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 54 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
52 } 55 }
53 } 56 }
54 57
55 await this.removeExpired() 58 await this.extendsLocalExpiration()
59
60 await this.purgeRemoteExpired()
56 61
57 this.executing = false 62 this.executing = false
58 } 63 }
@@ -61,16 +66,27 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
61 return this.instance || (this.instance = new this()) 66 return this.instance || (this.instance = new this())
62 } 67 }
63 68
64 private async removeExpired () { 69 private async extendsLocalExpiration () {
65 const expired = await VideoRedundancyModel.listAllExpired() 70 const expired = await VideoRedundancyModel.listLocalExpired()
71
72 for (const redundancyModel of expired) {
73 try {
74 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
75 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
76 } catch (err) {
77 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel))
78 }
79 }
80 }
66 81
67 for (const m of expired) { 82 private async purgeRemoteExpired () {
68 logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m)) 83 const expired = await VideoRedundancyModel.listRemoteExpired()
69 84
85 for (const redundancyModel of expired) {
70 try { 86 try {
71 await m.destroy() 87 await removeVideoRedundancy(redundancyModel)
72 } catch (err) { 88 } catch (err) {
73 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) 89 logger.error('Cannot remove redundancy %s from our redundancy system.', this.buildEntryLogId(redundancyModel))
74 } 90 }
75 } 91 }
76 } 92 }
@@ -90,18 +106,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
90 } 106 }
91 } 107 }
92 108
93 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 109 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
94 const serverActor = await getServerActor() 110 const serverActor = await getServerActor()
95 111
96 for (const file of filesToDuplicate) { 112 for (const file of filesToDuplicate) {
97 const existing = await VideoRedundancyModel.loadByFileId(file.id) 113 const existing = await VideoRedundancyModel.loadByFileId(file.id)
98 if (existing) { 114 if (existing) {
99 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy) 115 await this.extendsExpirationOf(existing, redundancy.minLifetime)
100 116
101 existing.expiresOn = this.buildNewExpiration()
102 await existing.save()
103
104 await sendUpdateCacheFile(serverActor, existing)
105 continue 117 continue
106 } 118 }
107 119
@@ -109,7 +121,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
109 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) 121 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
110 if (!video) continue 122 if (!video) continue
111 123
112 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) 124 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
113 125
114 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 126 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
115 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 127 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
@@ -120,10 +132,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
120 await rename(tmpPath, destPath) 132 await rename(tmpPath, destPath)
121 133
122 const createdModel = await VideoRedundancyModel.create({ 134 const createdModel = await VideoRedundancyModel.create({
123 expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS), 135 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
124 url: getVideoCacheFileActivityPubUrl(file), 136 url: getVideoCacheFileActivityPubUrl(file),
125 fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), 137 fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
126 strategy, 138 strategy: redundancy.strategy,
127 videoFileId: file.id, 139 videoFileId: file.id,
128 actorId: serverActor.id 140 actorId: serverActor.id
129 }) 141 })
@@ -133,16 +145,36 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
133 } 145 }
134 } 146 }
135 147
136 private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { 148 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
137 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) 149 logger.info('Extending expiration of %s.', redundancy.url)
150
151 const serverActor = await getServerActor()
152
153 redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs)
154 await redundancy.save()
155
156 await sendUpdateCacheFile(serverActor, redundancy)
157 }
158
159 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
160 while (this.isTooHeavy(redundancy, filesToDuplicate)) {
161 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
162 if (!toDelete) return
163
164 await removeVideoRedundancy(toDelete)
165 }
166 }
167
168 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
169 const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate)
138 170
139 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) 171 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
140 172
141 return totalDuplicated > maxSize 173 return totalDuplicated > maxSize
142 } 174 }
143 175
144 private buildNewExpiration () { 176 private buildNewExpiration (expiresAfterMs: number) {
145 return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS) 177 return new Date(Date.now() + expiresAfterMs)
146 } 178 }
147 179
148 private buildEntryLogId (object: VideoRedundancyModel) { 180 private buildEntryLogId (object: VideoRedundancyModel) {
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index f8bb59323..12b83916e 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -37,6 +37,7 @@ import { ServerModel } from '../server/server'
37import { throwIfNotValid } from '../utils' 37import { throwIfNotValid } from '../utils'
38import { VideoChannelModel } from '../video/video-channel' 38import { VideoChannelModel } from '../video/video-channel'
39import { ActorFollowModel } from './actor-follow' 39import { ActorFollowModel } from './actor-follow'
40import { VideoModel } from '../video/video'
40 41
41enum ScopeNames { 42enum ScopeNames {
42 FULL = 'FULL' 43 FULL = 'FULL'
@@ -266,6 +267,36 @@ export class ActorModel extends Model<ActorModel> {
266 return ActorModel.unscoped().findById(id) 267 return ActorModel.unscoped().findById(id)
267 } 268 }
268 269
270 static loadAccountActorByVideoId (videoId: number, transaction: Sequelize.Transaction) {
271 const query = {
272 include: [
273 {
274 attributes: [ 'id' ],
275 model: AccountModel.unscoped(),
276 required: true,
277 include: [
278 {
279 attributes: [ 'id' ],
280 model: VideoChannelModel.unscoped(),
281 required: true,
282 include: {
283 attributes: [ 'id' ],
284 model: VideoModel.unscoped(),
285 required: true,
286 where: {
287 id: videoId
288 }
289 }
290 }
291 ]
292 }
293 ],
294 transaction
295 }
296
297 return ActorModel.unscoped().findOne(query as any) // FIXME: typings
298 }
299
269 static isActorUrlExist (url: string) { 300 static isActorUrlExist (url: string) {
270 const query = { 301 const query = {
271 raw: true, 302 raw: true,
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index fb07287a8..970d2fe06 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -9,7 +9,6 @@ import {
9 Is, 9 Is,
10 Model, 10 Model,
11 Scopes, 11 Scopes,
12 Sequelize,
13 Table, 12 Table,
14 UpdatedAt 13 UpdatedAt
15} from 'sequelize-typescript' 14} from 'sequelize-typescript'
@@ -28,6 +27,7 @@ import { ServerModel } from '../server/server'
28import { sample } from 'lodash' 27import { sample } from 'lodash'
29import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
30import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import * as Sequelize from 'sequelize'
31 31
32export enum ScopeNames { 32export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO' 33 WITH_VIDEO = 'WITH_VIDEO'
@@ -116,11 +116,11 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
116 Actor: ActorModel 116 Actor: ActorModel
117 117
118 @AfterDestroy 118 @AfterDestroy
119 static removeFilesAndSendDelete (instance: VideoRedundancyModel) { 119 static removeFile (instance: VideoRedundancyModel) {
120 // Not us 120 // Not us
121 if (!instance.strategy) return 121 if (!instance.strategy) return
122 122
123 logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) 123 logger.info('Removing duplicated video file %s-%s.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
124 124
125 return instance.VideoFile.Video.removeFile(instance.VideoFile) 125 return instance.VideoFile.Video.removeFile(instance.VideoFile)
126 } 126 }
@@ -135,11 +135,12 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
135 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 135 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
136 } 136 }
137 137
138 static loadByUrl (url: string) { 138 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
139 const query = { 139 const query = {
140 where: { 140 where: {
141 url 141 url
142 } 142 },
143 transaction
143 } 144 }
144 145
145 return VideoRedundancyModel.findOne(query) 146 return VideoRedundancyModel.findOne(query)
@@ -157,7 +158,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
157 // On VideoModel! 158 // On VideoModel!
158 const query = { 159 const query = {
159 attributes: [ 'id', 'views' ], 160 attributes: [ 'id', 'views' ],
160 logging: !isTestInstance(),
161 limit: randomizedFactor, 161 limit: randomizedFactor,
162 order: getVideoSort('-views'), 162 order: getVideoSort('-views'),
163 include: [ 163 include: [
@@ -174,7 +174,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
174 const query = { 174 const query = {
175 attributes: [ 'id', 'views' ], 175 attributes: [ 'id', 'views' ],
176 subQuery: false, 176 subQuery: false,
177 logging: !isTestInstance(),
178 group: 'VideoModel.id', 177 group: 'VideoModel.id',
179 limit: randomizedFactor, 178 limit: randomizedFactor,
180 order: getVideoSort('-trending'), 179 order: getVideoSort('-trending'),
@@ -193,7 +192,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
193 // On VideoModel! 192 // On VideoModel!
194 const query = { 193 const query = {
195 attributes: [ 'id', 'publishedAt' ], 194 attributes: [ 'id', 'publishedAt' ],
196 logging: !isTestInstance(),
197 limit: randomizedFactor, 195 limit: randomizedFactor,
198 order: getVideoSort('-publishedAt'), 196 order: getVideoSort('-publishedAt'),
199 where: { 197 where: {
@@ -210,11 +208,29 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
210 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) 208 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
211 } 209 }
212 210
211 static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
212 const expiredDate = new Date()
213 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
214
215 const actor = await getServerActor()
216
217 const query = {
218 where: {
219 actorId: actor.id,
220 strategy,
221 createdAt: {
222 [ Sequelize.Op.lt ]: expiredDate
223 }
224 }
225 }
226
227 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
228 }
229
213 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { 230 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
214 const actor = await getServerActor() 231 const actor = await getServerActor()
215 232
216 const options = { 233 const options = {
217 logging: !isTestInstance(),
218 include: [ 234 include: [
219 { 235 {
220 attributes: [], 236 attributes: [],
@@ -228,21 +244,39 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
228 ] 244 ]
229 } 245 }
230 246
231 return VideoFileModel.sum('size', options) 247 return VideoFileModel.sum('size', options as any) // FIXME: typings
232 } 248 }
233 249
234 static listAllExpired () { 250 static async listLocalExpired () {
251 const actor = await getServerActor()
252
253 const query = {
254 where: {
255 actorId: actor.id,
256 expiresOn: {
257 [ Sequelize.Op.lt ]: new Date()
258 }
259 }
260 }
261
262 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
263 }
264
265 static async listRemoteExpired () {
266 const actor = await getServerActor()
267
235 const query = { 268 const query = {
236 logging: !isTestInstance(),
237 where: { 269 where: {
270 actorId: {
271 [Sequelize.Op.ne]: actor.id
272 },
238 expiresOn: { 273 expiresOn: {
239 [ Sequelize.Op.lt ]: new Date() 274 [ Sequelize.Op.lt ]: new Date()
240 } 275 }
241 } 276 }
242 } 277 }
243 278
244 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) 279 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
245 .findAll(query)
246 } 280 }
247 281
248 static async getStats (strategy: VideoRedundancyStrategy) { 282 static async getStats (strategy: VideoRedundancyStrategy) {
@@ -299,7 +333,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
299 333
300 const notIn = Sequelize.literal( 334 const notIn = Sequelize.literal(
301 '(' + 335 '(' +
302 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + 336 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
303 ')' 337 ')'
304 ) 338 )
305 339
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 0907ea569..0887a3738 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -106,4 +106,10 @@ export class VideoFileModel extends Model<VideoFileModel> {
106 return results.length === 1 106 return results.length === 1
107 }) 107 })
108 } 108 }
109
110 hasSameUniqueKeysThan (other: VideoFileModel) {
111 return this.fps === other.fps &&
112 this.resolution === other.resolution &&
113 this.videoId === other.videoId
114 }
109} 115}
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 6ce4b9dd1..a773e3de4 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -31,14 +31,13 @@ const expect = chai.expect
31 31
32let servers: ServerInfo[] = [] 32let servers: ServerInfo[] = []
33let video1Server2UUID: string 33let video1Server2UUID: string
34let video2Server2UUID: string
35 34
36function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { 35function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
37 const parsed = magnetUtil.decode(file.magnetUri) 36 const parsed = magnetUtil.decode(file.magnetUri)
38 37
39 for (const ws of baseWebseeds) { 38 for (const ws of baseWebseeds) {
40 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) 39 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
41 expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined 40 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
42 } 41 }
43} 42}
44 43
@@ -49,6 +48,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
49 check_interval: '5 seconds', 48 check_interval: '5 seconds',
50 strategies: [ 49 strategies: [
51 immutableAssign({ 50 immutableAssign({
51 min_lifetime: '1 hour',
52 strategy: strategy, 52 strategy: strategy,
53 size: '100KB' 53 size: '100KB'
54 }, additionalParams) 54 }, additionalParams)
@@ -68,11 +68,6 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
68 await viewVideo(servers[ 1 ].url, video1Server2UUID) 68 await viewVideo(servers[ 1 ].url, video1Server2UUID)
69 } 69 }
70 70
71 {
72 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
73 video2Server2UUID = res.body.video.uuid
74 }
75
76 await waitJobs(servers) 71 await waitJobs(servers)
77 72
78 // Server 1 and server 2 follow each other 73 // Server 1 and server 2 follow each other
@@ -85,68 +80,69 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
85 await waitJobs(servers) 80 await waitJobs(servers)
86} 81}
87 82
88async function check1WebSeed (strategy: VideoRedundancyStrategy) { 83async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
84 if (!videoUUID) videoUUID = video1Server2UUID
85
89 const webseeds = [ 86 const webseeds = [
90 'http://localhost:9002/static/webseed/' + video1Server2UUID 87 'http://localhost:9002/static/webseed/' + videoUUID
91 ] 88 ]
92 89
93 for (const server of servers) { 90 for (const server of servers) {
94 { 91 {
95 const res = await getVideo(server.url, video1Server2UUID) 92 const res = await getVideo(server.url, videoUUID)
96 93
97 const video: VideoDetails = res.body 94 const video: VideoDetails = res.body
98 video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) 95 for (const f of video.files) {
96 checkMagnetWebseeds(f, webseeds, server)
97 }
99 } 98 }
99 }
100}
100 101
101 { 102async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
102 const res = await getStats(server.url) 103 const res = await getStats(servers[0].url)
103 const data: ServerStats = res.body 104 const data: ServerStats = res.body
104 105
105 expect(data.videosRedundancy).to.have.lengthOf(1) 106 expect(data.videosRedundancy).to.have.lengthOf(1)
107 const stat = data.videosRedundancy[0]
106 108
107 const stat = data.videosRedundancy[0] 109 expect(stat.strategy).to.equal(strategy)
108 expect(stat.strategy).to.equal(strategy) 110 expect(stat.totalSize).to.equal(102400)
109 expect(stat.totalSize).to.equal(102400) 111 expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
110 expect(stat.totalUsed).to.equal(0) 112 expect(stat.totalVideoFiles).to.equal(4)
111 expect(stat.totalVideoFiles).to.equal(0) 113 expect(stat.totalVideos).to.equal(1)
112 expect(stat.totalVideos).to.equal(0)
113 }
114 }
115} 114}
116 115
117async function enableRedundancy () { 116async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
118 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) 117 const res = await getStats(servers[0].url)
118 const data: ServerStats = res.body
119 119
120 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') 120 expect(data.videosRedundancy).to.have.lengthOf(1)
121 const follows: ActorFollow[] = res.body.data
122 const server2 = follows.find(f => f.following.host === 'localhost:9002')
123 const server3 = follows.find(f => f.following.host === 'localhost:9003')
124
125 expect(server3).to.not.be.undefined
126 expect(server3.following.hostRedundancyAllowed).to.be.false
127 121
128 expect(server2).to.not.be.undefined 122 const stat = data.videosRedundancy[0]
129 expect(server2.following.hostRedundancyAllowed).to.be.true 123 expect(stat.strategy).to.equal(strategy)
124 expect(stat.totalSize).to.equal(102400)
125 expect(stat.totalUsed).to.equal(0)
126 expect(stat.totalVideoFiles).to.equal(0)
127 expect(stat.totalVideos).to.equal(0)
130} 128}
131 129
132async function check2Webseeds (strategy: VideoRedundancyStrategy) { 130async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
133 await waitJobs(servers) 131 if (!videoUUID) videoUUID = video1Server2UUID
134 await wait(15000)
135 await waitJobs(servers)
136 132
137 const webseeds = [ 133 const webseeds = [
138 'http://localhost:9001/static/webseed/' + video1Server2UUID, 134 'http://localhost:9001/static/webseed/' + videoUUID,
139 'http://localhost:9002/static/webseed/' + video1Server2UUID 135 'http://localhost:9002/static/webseed/' + videoUUID
140 ] 136 ]
141 137
142 for (const server of servers) { 138 for (const server of servers) {
143 { 139 {
144 const res = await getVideo(server.url, video1Server2UUID) 140 const res = await getVideo(server.url, videoUUID)
145 141
146 const video: VideoDetails = res.body 142 const video: VideoDetails = res.body
147 143
148 for (const file of video.files) { 144 for (const file of video.files) {
149 checkMagnetWebseeds(file, webseeds) 145 checkMagnetWebseeds(file, webseeds, server)
150 } 146 }
151 } 147 }
152 } 148 }
@@ -155,22 +151,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy) {
155 expect(files).to.have.lengthOf(4) 151 expect(files).to.have.lengthOf(4)
156 152
157 for (const resolution of [ 240, 360, 480, 720 ]) { 153 for (const resolution of [ 240, 360, 480, 720 ]) {
158 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined 154 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
159 } 155 }
156}
160 157
161 { 158async function enableRedundancyOnServer1 () {
162 const res = await getStats(servers[0].url) 159 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
163 const data: ServerStats = res.body
164 160
165 expect(data.videosRedundancy).to.have.lengthOf(1) 161 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
166 const stat = data.videosRedundancy[0] 162 const follows: ActorFollow[] = res.body.data
163 const server2 = follows.find(f => f.following.host === 'localhost:9002')
164 const server3 = follows.find(f => f.following.host === 'localhost:9003')
167 165
168 expect(stat.strategy).to.equal(strategy) 166 expect(server3).to.not.be.undefined
169 expect(stat.totalSize).to.equal(102400) 167 expect(server3.following.hostRedundancyAllowed).to.be.false
170 expect(stat.totalUsed).to.be.at.least(1).and.below(102401) 168
171 expect(stat.totalVideoFiles).to.equal(4) 169 expect(server2).to.not.be.undefined
172 expect(stat.totalVideos).to.equal(1) 170 expect(server2.following.hostRedundancyAllowed).to.be.true
173 }
174} 171}
175 172
176async function cleanServers () { 173async function cleanServers () {
@@ -188,18 +185,24 @@ describe('Test videos redundancy', function () {
188 return runServers(strategy) 185 return runServers(strategy)
189 }) 186 })
190 187
191 it('Should have 1 webseed on the first video', function () { 188 it('Should have 1 webseed on the first video', async function () {
192 return check1WebSeed(strategy) 189 await check1WebSeed(strategy)
190 await checkStatsWith1Webseed(strategy)
193 }) 191 })
194 192
195 it('Should enable redundancy on server 1', function () { 193 it('Should enable redundancy on server 1', function () {
196 return enableRedundancy() 194 return enableRedundancyOnServer1()
197 }) 195 })
198 196
199 it('Should have 2 webseed on the first video', function () { 197 it('Should have 2 webseed on the first video', async function () {
200 this.timeout(40000) 198 this.timeout(40000)
201 199
202 return check2Webseeds(strategy) 200 await waitJobs(servers)
201 await wait(15000)
202 await waitJobs(servers)
203
204 await check2Webseeds(strategy)
205 await checkStatsWith2Webseed(strategy)
203 }) 206 })
204 207
205 after(function () { 208 after(function () {
@@ -216,18 +219,24 @@ describe('Test videos redundancy', function () {
216 return runServers(strategy) 219 return runServers(strategy)
217 }) 220 })
218 221
219 it('Should have 1 webseed on the first video', function () { 222 it('Should have 1 webseed on the first video', async function () {
220 return check1WebSeed(strategy) 223 await check1WebSeed(strategy)
224 await checkStatsWith1Webseed(strategy)
221 }) 225 })
222 226
223 it('Should enable redundancy on server 1', function () { 227 it('Should enable redundancy on server 1', function () {
224 return enableRedundancy() 228 return enableRedundancyOnServer1()
225 }) 229 })
226 230
227 it('Should have 2 webseed on the first video', function () { 231 it('Should have 2 webseed on the first video', async function () {
228 this.timeout(40000) 232 this.timeout(40000)
229 233
230 return check2Webseeds(strategy) 234 await waitJobs(servers)
235 await wait(15000)
236 await waitJobs(servers)
237
238 await check2Webseeds(strategy)
239 await checkStatsWith2Webseed(strategy)
231 }) 240 })
232 241
233 after(function () { 242 after(function () {
@@ -241,15 +250,16 @@ describe('Test videos redundancy', function () {
241 before(function () { 250 before(function () {
242 this.timeout(120000) 251 this.timeout(120000)
243 252
244 return runServers(strategy, { minViews: 3 }) 253 return runServers(strategy, { min_views: 3 })
245 }) 254 })
246 255
247 it('Should have 1 webseed on the first video', function () { 256 it('Should have 1 webseed on the first video', async function () {
248 return check1WebSeed(strategy) 257 await check1WebSeed(strategy)
258 await checkStatsWith1Webseed(strategy)
249 }) 259 })
250 260
251 it('Should enable redundancy on server 1', function () { 261 it('Should enable redundancy on server 1', function () {
252 return enableRedundancy() 262 return enableRedundancyOnServer1()
253 }) 263 })
254 264
255 it('Should still have 1 webseed on the first video', async function () { 265 it('Should still have 1 webseed on the first video', async function () {
@@ -259,10 +269,11 @@ describe('Test videos redundancy', function () {
259 await wait(15000) 269 await wait(15000)
260 await waitJobs(servers) 270 await waitJobs(servers)
261 271
262 return check1WebSeed(strategy) 272 await check1WebSeed(strategy)
273 await checkStatsWith1Webseed(strategy)
263 }) 274 })
264 275
265 it('Should view 2 times the first video', async function () { 276 it('Should view 2 times the first video to have > min_views config', async function () {
266 this.timeout(40000) 277 this.timeout(40000)
267 278
268 await viewVideo(servers[ 0 ].url, video1Server2UUID) 279 await viewVideo(servers[ 0 ].url, video1Server2UUID)
@@ -272,10 +283,117 @@ describe('Test videos redundancy', function () {
272 await waitJobs(servers) 283 await waitJobs(servers)
273 }) 284 })
274 285
275 it('Should have 2 webseed on the first video', function () { 286 it('Should have 2 webseed on the first video', async function () {
276 this.timeout(40000) 287 this.timeout(40000)
277 288
278 return check2Webseeds(strategy) 289 await waitJobs(servers)
290 await wait(15000)
291 await waitJobs(servers)
292
293 await check2Webseeds(strategy)
294 await checkStatsWith2Webseed(strategy)
295 })
296
297 after(function () {
298 return cleanServers()
299 })
300 })
301
302 describe('Test expiration', function () {
303 const strategy = 'recently-added'
304
305 async function checkContains (servers: ServerInfo[], str: string) {
306 for (const server of servers) {
307 const res = await getVideo(server.url, video1Server2UUID)
308 const video: VideoDetails = res.body
309
310 for (const f of video.files) {
311 expect(f.magnetUri).to.contain(str)
312 }
313 }
314 }
315
316 async function checkNotContains (servers: ServerInfo[], str: string) {
317 for (const server of servers) {
318 const res = await getVideo(server.url, video1Server2UUID)
319 const video: VideoDetails = res.body
320
321 for (const f of video.files) {
322 expect(f.magnetUri).to.not.contain(str)
323 }
324 }
325 }
326
327 before(async function () {
328 this.timeout(120000)
329
330 await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
331
332 await enableRedundancyOnServer1()
333 })
334
335 it('Should still have 2 webseeds after 10 seconds', async function () {
336 this.timeout(40000)
337
338 await wait(10000)
339
340 try {
341 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
342 } catch {
343 // Maybe a server deleted a redundancy in the scheduler
344 await wait(2000)
345
346 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
347 }
348 })
349
350 it('Should stop server 1 and expire video redundancy', async function () {
351 this.timeout(40000)
352
353 killallServers([ servers[0] ])
354
355 await wait(10000)
356
357 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
358 })
359
360 after(function () {
361 return killallServers([ servers[1], servers[2] ])
362 })
363 })
364
365 describe('Test file replacement', function () {
366 let video2Server2UUID: string
367 const strategy = 'recently-added'
368
369 before(async function () {
370 this.timeout(120000)
371
372 await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
373
374 await enableRedundancyOnServer1()
375
376 await waitJobs(servers)
377 await wait(5000)
378 await waitJobs(servers)
379
380 await check2Webseeds(strategy)
381 await checkStatsWith2Webseed(strategy)
382
383 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
384 video2Server2UUID = res.body.video.uuid
385 })
386
387 it('Should cache video 2 webseed on the first video', async function () {
388 this.timeout(40000)
389 this.retries(3)
390
391 await waitJobs(servers)
392
393 await wait(7000)
394
395 await check1WebSeed(strategy, video1Server2UUID)
396 await check2Webseeds(strategy, video2Server2UUID)
279 }) 397 })
280 398
281 after(function () { 399 after(function () {
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 26ab4e1bb..fbfc83ca1 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -144,8 +144,8 @@ function runServer (serverNumber: number, configOverride?: Object) {
144 }) 144 })
145} 145}
146 146
147async function reRunServer (server: ServerInfo) { 147async function reRunServer (server: ServerInfo, configOverride?: any) {
148 const newServer = await runServer(server.serverNumber) 148 const newServer = await runServer(server.serverNumber, configOverride)
149 server.app = newServer.app 149 server.app = newServer.app
150 150
151 return server 151 return server