aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/activitypub/client.ts16
-rw-r--r--server/controllers/activitypub/inbox.ts23
-rw-r--r--server/controllers/api/config.ts9
-rw-r--r--server/controllers/api/overviews.ts42
-rw-r--r--server/controllers/api/search.ts7
-rw-r--r--server/controllers/api/server/stats.ts14
-rw-r--r--server/controllers/api/users/index.ts32
-rw-r--r--server/controllers/api/users/me.ts41
-rw-r--r--server/controllers/api/video-channel.ts28
-rw-r--r--server/controllers/api/videos/abuse.ts23
-rw-r--r--server/controllers/api/videos/comment.ts24
-rw-r--r--server/controllers/api/videos/import.ts6
-rw-r--r--server/controllers/api/videos/index.ts12
-rw-r--r--server/controllers/api/videos/ownership.ts13
-rw-r--r--server/controllers/api/videos/rate.ts11
-rw-r--r--server/controllers/client.ts2
-rw-r--r--server/helpers/actor.ts13
-rw-r--r--server/helpers/audit-logger.ts8
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts2
-rw-r--r--server/helpers/custom-validators/video-ownership.ts2
-rw-r--r--server/helpers/custom-validators/videos.ts13
-rw-r--r--server/helpers/utils.ts24
-rw-r--r--server/helpers/video.ts25
-rw-r--r--server/helpers/webfinger.ts5
-rw-r--r--server/helpers/webtorrent.ts2
-rw-r--r--server/helpers/youtube-dl.ts69
-rw-r--r--server/initializers/checker.ts16
-rw-r--r--server/initializers/constants.ts32
-rw-r--r--server/lib/activitypub/actor.ts18
-rw-r--r--server/lib/activitypub/audience.ts10
-rw-r--r--server/lib/activitypub/cache-file.ts11
-rw-r--r--server/lib/activitypub/process/process-accept.ts6
-rw-r--r--server/lib/activitypub/process/process-announce.ts8
-rw-r--r--server/lib/activitypub/process/process-create.ts38
-rw-r--r--server/lib/activitypub/process/process-delete.ts34
-rw-r--r--server/lib/activitypub/process/process-follow.ts8
-rw-r--r--server/lib/activitypub/process/process-like.ts9
-rw-r--r--server/lib/activitypub/process/process-reject.ts12
-rw-r--r--server/lib/activitypub/process/process-undo.ts61
-rw-r--r--server/lib/activitypub/process/process-update.ts31
-rw-r--r--server/lib/activitypub/process/process.ts15
-rw-r--r--server/lib/activitypub/send/send-announce.ts4
-rw-r--r--server/lib/activitypub/send/send-create.ts95
-rw-r--r--server/lib/activitypub/send/send-delete.ts13
-rw-r--r--server/lib/activitypub/send/send-like.ts23
-rw-r--r--server/lib/activitypub/send/send-undo.ts85
-rw-r--r--server/lib/activitypub/send/send-update.ts16
-rw-r--r--server/lib/activitypub/send/utils.ts30
-rw-r--r--server/lib/activitypub/video-comments.ts2
-rw-r--r--server/lib/activitypub/videos.ts531
-rw-r--r--server/lib/avatar.ts11
-rw-r--r--server/lib/cache/videos-caption-cache.ts2
-rw-r--r--server/lib/cache/videos-preview-cache.ts4
-rw-r--r--server/lib/client-html.ts9
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts4
-rw-r--r--server/lib/job-queue/handlers/video-file.ts15
-rw-r--r--server/lib/job-queue/handlers/video-import.ts2
-rw-r--r--server/lib/oauth-model.ts40
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts64
-rw-r--r--server/lib/schedulers/youtube-dl-update-scheduler.ts65
-rw-r--r--server/lib/video-transcoding.ts130
-rw-r--r--server/middlewares/validators/users.ts2
-rw-r--r--server/middlewares/validators/video-captions.ts2
-rw-r--r--server/middlewares/validators/video-comments.ts6
-rw-r--r--server/middlewares/validators/videos.ts66
-rw-r--r--server/models/account/account.ts4
-rw-r--r--server/models/account/user.ts9
-rw-r--r--server/models/activitypub/actor.ts35
-rw-r--r--server/models/oauth/oauth-token.ts47
-rw-r--r--server/models/redundancy/video-redundancy.ts189
-rw-r--r--server/models/video/tag.ts5
-rw-r--r--server/models/video/video-format-utils.ts296
-rw-r--r--server/models/video/video.ts555
-rw-r--r--server/tests/api/server/jobs.ts4
-rw-r--r--server/tests/api/server/redundancy.ts285
-rw-r--r--server/tests/api/server/stats.ts2
-rw-r--r--server/tests/utils/server/servers.ts7
-rw-r--r--server/tests/utils/server/stats.ts7
78 files changed, 1984 insertions, 1457 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 2e168ea78..6229c44aa 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -6,7 +6,13 @@ import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
6import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send' 6import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send'
7import { audiencify, getAudience } from '../../lib/activitypub/audience' 7import { audiencify, getAudience } from '../../lib/activitypub/audience'
8import { buildCreateActivity } from '../../lib/activitypub/send/send-create' 8import { buildCreateActivity } from '../../lib/activitypub/send/send-create'
9import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' 9import {
10 asyncMiddleware,
11 executeIfActivityPub,
12 localAccountValidator,
13 localVideoChannelValidator,
14 videosCustomGetValidator
15} from '../../middlewares'
10import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' 16import { videosGetValidator, videosShareValidator } from '../../middlewares/validators'
11import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' 17import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
12import { AccountModel } from '../../models/account/account' 18import { AccountModel } from '../../models/account/account'
@@ -54,7 +60,7 @@ activityPubClientRouter.get('/videos/watch/:id/activity',
54 executeIfActivityPub(asyncMiddleware(videoController)) 60 executeIfActivityPub(asyncMiddleware(videoController))
55) 61)
56activityPubClientRouter.get('/videos/watch/:id/announces', 62activityPubClientRouter.get('/videos/watch/:id/announces',
57 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 63 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
58 executeIfActivityPub(asyncMiddleware(videoAnnouncesController)) 64 executeIfActivityPub(asyncMiddleware(videoAnnouncesController))
59) 65)
60activityPubClientRouter.get('/videos/watch/:id/announces/:accountId', 66activityPubClientRouter.get('/videos/watch/:id/announces/:accountId',
@@ -62,15 +68,15 @@ activityPubClientRouter.get('/videos/watch/:id/announces/:accountId',
62 executeIfActivityPub(asyncMiddleware(videoAnnounceController)) 68 executeIfActivityPub(asyncMiddleware(videoAnnounceController))
63) 69)
64activityPubClientRouter.get('/videos/watch/:id/likes', 70activityPubClientRouter.get('/videos/watch/:id/likes',
65 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 71 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
66 executeIfActivityPub(asyncMiddleware(videoLikesController)) 72 executeIfActivityPub(asyncMiddleware(videoLikesController))
67) 73)
68activityPubClientRouter.get('/videos/watch/:id/dislikes', 74activityPubClientRouter.get('/videos/watch/:id/dislikes',
69 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 75 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
70 executeIfActivityPub(asyncMiddleware(videoDislikesController)) 76 executeIfActivityPub(asyncMiddleware(videoDislikesController))
71) 77)
72activityPubClientRouter.get('/videos/watch/:id/comments', 78activityPubClientRouter.get('/videos/watch/:id/comments',
73 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 79 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))),
74 executeIfActivityPub(asyncMiddleware(videoCommentsController)) 80 executeIfActivityPub(asyncMiddleware(videoCommentsController))
75) 81)
76activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', 82activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId',
diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts
index 20bd20ed4..738d155eb 100644
--- a/server/controllers/activitypub/inbox.ts
+++ b/server/controllers/activitypub/inbox.ts
@@ -7,6 +7,8 @@ import { asyncMiddleware, checkSignature, localAccountValidator, localVideoChann
7import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' 7import { activityPubValidator } from '../../middlewares/validators/activitypub/activity'
8import { VideoChannelModel } from '../../models/video/video-channel' 8import { VideoChannelModel } from '../../models/video/video-channel'
9import { AccountModel } from '../../models/account/account' 9import { AccountModel } from '../../models/account/account'
10import { queue } from 'async'
11import { ActorModel } from '../../models/activitypub/actor'
10 12
11const inboxRouter = express.Router() 13const inboxRouter = express.Router()
12 14
@@ -14,7 +16,7 @@ inboxRouter.post('/inbox',
14 signatureValidator, 16 signatureValidator,
15 asyncMiddleware(checkSignature), 17 asyncMiddleware(checkSignature),
16 asyncMiddleware(activityPubValidator), 18 asyncMiddleware(activityPubValidator),
17 asyncMiddleware(inboxController) 19 inboxController
18) 20)
19 21
20inboxRouter.post('/accounts/:name/inbox', 22inboxRouter.post('/accounts/:name/inbox',
@@ -22,14 +24,14 @@ inboxRouter.post('/accounts/:name/inbox',
22 asyncMiddleware(checkSignature), 24 asyncMiddleware(checkSignature),
23 asyncMiddleware(localAccountValidator), 25 asyncMiddleware(localAccountValidator),
24 asyncMiddleware(activityPubValidator), 26 asyncMiddleware(activityPubValidator),
25 asyncMiddleware(inboxController) 27 inboxController
26) 28)
27inboxRouter.post('/video-channels/:name/inbox', 29inboxRouter.post('/video-channels/:name/inbox',
28 signatureValidator, 30 signatureValidator,
29 asyncMiddleware(checkSignature), 31 asyncMiddleware(checkSignature),
30 asyncMiddleware(localVideoChannelValidator), 32 asyncMiddleware(localVideoChannelValidator),
31 asyncMiddleware(activityPubValidator), 33 asyncMiddleware(activityPubValidator),
32 asyncMiddleware(inboxController) 34 inboxController
33) 35)
34 36
35// --------------------------------------------------------------------------- 37// ---------------------------------------------------------------------------
@@ -40,7 +42,12 @@ export {
40 42
41// --------------------------------------------------------------------------- 43// ---------------------------------------------------------------------------
42 44
43async function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) { 45const inboxQueue = queue<{ activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel }, Error>((task, cb) => {
46 processActivities(task.activities, task.signatureActor, task.inboxActor)
47 .then(() => cb())
48})
49
50function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) {
44 const rootActivity: RootActivity = req.body 51 const rootActivity: RootActivity = req.body
45 let activities: Activity[] = [] 52 let activities: Activity[] = []
46 53
@@ -66,7 +73,11 @@ async function inboxController (req: express.Request, res: express.Response, nex
66 73
67 logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url) 74 logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url)
68 75
69 await processActivities(activities, res.locals.signature.actor, accountOrChannel ? accountOrChannel.Actor : undefined) 76 inboxQueue.push({
77 activities,
78 signatureActor: res.locals.signature.actor,
79 inboxActor: accountOrChannel ? accountOrChannel.Actor : undefined
80 })
70 81
71 res.status(204).end() 82 return res.status(204).end()
72} 83}
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 6edbe4820..95549b724 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -8,7 +8,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers'
8import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' 8import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
9import { customConfigUpdateValidator } from '../../middlewares/validators/config' 9import { customConfigUpdateValidator } from '../../middlewares/validators/config'
10import { ClientHtml } from '../../lib/client-html' 10import { ClientHtml } from '../../lib/client-html'
11import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger' 11import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger'
12import { remove, writeJSON } from 'fs-extra' 12import { remove, writeJSON } from 'fs-extra'
13 13
14const packageJSON = require('../../../../package.json') 14const packageJSON = require('../../../../package.json')
@@ -134,10 +134,7 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex
134async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { 134async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) {
135 await remove(CONFIG.CUSTOM_FILE) 135 await remove(CONFIG.CUSTOM_FILE)
136 136
137 auditLogger.delete( 137 auditLogger.delete(getAuditIdFromRes(res), new CustomConfigAuditView(customConfig()))
138 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
139 new CustomConfigAuditView(customConfig())
140 )
141 138
142 reloadConfig() 139 reloadConfig()
143 ClientHtml.invalidCache() 140 ClientHtml.invalidCache()
@@ -183,7 +180,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response,
183 const data = customConfig() 180 const data = customConfig()
184 181
185 auditLogger.update( 182 auditLogger.update(
186 res.locals.oauth.token.User.Account.Actor.getIdentifier(), 183 getAuditIdFromRes(res),
187 new CustomConfigAuditView(data), 184 new CustomConfigAuditView(data),
188 oldCustomConfigAuditKeys 185 oldCustomConfigAuditKeys
189 ) 186 )
diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts
index da941c0ac..8b6773056 100644
--- a/server/controllers/api/overviews.ts
+++ b/server/controllers/api/overviews.ts
@@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video'
4import { asyncMiddleware } from '../../middlewares' 4import { asyncMiddleware } from '../../middlewares'
5import { TagModel } from '../../models/video/tag' 5import { TagModel } from '../../models/video/tag'
6import { VideosOverview } from '../../../shared/models/overviews' 6import { VideosOverview } from '../../../shared/models/overviews'
7import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' 7import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
8import { cacheRoute } from '../../middlewares/cache' 8import { cacheRoute } from '../../middlewares/cache'
9import * as memoizee from 'memoizee'
9 10
10const overviewsRouter = express.Router() 11const overviewsRouter = express.Router()
11 12
@@ -20,13 +21,30 @@ export { overviewsRouter }
20 21
21// --------------------------------------------------------------------------- 22// ---------------------------------------------------------------------------
22 23
24const buildSamples = memoizee(async function () {
25 const [ categories, channels, tags ] = await Promise.all([
26 VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
27 VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
28 TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
29 ])
30
31 return { categories, channels, tags }
32}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
33
23// This endpoint could be quite long, but we cache it 34// This endpoint could be quite long, but we cache it
24async function getVideosOverview (req: express.Request, res: express.Response) { 35async function getVideosOverview (req: express.Request, res: express.Response) {
25 const attributes = await buildSamples() 36 const attributes = await buildSamples()
37
38 const [ categories, channels, tags ] = await Promise.all([
39 Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
40 Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
41 Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
42 ])
43
26 const result: VideosOverview = { 44 const result: VideosOverview = {
27 categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), 45 categories,
28 channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), 46 channels,
29 tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) 47 tags
30 } 48 }
31 49
32 // Cleanup our object 50 // Cleanup our object
@@ -37,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
37 return res.json(result) 55 return res.json(result)
38} 56}
39 57
40async function buildSamples () {
41 const [ categories, channels, tags ] = await Promise.all([
42 VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
43 VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
44 TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
45 ])
46
47 return { categories, channels, tags }
48}
49
50async function getVideosByTag (tag: string, res: express.Response) { 58async function getVideosByTag (tag: string, res: express.Response) {
51 const videos = await getVideos(res, { tagsOneOf: [ tag ] }) 59 const videos = await getVideos(res, { tagsOneOf: [ tag ] })
52 60
@@ -84,14 +92,16 @@ async function getVideos (
84 res: express.Response, 92 res: express.Response,
85 where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } 93 where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
86) { 94) {
87 const { data } = await VideoModel.listForApi(Object.assign({ 95 const query = Object.assign({
88 start: 0, 96 start: 0,
89 count: 10, 97 count: 10,
90 sort: '-createdAt', 98 sort: '-createdAt',
91 includeLocalVideos: true, 99 includeLocalVideos: true,
92 nsfw: buildNSFWFilter(res), 100 nsfw: buildNSFWFilter(res),
93 withFiles: false 101 withFiles: false
94 }, where)) 102 }, where)
103
104 const { data } = await VideoModel.listForApi(query, false)
95 105
96 return data.map(d => d.toFormattedJSON()) 106 return data.map(d => d.toFormattedJSON())
97} 107}
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts
index 28a7a04ca..fd4db7a54 100644
--- a/server/controllers/api/search.ts
+++ b/server/controllers/api/search.ts
@@ -56,6 +56,9 @@ function searchVideoChannels (req: express.Request, res: express.Response) {
56 const isURISearch = search.startsWith('http://') || search.startsWith('https://') 56 const isURISearch = search.startsWith('http://') || search.startsWith('https://')
57 57
58 const parts = search.split('@') 58 const parts = search.split('@')
59
60 // Handle strings like @toto@example.com
61 if (parts.length === 3 && parts[0].length === 0) parts.shift()
59 const isWebfingerSearch = parts.length === 2 && parts.every(p => p.indexOf(' ') === -1) 62 const isWebfingerSearch = parts.length === 2 && parts.every(p => p.indexOf(' ') === -1)
60 63
61 if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res) 64 if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res)
@@ -86,7 +89,7 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean
86 89
87 if (isUserAbleToSearchRemoteURI(res)) { 90 if (isUserAbleToSearchRemoteURI(res)) {
88 try { 91 try {
89 const actor = await getOrCreateActorAndServerAndModel(uri, true, true) 92 const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true)
90 videoChannel = actor.VideoChannel 93 videoChannel = actor.VideoChannel
91 } catch (err) { 94 } catch (err) {
92 logger.info('Cannot search remote video channel %s.', uri, { err }) 95 logger.info('Cannot search remote video channel %s.', uri, { err })
@@ -136,7 +139,7 @@ async function searchVideoURI (url: string, res: express.Response) {
136 refreshVideo: false 139 refreshVideo: false
137 } 140 }
138 141
139 const result = await getOrCreateVideoAndAccountAndChannel(url, syncParam) 142 const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam })
140 video = result ? result.video : undefined 143 video = result ? result.video : undefined
141 } catch (err) { 144 } catch (err) {
142 logger.info('Cannot search remote video %s.', url, { err }) 145 logger.info('Cannot search remote video %s.', url, { err })
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts
index 6f4fe938c..85803f69e 100644
--- a/server/controllers/api/server/stats.ts
+++ b/server/controllers/api/server/stats.ts
@@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { VideoModel } from '../../../models/video/video' 6import { VideoModel } from '../../../models/video/video'
7import { VideoCommentModel } from '../../../models/video/video-comment' 7import { VideoCommentModel } from '../../../models/video/video-comment'
8import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
9import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
10import { cacheRoute } from '../../../middlewares/cache'
8 11
9const statsRouter = express.Router() 12const statsRouter = express.Router()
10 13
11statsRouter.get('/stats', 14statsRouter.get('/stats',
15 asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
12 asyncMiddleware(getStats) 16 asyncMiddleware(getStats)
13) 17)
14 18
@@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr
18 const { totalUsers } = await UserModel.getStats() 22 const { totalUsers } = await UserModel.getStats()
19 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() 23 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
20 24
25 const videosRedundancyStats = await Promise.all(
26 CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => {
27 return VideoRedundancyModel.getStats(r.strategy)
28 .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
29 })
30 )
31
21 const data: ServerStats = { 32 const data: ServerStats = {
22 totalLocalVideos, 33 totalLocalVideos,
23 totalLocalVideoViews, 34 totalLocalVideoViews,
@@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr
26 totalVideoComments, 37 totalVideoComments,
27 totalUsers, 38 totalUsers,
28 totalInstanceFollowers, 39 totalInstanceFollowers,
29 totalInstanceFollowing 40 totalInstanceFollowing,
41 videosRedundancy: videosRedundancyStats
30 } 42 }
31 43
32 return res.json(data).end() 44 return res.json(data).end()
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 07edf3727..8b8ebcd23 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -27,13 +27,17 @@ import {
27 usersUpdateValidator 27 usersUpdateValidator
28} from '../../../middlewares' 28} from '../../../middlewares'
29import { 29import {
30 usersAskResetPasswordValidator, usersBlockingValidator, usersResetPasswordValidator, 30 usersAskResetPasswordValidator,
31 usersAskSendVerifyEmailValidator, usersVerifyEmailValidator 31 usersAskSendVerifyEmailValidator,
32 usersBlockingValidator,
33 usersResetPasswordValidator,
34 usersVerifyEmailValidator
32} from '../../../middlewares/validators' 35} from '../../../middlewares/validators'
33import { UserModel } from '../../../models/account/user' 36import { UserModel } from '../../../models/account/user'
34import { OAuthTokenModel } from '../../../models/oauth/oauth-token' 37import { OAuthTokenModel } from '../../../models/oauth/oauth-token'
35import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' 38import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger'
36import { meRouter } from './me' 39import { meRouter } from './me'
40import { deleteUserToken } from '../../../lib/oauth-model'
37 41
38const auditLogger = auditLoggerFactory('users') 42const auditLogger = auditLoggerFactory('users')
39 43
@@ -166,7 +170,7 @@ async function createUser (req: express.Request, res: express.Response) {
166 170
167 const { user, account } = await createUserAccountAndChannel(userToCreate) 171 const { user, account } = await createUserAccountAndChannel(userToCreate)
168 172
169 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) 173 auditLogger.create(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
170 logger.info('User %s with its channel and account created.', body.username) 174 logger.info('User %s with its channel and account created.', body.username)
171 175
172 return res.json({ 176 return res.json({
@@ -245,7 +249,7 @@ async function removeUser (req: express.Request, res: express.Response, next: ex
245 249
246 await user.destroy() 250 await user.destroy()
247 251
248 auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) 252 auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
249 253
250 return res.sendStatus(204) 254 return res.sendStatus(204)
251} 255}
@@ -264,15 +268,9 @@ async function updateUser (req: express.Request, res: express.Response, next: ex
264 const user = await userToUpdate.save() 268 const user = await userToUpdate.save()
265 269
266 // Destroy user token to refresh rights 270 // Destroy user token to refresh rights
267 if (roleChanged) { 271 if (roleChanged) await deleteUserToken(userToUpdate.id)
268 await OAuthTokenModel.deleteUserToken(userToUpdate.id)
269 }
270 272
271 auditLogger.update( 273 auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
272 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
273 new UserAuditView(user.toFormattedJSON()),
274 oldUserAuditView
275 )
276 274
277 // Don't need to send this update to followers, these attributes are not propagated 275 // Don't need to send this update to followers, these attributes are not propagated
278 276
@@ -333,16 +331,12 @@ async function changeUserBlock (res: express.Response, user: UserModel, block: b
333 user.blockedReason = reason || null 331 user.blockedReason = reason || null
334 332
335 await sequelizeTypescript.transaction(async t => { 333 await sequelizeTypescript.transaction(async t => {
336 await OAuthTokenModel.deleteUserToken(user.id, t) 334 await deleteUserToken(user.id, t)
337 335
338 await user.save({ transaction: t }) 336 await user.save({ transaction: t })
339 }) 337 })
340 338
341 await Emailer.Instance.addUserBlockJob(user, block, reason) 339 await Emailer.Instance.addUserBlockJob(user, block, reason)
342 340
343 auditLogger.update( 341 auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
344 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
345 new UserAuditView(user.toFormattedJSON()),
346 oldUserAuditView
347 )
348} 342}
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts
index e886d4b2a..ff3a87b7f 100644
--- a/server/controllers/api/users/me.ts
+++ b/server/controllers/api/users/me.ts
@@ -5,7 +5,8 @@ import { getFormattedObjects } from '../../../helpers/utils'
5import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers' 5import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers'
6import { sendUpdateActor } from '../../../lib/activitypub/send' 6import { sendUpdateActor } from '../../../lib/activitypub/send'
7import { 7import {
8 asyncMiddleware, asyncRetryTransactionMiddleware, 8 asyncMiddleware,
9 asyncRetryTransactionMiddleware,
9 authenticate, 10 authenticate,
10 commonVideosFiltersValidator, 11 commonVideosFiltersValidator,
11 paginationValidator, 12 paginationValidator,
@@ -17,11 +18,11 @@ import {
17 usersVideoRatingValidator 18 usersVideoRatingValidator
18} from '../../../middlewares' 19} from '../../../middlewares'
19import { 20import {
21 areSubscriptionsExistValidator,
20 deleteMeValidator, 22 deleteMeValidator,
21 userSubscriptionsSortValidator, 23 userSubscriptionsSortValidator,
22 videoImportsSortValidator, 24 videoImportsSortValidator,
23 videosSortValidator, 25 videosSortValidator
24 areSubscriptionsExistValidator
25} from '../../../middlewares/validators' 26} from '../../../middlewares/validators'
26import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 27import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
27import { UserModel } from '../../../models/account/user' 28import { UserModel } from '../../../models/account/user'
@@ -31,12 +32,13 @@ import { buildNSFWFilter, createReqFiles } from '../../../helpers/express-utils'
31import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model' 32import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model'
32import { updateAvatarValidator } from '../../../middlewares/validators/avatar' 33import { updateAvatarValidator } from '../../../middlewares/validators/avatar'
33import { updateActorAvatarFile } from '../../../lib/avatar' 34import { updateActorAvatarFile } from '../../../lib/avatar'
34import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' 35import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger'
35import { VideoImportModel } from '../../../models/video/video-import' 36import { VideoImportModel } from '../../../models/video/video-import'
36import { VideoFilter } from '../../../../shared/models/videos/video-query.type' 37import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
37import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 38import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
38import { JobQueue } from '../../../lib/job-queue' 39import { JobQueue } from '../../../lib/job-queue'
39import { logger } from '../../../helpers/logger' 40import { logger } from '../../../helpers/logger'
41import { AccountModel } from '../../../models/account/account'
40 42
41const auditLogger = auditLoggerFactory('users-me') 43const auditLogger = auditLoggerFactory('users-me')
42 44
@@ -293,7 +295,7 @@ async function getUserVideoQuotaUsed (req: express.Request, res: express.Respons
293} 295}
294 296
295async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { 297async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) {
296 const videoId = +req.params.videoId 298 const videoId = res.locals.video.id
297 const accountId = +res.locals.oauth.token.User.Account.id 299 const accountId = +res.locals.oauth.token.User.Account.id
298 300
299 const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null) 301 const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null)
@@ -311,7 +313,7 @@ async function deleteMe (req: express.Request, res: express.Response) {
311 313
312 await user.destroy() 314 await user.destroy()
313 315
314 auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) 316 auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
315 317
316 return res.sendStatus(204) 318 return res.sendStatus(204)
317} 319}
@@ -328,19 +330,17 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
328 if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo 330 if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo
329 331
330 await sequelizeTypescript.transaction(async t => { 332 await sequelizeTypescript.transaction(async t => {
333 const userAccount = await AccountModel.load(user.Account.id)
334
331 await user.save({ transaction: t }) 335 await user.save({ transaction: t })
332 336
333 if (body.displayName !== undefined) user.Account.name = body.displayName 337 if (body.displayName !== undefined) userAccount.name = body.displayName
334 if (body.description !== undefined) user.Account.description = body.description 338 if (body.description !== undefined) userAccount.description = body.description
335 await user.Account.save({ transaction: t }) 339 await userAccount.save({ transaction: t })
336 340
337 await sendUpdateActor(user.Account, t) 341 await sendUpdateActor(userAccount, t)
338 342
339 auditLogger.update( 343 auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
340 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
341 new UserAuditView(user.toFormattedJSON()),
342 oldUserAuditView
343 )
344 }) 344 })
345 345
346 return res.sendStatus(204) 346 return res.sendStatus(204)
@@ -350,15 +350,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next
350 const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] 350 const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
351 const user: UserModel = res.locals.oauth.token.user 351 const user: UserModel = res.locals.oauth.token.user
352 const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) 352 const oldUserAuditView = new UserAuditView(user.toFormattedJSON())
353 const account = user.Account
354 353
355 const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) 354 const userAccount = await AccountModel.load(user.Account.id)
356 355
357 auditLogger.update( 356 const avatar = await updateActorAvatarFile(avatarPhysicalFile, userAccount)
358 res.locals.oauth.token.User.Account.Actor.getIdentifier(), 357
359 new UserAuditView(user.toFormattedJSON()), 358 auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView)
360 oldUserAuditView
361 )
362 359
363 return res.json({ avatar: avatar.toFormattedJSON() }) 360 return res.json({ avatar: avatar.toFormattedJSON() })
364} 361}
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index a7a36080b..ff6bbe44c 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -27,8 +27,9 @@ import { logger } from '../../helpers/logger'
27import { VideoModel } from '../../models/video/video' 27import { VideoModel } from '../../models/video/video'
28import { updateAvatarValidator } from '../../middlewares/validators/avatar' 28import { updateAvatarValidator } from '../../middlewares/validators/avatar'
29import { updateActorAvatarFile } from '../../lib/avatar' 29import { updateActorAvatarFile } from '../../lib/avatar'
30import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger' 30import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger'
31import { resetSequelizeInstance } from '../../helpers/database-utils' 31import { resetSequelizeInstance } from '../../helpers/database-utils'
32import { UserModel } from '../../models/account/user'
32 33
33const auditLogger = auditLoggerFactory('channels') 34const auditLogger = auditLoggerFactory('channels')
34const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) 35const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
@@ -55,7 +56,7 @@ videoChannelRouter.post('/:nameWithHost/avatar/pick',
55 // Check the rights 56 // Check the rights
56 asyncMiddleware(videoChannelsUpdateValidator), 57 asyncMiddleware(videoChannelsUpdateValidator),
57 updateAvatarValidator, 58 updateAvatarValidator,
58 asyncMiddleware(updateVideoChannelAvatar) 59 asyncRetryTransactionMiddleware(updateVideoChannelAvatar)
59) 60)
60 61
61videoChannelRouter.put('/:nameWithHost', 62videoChannelRouter.put('/:nameWithHost',
@@ -106,13 +107,9 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
106 const videoChannel = res.locals.videoChannel as VideoChannelModel 107 const videoChannel = res.locals.videoChannel as VideoChannelModel
107 const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) 108 const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
108 109
109 const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) 110 const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel)
110 111
111 auditLogger.update( 112 auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
112 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
113 new VideoChannelAuditView(videoChannel.toFormattedJSON()),
114 oldVideoChannelAuditKeys
115 )
116 113
117 return res 114 return res
118 .json({ 115 .json({
@@ -123,19 +120,17 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
123 120
124async function addVideoChannel (req: express.Request, res: express.Response) { 121async function addVideoChannel (req: express.Request, res: express.Response) {
125 const videoChannelInfo: VideoChannelCreate = req.body 122 const videoChannelInfo: VideoChannelCreate = req.body
126 const account: AccountModel = res.locals.oauth.token.User.Account
127 123
128 const videoChannelCreated: VideoChannelModel = await sequelizeTypescript.transaction(async t => { 124 const videoChannelCreated: VideoChannelModel = await sequelizeTypescript.transaction(async t => {
125 const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
126
129 return createVideoChannel(videoChannelInfo, account, t) 127 return createVideoChannel(videoChannelInfo, account, t)
130 }) 128 })
131 129
132 setAsyncActorKeys(videoChannelCreated.Actor) 130 setAsyncActorKeys(videoChannelCreated.Actor)
133 .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) 131 .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err }))
134 132
135 auditLogger.create( 133 auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()))
136 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
137 new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())
138 )
139 logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) 134 logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid)
140 135
141 return res.json({ 136 return res.json({
@@ -166,7 +161,7 @@ async function updateVideoChannel (req: express.Request, res: express.Response)
166 await sendUpdateActor(videoChannelInstanceUpdated, t) 161 await sendUpdateActor(videoChannelInstanceUpdated, t)
167 162
168 auditLogger.update( 163 auditLogger.update(
169 res.locals.oauth.token.User.Account.Actor.getIdentifier(), 164 getAuditIdFromRes(res),
170 new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), 165 new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()),
171 oldVideoChannelAuditKeys 166 oldVideoChannelAuditKeys
172 ) 167 )
@@ -192,10 +187,7 @@ async function removeVideoChannel (req: express.Request, res: express.Response)
192 await sequelizeTypescript.transaction(async t => { 187 await sequelizeTypescript.transaction(async t => {
193 await videoChannelInstance.destroy({ transaction: t }) 188 await videoChannelInstance.destroy({ transaction: t })
194 189
195 auditLogger.delete( 190 auditLogger.delete(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()))
196 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
197 new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())
198 )
199 logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) 191 logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid)
200 }) 192 })
201 193
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts
index 08e11b00b..d0c81804b 100644
--- a/server/controllers/api/videos/abuse.ts
+++ b/server/controllers/api/videos/abuse.ts
@@ -21,6 +21,7 @@ import { AccountModel } from '../../../models/account/account'
21import { VideoModel } from '../../../models/video/video' 21import { VideoModel } from '../../../models/video/video'
22import { VideoAbuseModel } from '../../../models/video/video-abuse' 22import { VideoAbuseModel } from '../../../models/video/video-abuse'
23import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger' 23import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger'
24import { UserModel } from '../../../models/account/user'
24 25
25const auditLogger = auditLoggerFactory('abuse') 26const auditLogger = auditLoggerFactory('abuse')
26const abuseVideoRouter = express.Router() 27const abuseVideoRouter = express.Router()
@@ -95,17 +96,18 @@ async function deleteVideoAbuse (req: express.Request, res: express.Response) {
95 96
96async function reportVideoAbuse (req: express.Request, res: express.Response) { 97async function reportVideoAbuse (req: express.Request, res: express.Response) {
97 const videoInstance = res.locals.video as VideoModel 98 const videoInstance = res.locals.video as VideoModel
98 const reporterAccount = res.locals.oauth.token.User.Account as AccountModel
99 const body: VideoAbuseCreate = req.body 99 const body: VideoAbuseCreate = req.body
100 100
101 const abuseToCreate = {
102 reporterAccountId: reporterAccount.id,
103 reason: body.reason,
104 videoId: videoInstance.id,
105 state: VideoAbuseState.PENDING
106 }
107
108 const videoAbuse: VideoAbuseModel = await sequelizeTypescript.transaction(async t => { 101 const videoAbuse: VideoAbuseModel = await sequelizeTypescript.transaction(async t => {
102 const reporterAccount = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
103
104 const abuseToCreate = {
105 reporterAccountId: reporterAccount.id,
106 reason: body.reason,
107 videoId: videoInstance.id,
108 state: VideoAbuseState.PENDING
109 }
110
109 const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) 111 const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t })
110 videoAbuseInstance.Video = videoInstance 112 videoAbuseInstance.Video = videoInstance
111 videoAbuseInstance.Account = reporterAccount 113 videoAbuseInstance.Account = reporterAccount
@@ -121,7 +123,6 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
121 }) 123 })
122 124
123 logger.info('Abuse report for video %s created.', videoInstance.name) 125 logger.info('Abuse report for video %s created.', videoInstance.name)
124 return res.json({ 126
125 videoAbuse: videoAbuse.toFormattedJSON() 127 return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()
126 }).end()
127} 128}
diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts
index e35247829..dc25e1e85 100644
--- a/server/controllers/api/videos/comment.ts
+++ b/server/controllers/api/videos/comment.ts
@@ -23,7 +23,9 @@ import {
23} from '../../../middlewares/validators/video-comments' 23} from '../../../middlewares/validators/video-comments'
24import { VideoModel } from '../../../models/video/video' 24import { VideoModel } from '../../../models/video/video'
25import { VideoCommentModel } from '../../../models/video/video-comment' 25import { VideoCommentModel } from '../../../models/video/video-comment'
26import { auditLoggerFactory, CommentAuditView } from '../../../helpers/audit-logger' 26import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger'
27import { AccountModel } from '../../../models/account/account'
28import { UserModel } from '../../../models/account/user'
27 29
28const auditLogger = auditLoggerFactory('comments') 30const auditLogger = auditLoggerFactory('comments')
29const videoCommentRouter = express.Router() 31const videoCommentRouter = express.Router()
@@ -86,7 +88,7 @@ async function listVideoThreadComments (req: express.Request, res: express.Respo
86 let resultList: ResultList<VideoCommentModel> 88 let resultList: ResultList<VideoCommentModel>
87 89
88 if (video.commentsEnabled === true) { 90 if (video.commentsEnabled === true) {
89 resultList = await VideoCommentModel.listThreadCommentsForApi(res.locals.video.id, res.locals.videoCommentThread.id) 91 resultList = await VideoCommentModel.listThreadCommentsForApi(video.id, res.locals.videoCommentThread.id)
90 } else { 92 } else {
91 resultList = { 93 resultList = {
92 total: 0, 94 total: 0,
@@ -101,15 +103,17 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons
101 const videoCommentInfo: VideoCommentCreate = req.body 103 const videoCommentInfo: VideoCommentCreate = req.body
102 104
103 const comment = await sequelizeTypescript.transaction(async t => { 105 const comment = await sequelizeTypescript.transaction(async t => {
106 const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
107
104 return createVideoComment({ 108 return createVideoComment({
105 text: videoCommentInfo.text, 109 text: videoCommentInfo.text,
106 inReplyToComment: null, 110 inReplyToComment: null,
107 video: res.locals.video, 111 video: res.locals.video,
108 account: res.locals.oauth.token.User.Account 112 account
109 }, t) 113 }, t)
110 }) 114 })
111 115
112 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) 116 auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))
113 117
114 return res.json({ 118 return res.json({
115 comment: comment.toFormattedJSON() 119 comment: comment.toFormattedJSON()
@@ -120,19 +124,19 @@ async function addVideoCommentReply (req: express.Request, res: express.Response
120 const videoCommentInfo: VideoCommentCreate = req.body 124 const videoCommentInfo: VideoCommentCreate = req.body
121 125
122 const comment = await sequelizeTypescript.transaction(async t => { 126 const comment = await sequelizeTypescript.transaction(async t => {
127 const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t)
128
123 return createVideoComment({ 129 return createVideoComment({
124 text: videoCommentInfo.text, 130 text: videoCommentInfo.text,
125 inReplyToComment: res.locals.videoComment, 131 inReplyToComment: res.locals.videoComment,
126 video: res.locals.video, 132 video: res.locals.video,
127 account: res.locals.oauth.token.User.Account 133 account
128 }, t) 134 }, t)
129 }) 135 })
130 136
131 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) 137 auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))
132 138
133 return res.json({ 139 return res.json({ comment: comment.toFormattedJSON() }).end()
134 comment: comment.toFormattedJSON()
135 }).end()
136} 140}
137 141
138async function removeVideoComment (req: express.Request, res: express.Response) { 142async function removeVideoComment (req: express.Request, res: express.Response) {
@@ -143,7 +147,7 @@ async function removeVideoComment (req: express.Request, res: express.Response)
143 }) 147 })
144 148
145 auditLogger.delete( 149 auditLogger.delete(
146 res.locals.oauth.token.User.Account.Actor.getIdentifier(), 150 getAuditIdFromRes(res),
147 new CommentAuditView(videoCommentInstance.toFormattedJSON()) 151 new CommentAuditView(videoCommentInstance.toFormattedJSON())
148 ) 152 )
149 logger.info('Video comment %d deleted.', videoCommentInstance.id) 153 logger.info('Video comment %d deleted.', videoCommentInstance.id)
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 44f15ef74..398fd5a7f 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import * as magnetUtil from 'magnet-uri' 2import * as magnetUtil from 'magnet-uri'
3import 'multer' 3import 'multer'
4import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' 4import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
5import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' 5import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
6import { 6import {
7 CONFIG, 7 CONFIG,
@@ -114,7 +114,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
114 } 114 }
115 await JobQueue.Instance.createJob({ type: 'video-import', payload }) 115 await JobQueue.Instance.createJob({ type: 'video-import', payload })
116 116
117 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) 117 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
118 118
119 return res.json(videoImport.toFormattedJSON()).end() 119 return res.json(videoImport.toFormattedJSON()).end()
120} 120}
@@ -158,7 +158,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
158 } 158 }
159 await JobQueue.Instance.createJob({ type: 'video-import', payload }) 159 await JobQueue.Instance.createJob({ type: 'video-import', payload })
160 160
161 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) 161 auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
162 162
163 return res.json(videoImport.toFormattedJSON()).end() 163 return res.json(videoImport.toFormattedJSON()).end()
164} 164}
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 0c9e6c2d1..581046782 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -4,7 +4,7 @@ import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../
4import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 4import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
5import { processImage } from '../../../helpers/image-utils' 5import { processImage } from '../../../helpers/image-utils'
6import { logger } from '../../../helpers/logger' 6import { logger } from '../../../helpers/logger'
7import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger' 7import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
8import { getFormattedObjects, getServerActor } from '../../../helpers/utils' 8import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
9import { 9import {
10 CONFIG, 10 CONFIG,
@@ -253,7 +253,7 @@ async function addVideo (req: express.Request, res: express.Response) {
253 253
254 await federateVideoIfNeeded(video, true, t) 254 await federateVideoIfNeeded(video, true, t)
255 255
256 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) 256 auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
257 logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) 257 logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid)
258 258
259 return videoCreated 259 return videoCreated
@@ -354,7 +354,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
354 await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) 354 await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t)
355 355
356 auditLogger.update( 356 auditLogger.update(
357 res.locals.oauth.token.User.Account.Actor.getIdentifier(), 357 getAuditIdFromRes(res),
358 new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), 358 new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()),
359 oldVideoAuditView 359 oldVideoAuditView
360 ) 360 )
@@ -393,9 +393,9 @@ async function viewVideo (req: express.Request, res: express.Response) {
393 Redis.Instance.setIPVideoView(ip, videoInstance.uuid) 393 Redis.Instance.setIPVideoView(ip, videoInstance.uuid)
394 ]) 394 ])
395 395
396 const serverAccount = await getServerActor() 396 const serverActor = await getServerActor()
397 397
398 await sendCreateView(serverAccount, videoInstance, undefined) 398 await sendCreateView(serverActor, videoInstance, undefined)
399 399
400 return res.status(204).end() 400 return res.status(204).end()
401} 401}
@@ -439,7 +439,7 @@ async function removeVideo (req: express.Request, res: express.Response) {
439 await videoInstance.destroy({ transaction: t }) 439 await videoInstance.destroy({ transaction: t })
440 }) 440 })
441 441
442 auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) 442 auditLogger.delete(getAuditIdFromRes(res), new VideoAuditView(videoInstance.toFormattedDetailsJSON()))
443 logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) 443 logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid)
444 444
445 return res.type('json').status(204).end() 445 return res.type('json').status(204).end()
diff --git a/server/controllers/api/videos/ownership.ts b/server/controllers/api/videos/ownership.ts
index d26ed6cfc..5ea7d7c6a 100644
--- a/server/controllers/api/videos/ownership.ts
+++ b/server/controllers/api/videos/ownership.ts
@@ -19,6 +19,7 @@ import { VideoChannelModel } from '../../../models/video/video-channel'
19import { getFormattedObjects } from '../../../helpers/utils' 19import { getFormattedObjects } from '../../../helpers/utils'
20import { changeVideoChannelShare } from '../../../lib/activitypub' 20import { changeVideoChannelShare } from '../../../lib/activitypub'
21import { sendUpdateVideo } from '../../../lib/activitypub/send' 21import { sendUpdateVideo } from '../../../lib/activitypub/send'
22import { UserModel } from '../../../models/account/user'
22 23
23const ownershipVideoRouter = express.Router() 24const ownershipVideoRouter = express.Router()
24 25
@@ -58,26 +59,25 @@ export {
58 59
59async function giveVideoOwnership (req: express.Request, res: express.Response) { 60async function giveVideoOwnership (req: express.Request, res: express.Response) {
60 const videoInstance = res.locals.video as VideoModel 61 const videoInstance = res.locals.video as VideoModel
61 const initiatorAccount = res.locals.oauth.token.User.Account as AccountModel 62 const initiatorAccountId = (res.locals.oauth.token.User as UserModel).Account.id
62 const nextOwner = res.locals.nextOwner as AccountModel 63 const nextOwner = res.locals.nextOwner as AccountModel
63 64
64 await sequelizeTypescript.transaction(t => { 65 await sequelizeTypescript.transaction(t => {
65 return VideoChangeOwnershipModel.findOrCreate({ 66 return VideoChangeOwnershipModel.findOrCreate({
66 where: { 67 where: {
67 initiatorAccountId: initiatorAccount.id, 68 initiatorAccountId,
68 nextOwnerAccountId: nextOwner.id, 69 nextOwnerAccountId: nextOwner.id,
69 videoId: videoInstance.id, 70 videoId: videoInstance.id,
70 status: VideoChangeOwnershipStatus.WAITING 71 status: VideoChangeOwnershipStatus.WAITING
71 }, 72 },
72 defaults: { 73 defaults: {
73 initiatorAccountId: initiatorAccount.id, 74 initiatorAccountId,
74 nextOwnerAccountId: nextOwner.id, 75 nextOwnerAccountId: nextOwner.id,
75 videoId: videoInstance.id, 76 videoId: videoInstance.id,
76 status: VideoChangeOwnershipStatus.WAITING 77 status: VideoChangeOwnershipStatus.WAITING
77 }, 78 },
78 transaction: t 79 transaction: t
79 }) 80 })
80
81 }) 81 })
82 82
83 logger.info('Ownership change for video %s created.', videoInstance.name) 83 logger.info('Ownership change for video %s created.', videoInstance.name)
@@ -85,9 +85,10 @@ async function giveVideoOwnership (req: express.Request, res: express.Response)
85} 85}
86 86
87async function listVideoOwnership (req: express.Request, res: express.Response) { 87async function listVideoOwnership (req: express.Request, res: express.Response) {
88 const currentAccount = res.locals.oauth.token.User.Account as AccountModel 88 const currentAccountId = (res.locals.oauth.token.User as UserModel).Account.id
89
89 const resultList = await VideoChangeOwnershipModel.listForApi( 90 const resultList = await VideoChangeOwnershipModel.listForApi(
90 currentAccount.id, 91 currentAccountId,
91 req.query.start || 0, 92 req.query.start || 0,
92 req.query.count || 10, 93 req.query.count || 10,
93 req.query.sort || 'createdAt' 94 req.query.sort || 'createdAt'
diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts
index b1732837d..dc322bb0c 100644
--- a/server/controllers/api/videos/rate.ts
+++ b/server/controllers/api/videos/rate.ts
@@ -28,10 +28,11 @@ async function rateVideo (req: express.Request, res: express.Response) {
28 const body: UserVideoRateUpdate = req.body 28 const body: UserVideoRateUpdate = req.body
29 const rateType = body.rating 29 const rateType = body.rating
30 const videoInstance: VideoModel = res.locals.video 30 const videoInstance: VideoModel = res.locals.video
31 const accountInstance: AccountModel = res.locals.oauth.token.User.Account
32 31
33 await sequelizeTypescript.transaction(async t => { 32 await sequelizeTypescript.transaction(async t => {
34 const sequelizeOptions = { transaction: t } 33 const sequelizeOptions = { transaction: t }
34
35 const accountInstance = await AccountModel.load(res.locals.oauth.token.User.Account.id, t)
35 const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) 36 const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t)
36 37
37 let likesToIncrement = 0 38 let likesToIncrement = 0
@@ -47,10 +48,10 @@ async function rateVideo (req: express.Request, res: express.Response) {
47 else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement-- 48 else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement--
48 49
49 if (rateType === 'none') { // Destroy previous rate 50 if (rateType === 'none') { // Destroy previous rate
50 await previousRate.destroy({ transaction: t }) 51 await previousRate.destroy(sequelizeOptions)
51 } else { // Update previous rate 52 } else { // Update previous rate
52 previousRate.type = rateType 53 previousRate.type = rateType
53 await previousRate.save({ transaction: t }) 54 await previousRate.save(sequelizeOptions)
54 } 55 }
55 } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate 56 } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate
56 const query = { 57 const query = {
@@ -70,9 +71,9 @@ async function rateVideo (req: express.Request, res: express.Response) {
70 await videoInstance.increment(incrementQuery, sequelizeOptions) 71 await videoInstance.increment(incrementQuery, sequelizeOptions)
71 72
72 await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) 73 await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t)
73 })
74 74
75 logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) 75 logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name)
76 })
76 77
77 return res.type('json').status(204).end() 78 return res.type('json').status(204).end()
78} 79}
diff --git a/server/controllers/client.ts b/server/controllers/client.ts
index c33061289..73b40cf65 100644
--- a/server/controllers/client.ts
+++ b/server/controllers/client.ts
@@ -35,7 +35,7 @@ clientsRouter.use('' +
35// Static HTML/CSS/JS client files 35// Static HTML/CSS/JS client files
36 36
37const staticClientFiles = [ 37const staticClientFiles = [
38 'manifest.json', 38 'manifest.webmanifest',
39 'ngsw-worker.js', 39 'ngsw-worker.js',
40 'ngsw.json' 40 'ngsw.json'
41] 41]
diff --git a/server/helpers/actor.ts b/server/helpers/actor.ts
new file mode 100644
index 000000000..12a7ace9f
--- /dev/null
+++ b/server/helpers/actor.ts
@@ -0,0 +1,13 @@
1import { ActorModel } from '../models/activitypub/actor'
2
3type ActorFetchByUrlType = 'all' | 'actor-and-association-ids'
4function fetchActorByUrl (url: string, fetchType: ActorFetchByUrlType) {
5 if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url)
6
7 if (fetchType === 'actor-and-association-ids') return ActorModel.loadByUrl(url)
8}
9
10export {
11 ActorFetchByUrlType,
12 fetchActorByUrl
13}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 7db72b69c..00311fce1 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -1,4 +1,5 @@
1import * as path from 'path' 1import * as path from 'path'
2import * as express from 'express'
2import { diff } from 'deep-object-diff' 3import { diff } from 'deep-object-diff'
3import { chain } from 'lodash' 4import { chain } from 'lodash'
4import * as flatten from 'flat' 5import * as flatten from 'flat'
@@ -8,6 +9,11 @@ import { jsonLoggerFormat, labelFormatter } from './logger'
8import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared' 9import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared'
9import { VideoComment } from '../../shared/models/videos/video-comment.model' 10import { VideoComment } from '../../shared/models/videos/video-comment.model'
10import { CustomConfig } from '../../shared/models/server/custom-config.model' 11import { CustomConfig } from '../../shared/models/server/custom-config.model'
12import { UserModel } from '../models/account/user'
13
14function getAuditIdFromRes (res: express.Response) {
15 return (res.locals.oauth.token.User as UserModel).username
16}
11 17
12enum AUDIT_TYPE { 18enum AUDIT_TYPE {
13 CREATE = 'create', 19 CREATE = 'create',
@@ -255,6 +261,8 @@ class CustomConfigAuditView extends EntityAuditView {
255} 261}
256 262
257export { 263export {
264 getAuditIdFromRes,
265
258 auditLoggerFactory, 266 auditLoggerFactory,
259 VideoImportAuditView, 267 VideoImportAuditView,
260 VideoChannelAuditView, 268 VideoChannelAuditView,
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index f76eba474..8772e74cf 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -171,5 +171,3 @@ function setRemoteVideoTruncatedContent (video: any) {
171 171
172 return true 172 return true
173} 173}
174
175
diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts
index aaa0c736b..a7771e07b 100644
--- a/server/helpers/custom-validators/video-ownership.ts
+++ b/server/helpers/custom-validators/video-ownership.ts
@@ -31,7 +31,7 @@ export function checkUserCanTerminateOwnershipChange (
31 videoChangeOwnership: VideoChangeOwnershipModel, 31 videoChangeOwnership: VideoChangeOwnershipModel,
32 res: Response 32 res: Response
33): boolean { 33): boolean {
34 if (videoChangeOwnership.NextOwner.userId === user.Account.userId) { 34 if (videoChangeOwnership.NextOwner.userId === user.id) {
35 return true 35 return true
36 } 36 }
37 37
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index edafba6e2..9875c68bd 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -18,6 +18,7 @@ import { exists, isArray, isFileValid } from './misc'
18import { VideoChannelModel } from '../../models/video/video-channel' 18import { VideoChannelModel } from '../../models/video/video-channel'
19import { UserModel } from '../../models/account/user' 19import { UserModel } from '../../models/account/user'
20import * as magnetUtil from 'magnet-uri' 20import * as magnetUtil from 'magnet-uri'
21import { fetchVideo, VideoFetchType } from '../video'
21 22
22const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS 23const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
23 24
@@ -152,14 +153,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use
152 return true 153 return true
153} 154}
154 155
155async function isVideoExist (id: string, res: Response) { 156async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') {
156 let video: VideoModel | null 157 const video = await fetchVideo(id, fetchType)
157
158 if (validator.isInt(id)) {
159 video = await VideoModel.loadAndPopulateAccountAndServerAndTags(+id)
160 } else { // UUID
161 video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(id)
162 }
163 158
164 if (video === null) { 159 if (video === null) {
165 res.status(404) 160 res.status(404)
@@ -169,7 +164,7 @@ async function isVideoExist (id: string, res: Response) {
169 return false 164 return false
170 } 165 }
171 166
172 res.locals.video = video 167 if (fetchType !== 'none') res.locals.video = video
173 return true 168 return true
174} 169}
175 170
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index a1ed8e72d..a42474417 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -1,12 +1,12 @@
1import { ResultList } from '../../shared' 1import { ResultList } from '../../shared'
2import { CONFIG } from '../initializers' 2import { CONFIG } from '../initializers'
3import { ActorModel } from '../models/activitypub/actor'
4import { ApplicationModel } from '../models/application/application' 3import { ApplicationModel } from '../models/application/application'
5import { pseudoRandomBytesPromise, sha256 } from './core-utils' 4import { pseudoRandomBytesPromise, sha256 } from './core-utils'
6import { logger } from './logger' 5import { logger } from './logger'
7import { join } from 'path' 6import { join } from 'path'
8import { Instance as ParseTorrent } from 'parse-torrent' 7import { Instance as ParseTorrent } from 'parse-torrent'
9import { remove } from 'fs-extra' 8import { remove } from 'fs-extra'
9import * as memoizee from 'memoizee'
10 10
11function deleteFileAsync (path: string) { 11function deleteFileAsync (path: string) {
12 remove(path) 12 remove(path)
@@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
36 } as ResultList<U> 36 } as ResultList<U>
37} 37}
38 38
39async function getServerActor () { 39const getServerActor = memoizee(async function () {
40 if (getServerActor.serverActor === undefined) { 40 const application = await ApplicationModel.load()
41 const application = await ApplicationModel.load() 41 if (!application) throw Error('Could not load Application from database.')
42 if (!application) throw Error('Could not load Application from database.')
43 42
44 getServerActor.serverActor = application.Account.Actor 43 return application.Account.Actor
45 } 44})
46
47 if (!getServerActor.serverActor) {
48 logger.error('Cannot load server actor.')
49 process.exit(0)
50 }
51
52 return Promise.resolve(getServerActor.serverActor)
53}
54namespace getServerActor {
55 export let serverActor: ActorModel
56}
57 45
58function generateVideoTmpPath (target: string | ParseTorrent) { 46function generateVideoTmpPath (target: string | ParseTorrent) {
59 const id = typeof target === 'string' ? target : target.infoHash 47 const id = typeof target === 'string' ? target : target.infoHash
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
new file mode 100644
index 000000000..b1577a6b0
--- /dev/null
+++ b/server/helpers/video.ts
@@ -0,0 +1,25 @@
1import { VideoModel } from '../models/video/video'
2
3type VideoFetchType = 'all' | 'only-video' | 'id' | 'none'
4
5function fetchVideo (id: number | string, fetchType: VideoFetchType) {
6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id)
7
8 if (fetchType === 'only-video') return VideoModel.load(id)
9
10 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
11}
12
13type VideoFetchByUrlType = 'all' | 'only-video'
14function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType) {
15 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
16
17 if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
18}
19
20export {
21 VideoFetchType,
22 VideoFetchByUrlType,
23 fetchVideo,
24 fetchVideoByUrl
25}
diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts
index 10fcec462..156376943 100644
--- a/server/helpers/webfinger.ts
+++ b/server/helpers/webfinger.ts
@@ -12,7 +12,10 @@ const webfinger = new WebFinger({
12 request_timeout: 3000 12 request_timeout: 3000
13}) 13})
14 14
15async function loadActorUrlOrGetFromWebfinger (uri: string) { 15async function loadActorUrlOrGetFromWebfinger (uriArg: string) {
16 // Handle strings like @toto@example.com
17 const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg
18
16 const [ name, host ] = uri.split('@') 19 const [ name, host ] = uri.split('@')
17 let actor: ActorModel 20 let actor: ActorModel
18 21
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 2fdfd1876..f4b44bc4f 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -24,7 +24,7 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: str
24 if (timer) clearTimeout(timer) 24 if (timer) clearTimeout(timer)
25 25
26 return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) 26 return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
27 .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId))) 27 .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
28 } 28 }
29 29
30 file = torrent.files[ 0 ] 30 file = torrent.files[ 0 ]
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 8b2bc1782..25e719cc3 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -2,7 +2,11 @@ import { truncate } from 'lodash'
2import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' 2import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
3import { logger } from './logger' 3import { logger } from './logger'
4import { generateVideoTmpPath } from './utils' 4import { generateVideoTmpPath } from './utils'
5import { YoutubeDlUpdateScheduler } from '../lib/schedulers/youtube-dl-update-scheduler' 5import { join } from 'path'
6import { root } from './core-utils'
7import { ensureDir, writeFile } from 'fs-extra'
8import * as request from 'request'
9import { createWriteStream } from 'fs'
6 10
7export type YoutubeDLInfo = { 11export type YoutubeDLInfo = {
8 name?: string 12 name?: string
@@ -40,7 +44,7 @@ function downloadYoutubeDLVideo (url: string) {
40 44
41 return new Promise<string>(async (res, rej) => { 45 return new Promise<string>(async (res, rej) => {
42 const youtubeDL = await safeGetYoutubeDL() 46 const youtubeDL = await safeGetYoutubeDL()
43 youtubeDL.exec(url, options, async (err, output) => { 47 youtubeDL.exec(url, options, err => {
44 if (err) return rej(err) 48 if (err) return rej(err)
45 49
46 return res(path) 50 return res(path)
@@ -48,6 +52,64 @@ function downloadYoutubeDLVideo (url: string) {
48 }) 52 })
49} 53}
50 54
55// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
56// We rewrote it to avoid sync calls
57async function updateYoutubeDLBinary () {
58 logger.info('Updating youtubeDL binary.')
59
60 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
61 const bin = join(binDirectory, 'youtube-dl')
62 const detailsPath = join(binDirectory, 'details')
63 const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
64
65 await ensureDir(binDirectory)
66
67 return new Promise(res => {
68 request.get(url, { followRedirect: false }, (err, result) => {
69 if (err) {
70 logger.error('Cannot update youtube-dl.', { err })
71 return res()
72 }
73
74 if (result.statusCode !== 302) {
75 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
76 return res()
77 }
78
79 const url = result.headers.location
80 const downloadFile = request.get(url)
81 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
82
83 downloadFile.on('response', result => {
84 if (result.statusCode !== 200) {
85 logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
86 return res()
87 }
88
89 downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
90 })
91
92 downloadFile.on('error', err => {
93 logger.error('youtube-dl update error.', { err })
94 return res()
95 })
96
97 downloadFile.on('end', () => {
98 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
99 writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
100 if (err) {
101 logger.error('youtube-dl update error: cannot write details.', { err })
102 return res()
103 }
104
105 logger.info('youtube-dl updated to version %s.', newVersion)
106 return res()
107 })
108 })
109 })
110 })
111}
112
51async function safeGetYoutubeDL () { 113async function safeGetYoutubeDL () {
52 let youtubeDL 114 let youtubeDL
53 115
@@ -55,7 +117,7 @@ async function safeGetYoutubeDL () {
55 youtubeDL = require('youtube-dl') 117 youtubeDL = require('youtube-dl')
56 } catch (e) { 118 } catch (e) {
57 // Download binary 119 // Download binary
58 await YoutubeDlUpdateScheduler.Instance.execute() 120 await updateYoutubeDLBinary()
59 youtubeDL = require('youtube-dl') 121 youtubeDL = require('youtube-dl')
60 } 122 }
61 123
@@ -65,6 +127,7 @@ async function safeGetYoutubeDL () {
65// --------------------------------------------------------------------------- 127// ---------------------------------------------------------------------------
66 128
67export { 129export {
130 updateYoutubeDLBinary,
68 downloadYoutubeDLVideo, 131 downloadYoutubeDLVideo,
69 getYoutubeDLInfo, 132 getYoutubeDLInfo,
70 safeGetYoutubeDL 133 safeGetYoutubeDL
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 6a2badd35..a54f6155b 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -7,7 +7,7 @@ import { parse } from 'url'
7import { CONFIG } from './constants' 7import { CONFIG } from './constants'
8import { logger } from '../helpers/logger' 8import { logger } from '../helpers/logger'
9import { getServerActor } from '../helpers/utils' 9import { getServerActor } from '../helpers/utils'
10import { VideosRedundancy } from '../../shared/models/redundancy' 10import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11import { isArray } from '../helpers/custom-validators/misc' 11import { isArray } from '../helpers/custom-validators/misc'
12import { uniq } from 'lodash' 12import { uniq } from 'lodash'
13 13
@@ -34,21 +34,28 @@ async function checkActivityPubUrls () {
34function checkConfig () { 34function checkConfig () {
35 const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy') 35 const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
36 36
37 // NSFW policy
37 if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { 38 if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
38 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy 39 return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
39 } 40 }
40 41
41 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos') 42 // Redundancies
43 const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
42 if (isArray(redundancyVideos)) { 44 if (isArray(redundancyVideos)) {
43 for (const r of redundancyVideos) { 45 for (const r of redundancyVideos) {
44 if ([ 'most-views' ].indexOf(r.strategy) === -1) { 46 if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
45 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy 47 return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
46 } 48 }
47 } 49 }
48 50
49 const filtered = uniq(redundancyVideos.map(r => r.strategy)) 51 const filtered = uniq(redundancyVideos.map(r => r.strategy))
50 if (filtered.length !== redundancyVideos.length) { 52 if (filtered.length !== redundancyVideos.length) {
51 return 'Redundancy video entries should have uniq strategies' 53 return 'Redundancy video entries should have unique strategies'
54 }
55
56 const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
57 if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
58 return 'Min views in recently added strategy is not a number'
52 } 59 }
53 } 60 }
54 61
@@ -68,6 +75,7 @@ function checkMissedConfig () {
68 'cache.previews.size', 'admin.email', 75 'cache.previews.size', 'admin.email',
69 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 76 'signup.enabled', 'signup.limit', 'signup.requires_email_verification',
70 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 77 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
78 'redundancy.videos.strategies', 'redundancy.videos.check_interval',
71 'transcoding.enabled', 'transcoding.threads', 79 'transcoding.enabled', 'transcoding.threads',
72 'import.videos.http.enabled', 'import.videos.torrent.enabled', 80 'import.videos.http.enabled', 'import.videos.torrent.enabled',
73 'trending.videos.interval_days', 81 'trending.videos.interval_days',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6b4afbfd8..03424ffb8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,11 +1,11 @@
1import { IConfig } from 'config' 1import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' 3import { JobType, VideoRateType, VideoState, VideosRedundancy } from '../../shared/models'
4import { ActivityPubActorType } from '../../shared/models/activitypub' 4import { ActivityPubActorType } from '../../shared/models/activitypub'
5import { FollowState } from '../../shared/models/actors' 5import { FollowState } from '../../shared/models/actors'
6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' 6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
7// Do not use barrels, remain constants as independent as possible 7// Do not use barrels, remain constants as independent as possible
8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' 8import { buildPath, isTestInstance, parseDuration, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
10import { invert } from 'lodash' 10import { invert } from 'lodash'
11import { CronRepeatOptions, EveryRepeatOptions } from 'bull' 11import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
@@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = {
66 }, 66 },
67 ACTIVITY_PUB: { 67 ACTIVITY_PUB: {
68 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example 68 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
69 } 69 },
70 STATS: '4 hours'
70} 71}
71 72
72// --------------------------------------------------------------------------- 73// ---------------------------------------------------------------------------
@@ -138,8 +139,7 @@ let SCHEDULER_INTERVALS_MS = {
138 badActorFollow: 60000 * 60, // 1 hour 139 badActorFollow: 60000 * 60, // 1 hour
139 removeOldJobs: 60000 * 60, // 1 hour 140 removeOldJobs: 60000 * 60, // 1 hour
140 updateVideos: 60000, // 1 minute 141 updateVideos: 60000, // 1 minute
141 youtubeDLUpdate: 60000 * 60 * 24, // 1 day 142 youtubeDLUpdate: 60000 * 60 * 24 // 1 day
142 videosRedundancy: 60000 * 2 // 2 hours
143} 143}
144 144
145// --------------------------------------------------------------------------- 145// ---------------------------------------------------------------------------
@@ -211,7 +211,10 @@ const CONFIG = {
211 } 211 }
212 }, 212 },
213 REDUNDANCY: { 213 REDUNDANCY: {
214 VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos')) 214 VIDEOS: {
215 CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')),
216 STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
217 }
215 }, 218 },
216 ADMIN: { 219 ADMIN: {
217 get EMAIL () { return config.get<string>('admin.email') } 220 get EMAIL () { return config.get<string>('admin.email') }
@@ -592,6 +595,10 @@ const CACHE = {
592 } 595 }
593} 596}
594 597
598const MEMOIZE_TTL = {
599 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
600}
601
595const REDUNDANCY = { 602const REDUNDANCY = {
596 VIDEOS: { 603 VIDEOS: {
597 EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days 604 EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
@@ -644,7 +651,6 @@ if (isTestInstance() === true) {
644 SCHEDULER_INTERVALS_MS.badActorFollow = 10000 651 SCHEDULER_INTERVALS_MS.badActorFollow = 10000
645 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 652 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
646 SCHEDULER_INTERVALS_MS.updateVideos = 5000 653 SCHEDULER_INTERVALS_MS.updateVideos = 5000
647 SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
648 REPEAT_JOBS['videos-views'] = { every: 5000 } 654 REPEAT_JOBS['videos-views'] = { every: 5000 }
649 655
650 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 656 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
@@ -654,6 +660,8 @@ if (isTestInstance() === true) {
654 JOB_ATTEMPTS['email'] = 1 660 JOB_ATTEMPTS['email'] = 1
655 661
656 CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 662 CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
663 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1
664 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms'
657} 665}
658 666
659updateWebserverConfig() 667updateWebserverConfig()
@@ -708,6 +716,7 @@ export {
708 VIDEO_ABUSE_STATES, 716 VIDEO_ABUSE_STATES,
709 JOB_REQUEST_TIMEOUT, 717 JOB_REQUEST_TIMEOUT,
710 USER_PASSWORD_RESET_LIFETIME, 718 USER_PASSWORD_RESET_LIFETIME,
719 MEMOIZE_TTL,
711 USER_EMAIL_VERIFY_LIFETIME, 720 USER_EMAIL_VERIFY_LIFETIME,
712 IMAGE_MIMETYPE_EXT, 721 IMAGE_MIMETYPE_EXT,
713 OVERVIEWS, 722 OVERVIEWS,
@@ -741,15 +750,10 @@ function updateWebserverConfig () {
741 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) 750 CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
742} 751}
743 752
744function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { 753function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] {
745 if (!objs) return [] 754 if (!objs) return []
746 755
747 return objs.map(obj => { 756 return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) }))
748 return {
749 strategy: obj.strategy,
750 size: bytes.parse(obj.size)
751 }
752 })
753} 757}
754 758
755function buildLanguages () { 759function buildLanguages () {
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 3464add03..d37a695a7 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -21,6 +21,7 @@ import { ServerModel } from '../../models/server/server'
21import { VideoChannelModel } from '../../models/video/video-channel' 21import { VideoChannelModel } from '../../models/video/video-channel'
22import { JobQueue } from '../job-queue' 22import { JobQueue } from '../job-queue'
23import { getServerActor } from '../../helpers/utils' 23import { getServerActor } from '../../helpers/utils'
24import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24 25
25// Set account keys, this could be long so process after the account creation and do not block the client 26// Set account keys, this could be long so process after the account creation and do not block the client
26function setAsyncActorKeys (actor: ActorModel) { 27function setAsyncActorKeys (actor: ActorModel) {
@@ -38,13 +39,14 @@ function setAsyncActorKeys (actor: ActorModel) {
38 39
39async function getOrCreateActorAndServerAndModel ( 40async function getOrCreateActorAndServerAndModel (
40 activityActor: string | ActivityPubActor, 41 activityActor: string | ActivityPubActor,
42 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
41 recurseIfNeeded = true, 43 recurseIfNeeded = true,
42 updateCollections = false 44 updateCollections = false
43) { 45) {
44 const actorUrl = getActorUrl(activityActor) 46 const actorUrl = getActorUrl(activityActor)
45 let created = false 47 let created = false
46 48
47 let actor = await ActorModel.loadByUrl(actorUrl) 49 let actor = await fetchActorByUrl(actorUrl, fetchType)
48 // Orphan actor (not associated to an account of channel) so recreate it 50 // Orphan actor (not associated to an account of channel) so recreate it
49 if (actor && (!actor.Account && !actor.VideoChannel)) { 51 if (actor && (!actor.Account && !actor.VideoChannel)) {
50 await actor.destroy() 52 await actor.destroy()
@@ -65,7 +67,7 @@ async function getOrCreateActorAndServerAndModel (
65 67
66 try { 68 try {
67 // Assert we don't recurse another time 69 // Assert we don't recurse another time
68 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, false) 70 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
69 } catch (err) { 71 } catch (err) {
70 logger.error('Cannot get or create account attributed to video channel ' + actor.url) 72 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
71 throw new Error(err) 73 throw new Error(err)
@@ -79,7 +81,7 @@ async function getOrCreateActorAndServerAndModel (
79 if (actor.Account) actor.Account.Actor = actor 81 if (actor.Account) actor.Account.Actor = actor
80 if (actor.VideoChannel) actor.VideoChannel.Actor = actor 82 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
81 83
82 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor) 84 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
83 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') 85 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
84 86
85 if ((created === true || refreshed === true) && updateCollections === true) { 87 if ((created === true || refreshed === true) && updateCollections === true) {
@@ -370,8 +372,14 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu
370 return videoChannelCreated 372 return videoChannelCreated
371} 373}
372 374
373async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorModel, refreshed: boolean }> { 375async function refreshActorIfNeeded (
374 if (!actor.isOutdated()) return { actor, refreshed: false } 376 actorArg: ActorModel,
377 fetchedType: ActorFetchByUrlType
378): Promise<{ actor: ActorModel, refreshed: boolean }> {
379 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
380
381 // We need more attributes
382 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
375 383
376 try { 384 try {
377 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) 385 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts
index 7b4067c11..a86428461 100644
--- a/server/lib/activitypub/audience.ts
+++ b/server/lib/activitypub/audience.ts
@@ -6,7 +6,7 @@ import { VideoModel } from '../../models/video/video'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoShareModel } from '../../models/video/video-share' 7import { VideoShareModel } from '../../models/video/video-share'
8 8
9function getVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]) { 9function getRemoteVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]): ActivityAudience {
10 return { 10 return {
11 to: [ video.VideoChannel.Account.Actor.url ], 11 to: [ video.VideoChannel.Account.Actor.url ],
12 cc: actorsInvolvedInVideo.map(a => a.followersUrl) 12 cc: actorsInvolvedInVideo.map(a => a.followersUrl)
@@ -18,7 +18,7 @@ function getVideoCommentAudience (
18 threadParentComments: VideoCommentModel[], 18 threadParentComments: VideoCommentModel[],
19 actorsInvolvedInVideo: ActorModel[], 19 actorsInvolvedInVideo: ActorModel[],
20 isOrigin = false 20 isOrigin = false
21) { 21): ActivityAudience {
22 const to = [ ACTIVITY_PUB.PUBLIC ] 22 const to = [ ACTIVITY_PUB.PUBLIC ]
23 const cc: string[] = [] 23 const cc: string[] = []
24 24
@@ -41,7 +41,7 @@ function getVideoCommentAudience (
41 } 41 }
42} 42}
43 43
44function getObjectFollowersAudience (actorsInvolvedInObject: ActorModel[]) { 44function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): ActivityAudience {
45 return { 45 return {
46 to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), 46 to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)),
47 cc: [] 47 cc: []
@@ -83,9 +83,9 @@ function audiencify<T> (object: T, audience: ActivityAudience) {
83export { 83export {
84 buildAudience, 84 buildAudience,
85 getAudience, 85 getAudience,
86 getVideoAudience, 86 getRemoteVideoAudience,
87 getActorsInvolvedInVideo, 87 getActorsInvolvedInVideo,
88 getObjectFollowersAudience, 88 getAudienceFromFollowersOf,
89 audiencify, 89 audiencify,
90 getVideoCommentAudience 90 getVideoCommentAudience
91} 91}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 7325ddcb6..87f8a4162 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,10 +1,9 @@
1import { CacheFileObject } from '../../../shared/index' 1import { CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { ActorModel } from '../../models/activitypub/actor'
4import { sequelizeTypescript } from '../../initializers' 3import { sequelizeTypescript } from '../../initializers'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 4import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6 5
7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { 6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
8 const url = cacheFileObject.url 7 const url = cacheFileObject.url
9 8
10 const videoFile = video.VideoFiles.find(f => { 9 const videoFile = video.VideoFiles.find(f => {
@@ -23,7 +22,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
23 } 22 }
24} 23}
25 24
26function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { 25function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
27 return sequelizeTypescript.transaction(async t => { 26 return sequelizeTypescript.transaction(async t => {
28 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) 27 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
29 28
@@ -31,7 +30,11 @@ function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, b
31 }) 30 })
32} 31}
33 32
34function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) { 33function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) {
34 if (redundancyModel.actorId !== byActor.id) {
35 throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
36 }
37
35 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) 38 const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
36 39
37 redundancyModel.set('expires', attributes.expiresOn) 40 redundancyModel.set('expires', attributes.expiresOn)
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts
index 046370b79..89bda9c32 100644
--- a/server/lib/activitypub/process/process-accept.ts
+++ b/server/lib/activitypub/process/process-accept.ts
@@ -1,15 +1,11 @@
1import { ActivityAccept } from '../../../../shared/models/activitypub' 1import { ActivityAccept } from '../../../../shared/models/activitypub'
2import { getActorUrl } from '../../../helpers/activitypub'
3import { ActorModel } from '../../../models/activitypub/actor' 2import { ActorModel } from '../../../models/activitypub/actor'
4import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 3import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
5import { addFetchOutboxJob } from '../actor' 4import { addFetchOutboxJob } from '../actor'
6 5
7async function processAcceptActivity (activity: ActivityAccept, inboxActor?: ActorModel) { 6async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) {
8 if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') 7 if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.')
9 8
10 const actorUrl = getActorUrl(activity.actor)
11 const targetActor = await ActorModel.loadByUrl(actorUrl)
12
13 return processAccept(inboxActor, targetActor) 9 return processAccept(inboxActor, targetActor)
14} 10}
15 11
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 814556817..cc88b5423 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -2,15 +2,11 @@ import { ActivityAnnounce } from '../../../../shared/models/activitypub'
2import { retryTransactionWrapper } from '../../../helpers/database-utils' 2import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers' 3import { sequelizeTypescript } from '../../../initializers'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoModel } from '../../../models/video/video'
6import { VideoShareModel } from '../../../models/video/video-share' 5import { VideoShareModel } from '../../../models/video/video-share'
7import { getOrCreateActorAndServerAndModel } from '../actor'
8import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
10 8
11async function processAnnounceActivity (activity: ActivityAnnounce) { 9async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) {
12 const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
13
14 return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) 10 return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity)
15} 11}
16 12
@@ -25,7 +21,7 @@ export {
25async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { 21async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
26 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id 22 const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
27 23
28 const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) 24 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri })
29 25
30 return sequelizeTypescript.transaction(async t => { 26 return sequelizeTypescript.transaction(async t => {
31 // Add share entry 27 // Add share entry
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 32e555acf..5197dac73 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -7,30 +7,28 @@ import { sequelizeTypescript } from '../../../initializers'
7import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 7import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
8import { ActorModel } from '../../../models/activitypub/actor' 8import { ActorModel } from '../../../models/activitypub/actor'
9import { VideoAbuseModel } from '../../../models/video/video-abuse' 9import { VideoAbuseModel } from '../../../models/video/video-abuse'
10import { getOrCreateActorAndServerAndModel } from '../actor'
11import { addVideoComment, resolveThread } from '../video-comments' 10import { addVideoComment, resolveThread } from '../video-comments'
12import { getOrCreateVideoAndAccountAndChannel } from '../videos' 11import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' 12import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
14import { Redis } from '../../redis' 13import { Redis } from '../../redis'
15import { createCacheFile } from '../cache-file' 14import { createCacheFile } from '../cache-file'
16 15
17async function processCreateActivity (activity: ActivityCreate) { 16async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
18 const activityObject = activity.object 17 const activityObject = activity.object
19 const activityType = activityObject.type 18 const activityType = activityObject.type
20 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
21 19
22 if (activityType === 'View') { 20 if (activityType === 'View') {
23 return processCreateView(actor, activity) 21 return processCreateView(byActor, activity)
24 } else if (activityType === 'Dislike') { 22 } else if (activityType === 'Dislike') {
25 return retryTransactionWrapper(processCreateDislike, actor, activity) 23 return retryTransactionWrapper(processCreateDislike, byActor, activity)
26 } else if (activityType === 'Video') { 24 } else if (activityType === 'Video') {
27 return processCreateVideo(activity) 25 return processCreateVideo(activity)
28 } else if (activityType === 'Flag') { 26 } else if (activityType === 'Flag') {
29 return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) 27 return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject)
30 } else if (activityType === 'Note') { 28 } else if (activityType === 'Note') {
31 return retryTransactionWrapper(processCreateVideoComment, actor, activity) 29 return retryTransactionWrapper(processCreateVideoComment, byActor, activity)
32 } else if (activityType === 'CacheFile') { 30 } else if (activityType === 'CacheFile') {
33 return retryTransactionWrapper(processCacheFile, actor, activity) 31 return retryTransactionWrapper(processCacheFile, byActor, activity)
34 } 32 }
35 33
36 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) 34 logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -48,7 +46,7 @@ export {
48async function processCreateVideo (activity: ActivityCreate) { 46async function processCreateVideo (activity: ActivityCreate) {
49 const videoToCreateData = activity.object as VideoTorrentObject 47 const videoToCreateData = activity.object as VideoTorrentObject
50 48
51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) 49 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData })
52 50
53 return video 51 return video
54} 52}
@@ -59,7 +57,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
59 57
60 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 58 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
61 59
62 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) 60 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
63 61
64 return sequelizeTypescript.transaction(async t => { 62 return sequelizeTypescript.transaction(async t => {
65 const rate = { 63 const rate = {
@@ -86,10 +84,14 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
86async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { 84async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
87 const view = activity.object as ViewObject 85 const view = activity.object as ViewObject
88 86
89 const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) 87 const options = {
88 videoObject: view.object,
89 fetchType: 'only-video' as 'only-video'
90 }
91 const { video } = await getOrCreateVideoAndAccountAndChannel(options)
90 92
91 const actor = await ActorModel.loadByUrl(view.actor) 93 const actorExists = await ActorModel.isActorUrlExist(view.actor)
92 if (!actor) throw new Error('Unknown actor ' + view.actor) 94 if (actorExists === false) throw new Error('Unknown actor ' + view.actor)
93 95
94 await Redis.Instance.addVideoView(video.id) 96 await Redis.Instance.addVideoView(video.id)
95 97
@@ -103,7 +105,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
103async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { 105async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
104 const cacheFile = activity.object as CacheFileObject 106 const cacheFile = activity.object as CacheFileObject
105 107
106 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) 108 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
107 109
108 await createCacheFile(cacheFile, video, byActor) 110 await createCacheFile(cacheFile, video, byActor)
109 111
@@ -114,13 +116,13 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate)
114 } 116 }
115} 117}
116 118
117async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { 119async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
118 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) 120 logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
119 121
120 const account = actor.Account 122 const account = byActor.Account
121 if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) 123 if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
122 124
123 const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) 125 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object })
124 126
125 return sequelizeTypescript.transaction(async t => { 127 return sequelizeTypescript.transaction(async t => {
126 const videoAbuseData = { 128 const videoAbuseData = {
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts
index 3c830abea..038d8c4d3 100644
--- a/server/lib/activitypub/process/process-delete.ts
+++ b/server/lib/activitypub/process/process-delete.ts
@@ -7,41 +7,41 @@ import { ActorModel } from '../../../models/activitypub/actor'
7import { VideoModel } from '../../../models/video/video' 7import { VideoModel } from '../../../models/video/video'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { VideoCommentModel } from '../../../models/video/video-comment' 9import { VideoCommentModel } from '../../../models/video/video-comment'
10import { getOrCreateActorAndServerAndModel } from '../actor'
11import { forwardActivity } from '../send/utils' 10import { forwardActivity } from '../send/utils'
12 11
13async function processDeleteActivity (activity: ActivityDelete) { 12async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) {
14 const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id 13 const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id
15 14
16 if (activity.actor === objectUrl) { 15 if (activity.actor === objectUrl) {
17 let actor = await ActorModel.loadByUrl(activity.actor) 16 // We need more attributes (all the account and channel)
18 if (!actor) return undefined 17 const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
19 18
20 if (actor.type === 'Person') { 19 if (byActorFull.type === 'Person') {
21 if (!actor.Account) throw new Error('Actor ' + actor.url + ' is a person but we cannot find it in database.') 20 if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.')
22 21
23 actor.Account.Actor = await actor.Account.$get('Actor') as ActorModel 22 byActorFull.Account.Actor = await byActorFull.Account.$get('Actor') as ActorModel
24 return retryTransactionWrapper(processDeleteAccount, actor.Account) 23 return retryTransactionWrapper(processDeleteAccount, byActorFull.Account)
25 } else if (actor.type === 'Group') { 24 } else if (byActorFull.type === 'Group') {
26 if (!actor.VideoChannel) throw new Error('Actor ' + actor.url + ' is a group but we cannot find it in database.') 25 if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.')
27 26
28 actor.VideoChannel.Actor = await actor.VideoChannel.$get('Actor') as ActorModel 27 byActorFull.VideoChannel.Actor = await byActorFull.VideoChannel.$get('Actor') as ActorModel
29 return retryTransactionWrapper(processDeleteVideoChannel, actor.VideoChannel) 28 return retryTransactionWrapper(processDeleteVideoChannel, byActorFull.VideoChannel)
30 } 29 }
31 } 30 }
32 31
33 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
34 { 32 {
35 const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl) 33 const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl)
36 if (videoCommentInstance) { 34 if (videoCommentInstance) {
37 return retryTransactionWrapper(processDeleteVideoComment, actor, videoCommentInstance, activity) 35 return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity)
38 } 36 }
39 } 37 }
40 38
41 { 39 {
42 const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) 40 const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl)
43 if (videoInstance) { 41 if (videoInstance) {
44 return retryTransactionWrapper(processDeleteVideo, actor, videoInstance) 42 if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)
43
44 return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance)
45 } 45 }
46 } 46 }
47 47
@@ -94,6 +94,10 @@ function processDeleteVideoComment (byActor: ActorModel, videoComment: VideoComm
94 logger.debug('Removing remote video comment "%s".', videoComment.url) 94 logger.debug('Removing remote video comment "%s".', videoComment.url)
95 95
96 return sequelizeTypescript.transaction(async t => { 96 return sequelizeTypescript.transaction(async t => {
97 if (videoComment.Account.id !== byActor.Account.id) {
98 throw new Error('Account ' + byActor.url + ' does not own video comment ' + videoComment.url)
99 }
100
97 await videoComment.destroy({ transaction: t }) 101 await videoComment.destroy({ transaction: t })
98 102
99 if (videoComment.Video.isOwned()) { 103 if (videoComment.Video.isOwned()) {
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts
index f34fd66cc..24c9085f7 100644
--- a/server/lib/activitypub/process/process-follow.ts
+++ b/server/lib/activitypub/process/process-follow.ts
@@ -4,14 +4,12 @@ import { logger } from '../../../helpers/logger'
4import { sequelizeTypescript } from '../../../initializers' 4import { sequelizeTypescript } from '../../../initializers'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 6import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
7import { getOrCreateActorAndServerAndModel } from '../actor'
8import { sendAccept } from '../send' 7import { sendAccept } from '../send'
9 8
10async function processFollowActivity (activity: ActivityFollow) { 9async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) {
11 const activityObject = activity.object 10 const activityObject = activity.object
12 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
13 11
14 return retryTransactionWrapper(processFollow, actor, activityObject) 12 return retryTransactionWrapper(processFollow, byActor, activityObject)
15} 13}
16 14
17// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
@@ -24,7 +22,7 @@ export {
24 22
25async function processFollow (actor: ActorModel, targetActorURL: string) { 23async function processFollow (actor: ActorModel, targetActorURL: string) {
26 await sequelizeTypescript.transaction(async t => { 24 await sequelizeTypescript.transaction(async t => {
27 const targetActor = await ActorModel.loadByUrl(targetActorURL, t) 25 const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t)
28 26
29 if (!targetActor) throw new Error('Unknown actor') 27 if (!targetActor) throw new Error('Unknown actor')
30 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') 28 if (targetActor.isOwned() === false) throw new Error('This is not a local actor.')
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 9e1664fd8..f7200db61 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -3,14 +3,11 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers' 3import { sequelizeTypescript } from '../../../initializers'
4import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 4import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { getOrCreateActorAndServerAndModel } from '../actor'
7import { forwardVideoRelatedActivity } from '../send/utils' 6import { forwardVideoRelatedActivity } from '../send/utils'
8import { getOrCreateVideoAndAccountAndChannel } from '../videos' 7import { getOrCreateVideoAndAccountAndChannel } from '../videos'
9 8
10async function processLikeActivity (activity: ActivityLike) { 9async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) {
11 const actor = await getOrCreateActorAndServerAndModel(activity.actor) 10 return retryTransactionWrapper(processLikeVideo, byActor, activity)
12
13 return retryTransactionWrapper(processLikeVideo, actor, activity)
14} 11}
15 12
16// --------------------------------------------------------------------------- 13// ---------------------------------------------------------------------------
@@ -27,7 +24,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
27 const byAccount = byActor.Account 24 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 25 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 26
30 const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) 27 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl })
31 28
32 return sequelizeTypescript.transaction(async t => { 29 return sequelizeTypescript.transaction(async t => {
33 const rate = { 30 const rate = {
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts
index f06b03772..709a65096 100644
--- a/server/lib/activitypub/process/process-reject.ts
+++ b/server/lib/activitypub/process/process-reject.ts
@@ -1,15 +1,11 @@
1import { ActivityReject } from '../../../../shared/models/activitypub/activity' 1import { ActivityReject } from '../../../../shared/models/activitypub/activity'
2import { getActorUrl } from '../../../helpers/activitypub'
3import { sequelizeTypescript } from '../../../initializers' 2import { sequelizeTypescript } from '../../../initializers'
4import { ActorModel } from '../../../models/activitypub/actor' 3import { ActorModel } from '../../../models/activitypub/actor'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 4import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6 5
7async function processRejectActivity (activity: ActivityReject, inboxActor?: ActorModel) { 6async function processRejectActivity (activity: ActivityReject, targetActor: ActorModel, inboxActor?: ActorModel) {
8 if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') 7 if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.')
9 8
10 const actorUrl = getActorUrl(activity.actor)
11 const targetActor = await ActorModel.loadByUrl(actorUrl)
12
13 return processReject(inboxActor, targetActor) 9 return processReject(inboxActor, targetActor)
14} 10}
15 11
@@ -21,11 +17,11 @@ export {
21 17
22// --------------------------------------------------------------------------- 18// ---------------------------------------------------------------------------
23 19
24async function processReject (actor: ActorModel, targetActor: ActorModel) { 20async function processReject (follower: ActorModel, targetActor: ActorModel) {
25 return sequelizeTypescript.transaction(async t => { 21 return sequelizeTypescript.transaction(async t => {
26 const actorFollow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id, t) 22 const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t)
27 23
28 if (!actorFollow) throw new Error(`'Unknown actor follow ${actor.id} -> ${targetActor.id}.`) 24 if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`)
29 25
30 await actorFollow.destroy({ transaction: t }) 26 await actorFollow.destroy({ transaction: t })
31 27
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 0eb5fa392..73ca0a17c 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -1,10 +1,8 @@
1import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' 1import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
2import { DislikeObject } from '../../../../shared/models/activitypub/objects' 2import { DislikeObject } from '../../../../shared/models/activitypub/objects'
3import { getActorUrl } from '../../../helpers/activitypub'
4import { retryTransactionWrapper } from '../../../helpers/database-utils' 3import { retryTransactionWrapper } from '../../../helpers/database-utils'
5import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
6import { sequelizeTypescript } from '../../../initializers' 5import { sequelizeTypescript } from '../../../initializers'
7import { AccountModel } from '../../../models/account/account'
8import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 6import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
9import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
10import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 8import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
@@ -13,29 +11,27 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos'
13import { VideoShareModel } from '../../../models/video/video-share' 11import { VideoShareModel } from '../../../models/video/video-share'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 12import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
15 13
16async function processUndoActivity (activity: ActivityUndo) { 14async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) {
17 const activityToUndo = activity.object 15 const activityToUndo = activity.object
18 16
19 const actorUrl = getActorUrl(activity.actor)
20
21 if (activityToUndo.type === 'Like') { 17 if (activityToUndo.type === 'Like') {
22 return retryTransactionWrapper(processUndoLike, actorUrl, activity) 18 return retryTransactionWrapper(processUndoLike, byActor, activity)
23 } 19 }
24 20
25 if (activityToUndo.type === 'Create') { 21 if (activityToUndo.type === 'Create') {
26 if (activityToUndo.object.type === 'Dislike') { 22 if (activityToUndo.object.type === 'Dislike') {
27 return retryTransactionWrapper(processUndoDislike, actorUrl, activity) 23 return retryTransactionWrapper(processUndoDislike, byActor, activity)
28 } else if (activityToUndo.object.type === 'CacheFile') { 24 } else if (activityToUndo.object.type === 'CacheFile') {
29 return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity) 25 return retryTransactionWrapper(processUndoCacheFile, byActor, activity)
30 } 26 }
31 } 27 }
32 28
33 if (activityToUndo.type === 'Follow') { 29 if (activityToUndo.type === 'Follow') {
34 return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) 30 return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo)
35 } 31 }
36 32
37 if (activityToUndo.type === 'Announce') { 33 if (activityToUndo.type === 'Announce') {
38 return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) 34 return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo)
39 } 35 }
40 36
41 logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) 37 logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id })
@@ -51,66 +47,63 @@ export {
51 47
52// --------------------------------------------------------------------------- 48// ---------------------------------------------------------------------------
53 49
54async function processUndoLike (actorUrl: string, activity: ActivityUndo) { 50async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) {
55 const likeActivity = activity.object as ActivityLike 51 const likeActivity = activity.object as ActivityLike
56 52
57 const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) 53 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object })
58 54
59 return sequelizeTypescript.transaction(async t => { 55 return sequelizeTypescript.transaction(async t => {
60 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 56 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
61 if (!byAccount) throw new Error('Unknown account ' + actorUrl)
62 57
63 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) 58 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
64 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) 59 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
65 60
66 await rate.destroy({ transaction: t }) 61 await rate.destroy({ transaction: t })
67 await video.decrement('likes', { transaction: t }) 62 await video.decrement('likes', { transaction: t })
68 63
69 if (video.isOwned()) { 64 if (video.isOwned()) {
70 // Don't resend the activity to the sender 65 // Don't resend the activity to the sender
71 const exceptions = [ byAccount.Actor ] 66 const exceptions = [ byActor ]
72 67
73 await forwardVideoRelatedActivity(activity, t, exceptions, video) 68 await forwardVideoRelatedActivity(activity, t, exceptions, video)
74 } 69 }
75 }) 70 })
76} 71}
77 72
78async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { 73async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) {
79 const dislike = activity.object.object as DislikeObject 74 const dislike = activity.object.object as DislikeObject
80 75
81 const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) 76 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object })
82 77
83 return sequelizeTypescript.transaction(async t => { 78 return sequelizeTypescript.transaction(async t => {
84 const byAccount = await AccountModel.loadByUrl(actorUrl, t) 79 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
85 if (!byAccount) throw new Error('Unknown account ' + actorUrl)
86 80
87 const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) 81 const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t)
88 if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) 82 if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`)
89 83
90 await rate.destroy({ transaction: t }) 84 await rate.destroy({ transaction: t })
91 await video.decrement('dislikes', { transaction: t }) 85 await video.decrement('dislikes', { transaction: t })
92 86
93 if (video.isOwned()) { 87 if (video.isOwned()) {
94 // Don't resend the activity to the sender 88 // Don't resend the activity to the sender
95 const exceptions = [ byAccount.Actor ] 89 const exceptions = [ byActor ]
96 90
97 await forwardVideoRelatedActivity(activity, t, exceptions, video) 91 await forwardVideoRelatedActivity(activity, t, exceptions, video)
98 } 92 }
99 }) 93 })
100} 94}
101 95
102async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { 96async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo) {
103 const cacheFileObject = activity.object.object as CacheFileObject 97 const cacheFileObject = activity.object.object as CacheFileObject
104 98
105 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) 99 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object })
106 100
107 return sequelizeTypescript.transaction(async t => { 101 return sequelizeTypescript.transaction(async t => {
108 const byActor = await ActorModel.loadByUrl(actorUrl)
109 if (!byActor) throw new Error('Unknown actor ' + actorUrl)
110
111 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 102 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
112 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) 103 if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
113 104
105 if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.')
106
114 await cacheFile.destroy() 107 await cacheFile.destroy()
115 108
116 if (video.isOwned()) { 109 if (video.isOwned()) {
@@ -122,10 +115,9 @@ async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
122 }) 115 })
123} 116}
124 117
125function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { 118function processUndoFollow (follower: ActorModel, followActivity: ActivityFollow) {
126 return sequelizeTypescript.transaction(async t => { 119 return sequelizeTypescript.transaction(async t => {
127 const follower = await ActorModel.loadByUrl(actorUrl, t) 120 const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t)
128 const following = await ActorModel.loadByUrl(followActivity.object, t)
129 const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) 121 const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t)
130 122
131 if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`) 123 if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`)
@@ -136,11 +128,8 @@ function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
136 }) 128 })
137} 129}
138 130
139function processUndoAnnounce (actorUrl: string, announceActivity: ActivityAnnounce) { 131function processUndoAnnounce (byActor: ActorModel, announceActivity: ActivityAnnounce) {
140 return sequelizeTypescript.transaction(async t => { 132 return sequelizeTypescript.transaction(async t => {
141 const byActor = await ActorModel.loadByUrl(actorUrl, t)
142 if (!byActor) throw new Error('Unknown actor ' + actorUrl)
143
144 const share = await VideoShareModel.loadByUrl(announceActivity.id, t) 133 const share = await VideoShareModel.loadByUrl(announceActivity.id, t)
145 if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`) 134 if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`)
146 135
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index d3af1a181..ed3489ebf 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -6,27 +6,30 @@ import { sequelizeTypescript } from '../../../initializers'
6import { AccountModel } from '../../../models/account/account' 6import { AccountModel } from '../../../models/account/account'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' 9import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
10import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos' 10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' 11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 13import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
14import { createCacheFile, updateCacheFile } from '../cache-file' 14import { createCacheFile, updateCacheFile } from '../cache-file'
15 15
16async function processUpdateActivity (activity: ActivityUpdate) { 16async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
17 const actor = await getOrCreateActorAndServerAndModel(activity.actor)
18 const objectType = activity.object.type 17 const objectType = activity.object.type
19 18
20 if (objectType === 'Video') { 19 if (objectType === 'Video') {
21 return retryTransactionWrapper(processUpdateVideo, actor, activity) 20 return retryTransactionWrapper(processUpdateVideo, byActor, activity)
22 } 21 }
23 22
24 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { 23 if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
25 return retryTransactionWrapper(processUpdateActor, actor, activity) 24 // We need more attributes
25 const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
26 return retryTransactionWrapper(processUpdateActor, byActorFull, activity)
26 } 27 }
27 28
28 if (objectType === 'CacheFile') { 29 if (objectType === 'CacheFile') {
29 return retryTransactionWrapper(processUpdateCacheFile, actor, activity) 30 // We need more attributes
31 const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
32 return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity)
30 } 33 }
31 34
32 return undefined 35 return undefined
@@ -48,10 +51,18 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
48 return undefined 51 return undefined
49 } 52 }
50 53
51 const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) 54 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id })
52 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 55 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
53 56
54 return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) 57 const updateOptions = {
58 video,
59 videoObject,
60 account: actor.Account,
61 channel: channelActor.VideoChannel,
62 updateViews: true,
63 overrideTo: activity.to
64 }
65 return updateVideoFromAP(updateOptions)
55} 66}
56 67
57async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { 68async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
@@ -64,7 +75,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp
64 75
65 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 76 const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
66 if (!redundancyModel) { 77 if (!redundancyModel) {
67 const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) 78 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id })
68 return createCacheFile(cacheFileObject, video, byActor) 79 return createCacheFile(cacheFileObject, video, byActor)
69 } 80 }
70 81
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index da91675ce..b263f1ea2 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -11,8 +11,9 @@ import { processLikeActivity } from './process-like'
11import { processRejectActivity } from './process-reject' 11import { processRejectActivity } from './process-reject'
12import { processUndoActivity } from './process-undo' 12import { processUndoActivity } from './process-undo'
13import { processUpdateActivity } from './process-update' 13import { processUpdateActivity } from './process-update'
14import { getOrCreateActorAndServerAndModel } from '../actor'
14 15
15const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?: ActorModel) => Promise<any> } = { 16const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = {
16 Create: processCreateActivity, 17 Create: processCreateActivity,
17 Update: processUpdateActivity, 18 Update: processUpdateActivity,
18 Delete: processDeleteActivity, 19 Delete: processDeleteActivity,
@@ -25,7 +26,14 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?
25} 26}
26 27
27async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { 28async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) {
29 const actorsCache: { [ url: string ]: ActorModel } = {}
30
28 for (const activity of activities) { 31 for (const activity of activities) {
32 if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) {
33 logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type)
34 continue
35 }
36
29 const actorUrl = getActorUrl(activity.actor) 37 const actorUrl = getActorUrl(activity.actor)
30 38
31 // When we fetch remote data, we don't have signature 39 // When we fetch remote data, we don't have signature
@@ -34,6 +42,9 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
34 continue 42 continue
35 } 43 }
36 44
45 const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl)
46 actorsCache[actorUrl] = byActor
47
37 const activityProcessor = processActivity[activity.type] 48 const activityProcessor = processActivity[activity.type]
38 if (activityProcessor === undefined) { 49 if (activityProcessor === undefined) {
39 logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) 50 logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id })
@@ -41,7 +52,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor
41 } 52 }
42 53
43 try { 54 try {
44 await activityProcessor(activity, inboxActor) 55 await activityProcessor(activity, byActor, inboxActor)
45 } catch (err) { 56 } catch (err) {
46 logger.warn('Cannot process activity %s.', activity.type, { err }) 57 logger.warn('Cannot process activity %s.', activity.type, { err })
47 } 58 }
diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts
index f137217f8..cd0cab7ee 100644
--- a/server/lib/activitypub/send/send-announce.ts
+++ b/server/lib/activitypub/send/send-announce.ts
@@ -4,14 +4,14 @@ import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video' 4import { VideoModel } from '../../../models/video/video'
5import { VideoShareModel } from '../../../models/video/video-share' 5import { VideoShareModel } from '../../../models/video/video-share'
6import { broadcastToFollowers } from './utils' 6import { broadcastToFollowers } from './utils'
7import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' 7import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9 9
10async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { 10async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
11 const announcedObject = video.url 11 const announcedObject = video.url
12 12
13 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 13 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
14 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 14 const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
15 15
16 const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) 16 const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
17 17
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 6f89b1a22..285edba3b 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -1,21 +1,13 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' 2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
3import { VideoPrivacy } from '../../../../shared/models/videos' 3import { VideoPrivacy } from '../../../../shared/models/videos'
4import { getServerActor } from '../../../helpers/utils'
5import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
6import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
7import { VideoAbuseModel } from '../../../models/video/video-abuse' 6import { VideoAbuseModel } from '../../../models/video/video-abuse'
8import { VideoCommentModel } from '../../../models/video/video-comment' 7import { VideoCommentModel } from '../../../models/video/video-comment'
9import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' 8import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url'
10import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' 9import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
11import { 10import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
12 audiencify,
13 getActorsInvolvedInVideo,
14 getAudience,
15 getObjectFollowersAudience,
16 getVideoAudience,
17 getVideoCommentAudience
18} from '../audience'
19import { logger } from '../../../helpers/logger' 11import { logger } from '../../../helpers/logger'
20import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 12import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
21 13
@@ -40,6 +32,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
40 32
41 logger.info('Creating job to send video abuse %s.', url) 33 logger.info('Creating job to send video abuse %s.', url)
42 34
35 // Custom audience, we only send the abuse to the origin instance
43 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } 36 const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
44 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) 37 const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
45 38
@@ -49,15 +42,15 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
49async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { 42async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
50 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
51 44
52 const redundancyObject = fileRedundancy.toActivityPubObject()
53
54 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) 45 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
55 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) 46 const redundancyObject = fileRedundancy.toActivityPubObject()
56
57 const audience = getVideoAudience(video, actorsInvolvedInVideo)
58 const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
59 47
60 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 48 return sendVideoRelatedCreateActivity({
49 byActor,
50 video,
51 url: fileRedundancy.url,
52 object: redundancyObject
53 })
61} 54}
62 55
63async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { 56async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
@@ -70,6 +63,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
70 const commentObject = comment.toActivityPubObject(threadParentComments) 63 const commentObject = comment.toActivityPubObject(threadParentComments)
71 64
72 const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t) 65 const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t)
66 // Add the actor that commented too
73 actorsInvolvedInComment.push(byActor) 67 actorsInvolvedInComment.push(byActor)
74 68
75 const parentsCommentActors = threadParentComments.map(c => c.Account.Actor) 69 const parentsCommentActors = threadParentComments.map(c => c.Account.Actor)
@@ -78,7 +72,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
78 if (isOrigin) { 72 if (isOrigin) {
79 audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) 73 audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin)
80 } else { 74 } else {
81 audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) 75 audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors))
82 } 76 }
83 77
84 const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) 78 const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
@@ -103,24 +97,14 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa
103 const url = getVideoViewActivityPubUrl(byActor, video) 97 const url = getVideoViewActivityPubUrl(byActor, video)
104 const viewActivity = buildViewActivity(byActor, video) 98 const viewActivity = buildViewActivity(byActor, video)
105 99
106 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 100 return sendVideoRelatedCreateActivity({
107 101 // Use the server actor to send the view
108 // Send to origin 102 byActor,
109 if (video.isOwned() === false) { 103 video,
110 const audience = getVideoAudience(video, actorsInvolvedInVideo) 104 url,
111 const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) 105 object: viewActivity,
112 106 transaction: t
113 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 107 })
114 }
115
116 // Send to followers
117 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
118 const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
119
120 // Use the server actor to send the view
121 const serverActor = await getServerActor()
122 const actorsException = [ byActor ]
123 return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
124} 108}
125 109
126async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { 110async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
@@ -129,22 +113,13 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra
129 const url = getVideoDislikeActivityPubUrl(byActor, video) 113 const url = getVideoDislikeActivityPubUrl(byActor, video)
130 const dislikeActivity = buildDislikeActivity(byActor, video) 114 const dislikeActivity = buildDislikeActivity(byActor, video)
131 115
132 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 116 return sendVideoRelatedCreateActivity({
133 117 byActor,
134 // Send to origin 118 video,
135 if (video.isOwned() === false) { 119 url,
136 const audience = getVideoAudience(video, actorsInvolvedInVideo) 120 object: dislikeActivity,
137 const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) 121 transaction: t
138 122 })
139 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
140 }
141
142 // Send to followers
143 const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
144 const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
145
146 const actorsException = [ byActor ]
147 return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
148} 123}
149 124
150function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { 125function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
@@ -189,3 +164,19 @@ export {
189 sendCreateVideoComment, 164 sendCreateVideoComment,
190 sendCreateCacheFile 165 sendCreateCacheFile
191} 166}
167
168// ---------------------------------------------------------------------------
169
170async function sendVideoRelatedCreateActivity (options: {
171 byActor: ActorModel,
172 video: VideoModel,
173 url: string,
174 object: any,
175 transaction?: Transaction
176}) {
177 const activityBuilder = (audience: ActivityAudience) => {
178 return buildCreateActivity(options.url, options.byActor, options.object, audience)
179 }
180
181 return sendVideoRelatedActivity(activityBuilder, options)
182}
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 479182543..18969433a 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -5,21 +5,22 @@ import { VideoModel } from '../../../models/video/video'
5import { VideoCommentModel } from '../../../models/video/video-comment' 5import { VideoCommentModel } from '../../../models/video/video-comment'
6import { VideoShareModel } from '../../../models/video/video-share' 6import { VideoShareModel } from '../../../models/video/video-share'
7import { getDeleteActivityPubUrl } from '../url' 7import { getDeleteActivityPubUrl } from '../url'
8import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' 8import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
9import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' 9import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience'
10import { logger } from '../../../helpers/logger' 10import { logger } from '../../../helpers/logger'
11 11
12async function sendDeleteVideo (video: VideoModel, t: Transaction) { 12async function sendDeleteVideo (video: VideoModel, transaction: Transaction) {
13 logger.info('Creating job to broadcast delete of video %s.', video.url) 13 logger.info('Creating job to broadcast delete of video %s.', video.url)
14 14
15 const url = getDeleteActivityPubUrl(video.url)
16 const byActor = video.VideoChannel.Account.Actor 15 const byActor = video.VideoChannel.Account.Actor
17 16
18 const activity = buildDeleteActivity(url, video.url, byActor) 17 const activityBuilder = (audience: ActivityAudience) => {
18 const url = getDeleteActivityPubUrl(video.url)
19 19
20 const actorsInvolved = await getActorsInvolvedInVideo(video, t) 20 return buildDeleteActivity(url, video.url, byActor, audience)
21 }
21 22
22 return broadcastToFollowers(activity, byActor, actorsInvolved, t) 23 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction })
23} 24}
24 25
25async function sendDeleteActor (byActor: ActorModel, t: Transaction) { 26async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts
index a5408ac6a..89307acc6 100644
--- a/server/lib/activitypub/send/send-like.ts
+++ b/server/lib/activitypub/send/send-like.ts
@@ -3,31 +3,20 @@ import { ActivityAudience, ActivityLike } from '../../../../shared/models/activi
3import { ActorModel } from '../../../models/activitypub/actor' 3import { ActorModel } from '../../../models/activitypub/actor'
4import { VideoModel } from '../../../models/video/video' 4import { VideoModel } from '../../../models/video/video'
5import { getVideoLikeActivityPubUrl } from '../url' 5import { getVideoLikeActivityPubUrl } from '../url'
6import { broadcastToFollowers, unicastTo } from './utils' 6import { sendVideoRelatedActivity } from './utils'
7import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' 7import { audiencify, getAudience } from '../audience'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9 9
10async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) { 10async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
11 logger.info('Creating job to like %s.', video.url) 11 logger.info('Creating job to like %s.', video.url)
12 12
13 const url = getVideoLikeActivityPubUrl(byActor, video) 13 const activityBuilder = (audience: ActivityAudience) => {
14 const url = getVideoLikeActivityPubUrl(byActor, video)
14 15
15 const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 16 return buildLikeActivity(url, byActor, video, audience)
16
17 // Send to origin
18 if (video.isOwned() === false) {
19 const audience = getVideoAudience(video, accountsInvolvedInVideo)
20 const data = buildLikeActivity(url, byActor, video, audience)
21
22 return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
23 } 17 }
24 18
25 // Send to followers 19 return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t })
26 const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
27 const activity = buildLikeActivity(url, byActor, video, audience)
28
29 const followersException = [ byActor ]
30 return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
31} 20}
32 21
33function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { 22function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index a50673c79..5236d2cb3 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -11,8 +11,8 @@ import { ActorModel } from '../../../models/activitypub/actor'
11import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 11import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
12import { VideoModel } from '../../../models/video/video' 12import { VideoModel } from '../../../models/video/video'
13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' 13import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
14import { broadcastToFollowers, unicastTo } from './utils' 14import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
15import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' 15import { audiencify, getAudience } from '../audience'
16import { buildCreateActivity, buildDislikeActivity } from './send-create' 16import { buildCreateActivity, buildDislikeActivity } from './send-create'
17import { buildFollowActivity } from './send-follow' 17import { buildFollowActivity } from './send-follow'
18import { buildLikeActivity } from './send-like' 18import { buildLikeActivity } from './send-like'
@@ -39,79 +39,44 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
39 return unicastTo(undoActivity, me, following.inboxUrl) 39 return unicastTo(undoActivity, me, following.inboxUrl)
40} 40}
41 41
42async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { 42async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
43 logger.info('Creating job to undo a like of video %s.', video.url) 43 logger.info('Creating job to undo announce %s.', videoShare.url)
44 44
45 const likeUrl = getVideoLikeActivityPubUrl(byActor, video) 45 const undoUrl = getUndoActivityPubUrl(videoShare.url)
46 const undoUrl = getUndoActivityPubUrl(likeUrl)
47 46
48 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) 47 const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
49 const likeActivity = buildLikeActivity(likeUrl, byActor, video) 48 const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
50 49
51 // Send to origin 50 const followersException = [ byActor ]
52 if (video.isOwned() === false) { 51 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
53 const audience = getVideoAudience(video, actorsInvolvedInVideo) 52}
54 const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
55 53
56 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 54async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
57 } 55 logger.info('Creating job to undo a like of video %s.', video.url)
58 56
59 const audience = getObjectFollowersAudience(actorsInvolvedInVideo) 57 const likeUrl = getVideoLikeActivityPubUrl(byActor, video)
60 const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) 58 const likeActivity = buildLikeActivity(likeUrl, byActor, video)
61 59
62 const followersException = [ byActor ] 60 return sendUndoVideoRelatedActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t })
63 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
64} 61}
65 62
66async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { 63async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
67 logger.info('Creating job to undo a dislike of video %s.', video.url) 64 logger.info('Creating job to undo a dislike of video %s.', video.url)
68 65
69 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) 66 const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video)
70 const undoUrl = getUndoActivityPubUrl(dislikeUrl)
71
72 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
73 const dislikeActivity = buildDislikeActivity(byActor, video) 67 const dislikeActivity = buildDislikeActivity(byActor, video)
74 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) 68 const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
75 69
76 if (video.isOwned() === false) { 70 return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t })
77 const audience = getVideoAudience(video, actorsInvolvedInVideo)
78 const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
79
80 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
81 }
82
83 const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
84
85 const followersException = [ byActor ]
86 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
87}
88
89async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
90 logger.info('Creating job to undo announce %s.', videoShare.url)
91
92 const undoUrl = getUndoActivityPubUrl(videoShare.url)
93
94 const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
95 const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
96
97 const followersException = [ byActor ]
98 return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
99} 71}
100 72
101async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
102 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
103 75
104 const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
105
106 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
107 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
108
109 const audience = getVideoAudience(video, actorsInvolvedInVideo)
110 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
111 78
112 const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience) 79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
113
114 return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
115} 80}
116 81
117// --------------------------------------------------------------------------- 82// ---------------------------------------------------------------------------
@@ -144,3 +109,19 @@ function undoActivityData (
144 audience 109 audience
145 ) 110 )
146} 111}
112
113async function sendUndoVideoRelatedActivity (options: {
114 byActor: ActorModel,
115 video: VideoModel,
116 url: string,
117 activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce,
118 transaction: Transaction
119}) {
120 const activityBuilder = (audience: ActivityAudience) => {
121 const undoUrl = getUndoActivityPubUrl(options.url)
122
123 return undoActivityData(undoUrl, options.byActor, options.activity, audience)
124 }
125
126 return sendVideoRelatedActivity(activityBuilder, options)
127}
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 605473338..ec46789b7 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video'
7import { VideoChannelModel } from '../../../models/video/video-channel' 7import { VideoChannelModel } from '../../../models/video/video-channel'
8import { VideoShareModel } from '../../../models/video/video-share' 8import { VideoShareModel } from '../../../models/video/video-share'
9import { getUpdateActivityPubUrl } from '../url' 9import { getUpdateActivityPubUrl } from '../url'
10import { broadcastToFollowers, unicastTo } from './utils' 10import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
11import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' 11import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience'
12import { logger } from '../../../helpers/logger' 12import { logger } from '../../../helpers/logger'
13import { VideoCaptionModel } from '../../../models/video/video-caption' 13import { VideoCaptionModel } from '../../../models/video/video-caption'
14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' 14import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
@@ -61,16 +61,16 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 63
64 const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
65 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
66 65
67 const redundancyObject = redundancyModel.toActivityPubObject() 66 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject()
68 const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
68 69
69 const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) 70 return buildUpdateActivity(url, byActor, redundancyObject, audience)
70 const audience = getObjectFollowersAudience(accountsInvolvedInVideo) 71 }
71 72
72 const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience) 73 return sendVideoRelatedActivity(activityBuilder, { byActor, video })
73 return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
74} 74}
75 75
76// --------------------------------------------------------------------------- 76// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index c20c15633..69706e620 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -1,13 +1,36 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { Activity } from '../../../../shared/models/activitypub' 2import { Activity, ActivityAudience } from '../../../../shared/models/activitypub'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { JobQueue } from '../../job-queue' 6import { JobQueue } from '../../job-queue'
7import { VideoModel } from '../../../models/video/video' 7import { VideoModel } from '../../../models/video/video'
8import { getActorsInvolvedInVideo } from '../audience' 8import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
9import { getServerActor } from '../../../helpers/utils' 9import { getServerActor } from '../../../helpers/utils'
10 10
11async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
12 byActor: ActorModel,
13 video: VideoModel,
14 transaction?: Transaction
15}) {
16 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(options.video, options.transaction)
17
18 // Send to origin
19 if (options.video.isOwned() === false) {
20 const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
21 const activity = activityBuilder(audience)
22
23 return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
24 }
25
26 // Send to followers
27 const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
28 const activity = activityBuilder(audience)
29
30 const actorsException = [ options.byActor ]
31 return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
32}
33
11async function forwardVideoRelatedActivity ( 34async function forwardVideoRelatedActivity (
12 activity: Activity, 35 activity: Activity,
13 t: Transaction, 36 t: Transaction,
@@ -110,7 +133,8 @@ export {
110 unicastTo, 133 unicastTo,
111 forwardActivity, 134 forwardActivity,
112 broadcastToActors, 135 broadcastToActors,
113 forwardVideoRelatedActivity 136 forwardVideoRelatedActivity,
137 sendVideoRelatedActivity
114} 138}
115 139
116// --------------------------------------------------------------------------- 140// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index ffbd3a64e..4ca8bf659 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
94 try { 94 try {
95 // Maybe it's a reply to a video? 95 // Maybe it's a reply to a video?
96 // If yes, it's done: we resolved all the thread 96 // If yes, it's done: we resolved all the thread
97 const { video } = await getOrCreateVideoAndAccountAndChannel(url) 97 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url })
98 98
99 if (comments.length !== 0) { 99 if (comments.length !== 0) {
100 const firstReply = comments[ comments.length - 1 ] 100 const firstReply = comments[ comments.length - 1 ]
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 783f78d3e..48c0e0a5c 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import { join } from 'path' 4import { join } from 'path'
5import * as request from 'request' 5import * as request from 'request'
6import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' 6import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index'
7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 7import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
8import { VideoPrivacy } from '../../../shared/models/videos' 8import { VideoPrivacy } from '../../../shared/models/videos'
9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 9import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub
28import { createRates } from './video-rates' 28import { createRates } from './video-rates'
29import { addVideoShares, shareVideoByServerAndChannel } from './share' 29import { addVideoShares, shareVideoByServerAndChannel } from './share'
30import { AccountModel } from '../../models/account/account' 30import { AccountModel } from '../../models/account/account'
31import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31 32
32async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 33async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
33 // If the video is not private and published, we federate it 34 // If the video is not private and published, we federate it
@@ -50,18 +51,29 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr
50 } 51 }
51} 52}
52 53
53function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { 54async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
54 const host = video.VideoChannel.Account.Actor.Server.host 55 const options = {
56 uri: videoUrl,
57 method: 'GET',
58 json: true,
59 activityPub: true
60 }
55 61
56 // We need to provide a callback, if no we could have an uncaught exception 62 logger.info('Fetching remote video %s.', videoUrl)
57 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { 63
58 if (err) reject(err) 64 const { response, body } = await doRequest(options)
59 }) 65
66 if (sanitizeAndCheckVideoTorrentObject(body) === false) {
67 logger.debug('Remote video JSON is not valid.', { body })
68 return { response, videoObject: undefined }
69 }
70
71 return { response, videoObject: body }
60} 72}
61 73
62async function fetchRemoteVideoDescription (video: VideoModel) { 74async function fetchRemoteVideoDescription (video: VideoModel) {
63 const host = video.VideoChannel.Account.Actor.Server.host 75 const host = video.VideoChannel.Account.Actor.Server.host
64 const path = video.getDescriptionPath() 76 const path = video.getDescriptionAPIPath()
65 const options = { 77 const options = {
66 uri: REMOTE_SCHEME.HTTP + '://' + host + path, 78 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
67 json: true 79 json: true
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
71 return body.description ? body.description : '' 83 return body.description ? body.description : ''
72} 84}
73 85
86function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
87 const host = video.VideoChannel.Account.Actor.Server.host
88
89 // We need to provide a callback, if no we could have an uncaught exception
90 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
91 if (err) reject(err)
92 })
93}
94
74function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { 95function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
75 const thumbnailName = video.getThumbnailName() 96 const thumbnailName = video.getThumbnailName()
76 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) 97 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
@@ -82,144 +103,11 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
82 return doRequestAndSaveToFile(options, thumbnailPath) 103 return doRequestAndSaveToFile(options, thumbnailPath)
83} 104}
84 105
85async function videoActivityObjectToDBAttributes (
86 videoChannel: VideoChannelModel,
87 videoObject: VideoTorrentObject,
88 to: string[] = []
89) {
90 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
91 const duration = videoObject.duration.replace(/[^\d]+/, '')
92
93 let language: string | undefined
94 if (videoObject.language) {
95 language = videoObject.language.identifier
96 }
97
98 let category: number | undefined
99 if (videoObject.category) {
100 category = parseInt(videoObject.category.identifier, 10)
101 }
102
103 let licence: number | undefined
104 if (videoObject.licence) {
105 licence = parseInt(videoObject.licence.identifier, 10)
106 }
107
108 const description = videoObject.content || null
109 const support = videoObject.support || null
110
111 return {
112 name: videoObject.name,
113 uuid: videoObject.uuid,
114 url: videoObject.id,
115 category,
116 licence,
117 language,
118 description,
119 support,
120 nsfw: videoObject.sensitive,
121 commentsEnabled: videoObject.commentsEnabled,
122 waitTranscoding: videoObject.waitTranscoding,
123 state: videoObject.state,
124 channelId: videoChannel.id,
125 duration: parseInt(duration, 10),
126 createdAt: new Date(videoObject.published),
127 publishedAt: new Date(videoObject.published),
128 // FIXME: updatedAt does not seems to be considered by Sequelize
129 updatedAt: new Date(videoObject.updated),
130 views: videoObject.views,
131 likes: 0,
132 dislikes: 0,
133 remote: true,
134 privacy
135 }
136}
137
138function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
139 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
140
141 if (fileUrls.length === 0) {
142 throw new Error('Cannot find video files for ' + videoCreated.url)
143 }
144
145 const attributes: VideoFileModel[] = []
146 for (const fileUrl of fileUrls) {
147 // Fetch associated magnet uri
148 const magnet = videoObject.url.find(u => {
149 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
150 })
151
152 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
153
154 const parsed = magnetUtil.decode(magnet.href)
155 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
156 throw new Error('Cannot parse magnet URI ' + magnet.href)
157 }
158
159 const attribute = {
160 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
161 infoHash: parsed.infoHash,
162 resolution: fileUrl.height,
163 size: fileUrl.size,
164 videoId: videoCreated.id,
165 fps: fileUrl.fps
166 } as VideoFileModel
167 attributes.push(attribute)
168 }
169
170 return attributes
171}
172
173function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 106function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
174 const channel = videoObject.attributedTo.find(a => a.type === 'Group') 107 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
175 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) 108 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
176 109
177 return getOrCreateActorAndServerAndModel(channel.id) 110 return getOrCreateActorAndServerAndModel(channel.id, 'all')
178}
179
180async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
181 logger.debug('Adding remote video %s.', videoObject.id)
182
183 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
184 const sequelizeOptions = { transaction: t }
185
186 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
187 const video = VideoModel.build(videoData)
188
189 const videoCreated = await video.save(sequelizeOptions)
190
191 // Process files
192 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
193 if (videoFileAttributes.length === 0) {
194 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
195 }
196
197 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
198 await Promise.all(videoFilePromises)
199
200 // Process tags
201 const tags = videoObject.tag.map(t => t.name)
202 const tagInstances = await TagModel.findOrCreateTags(tags, t)
203 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
204
205 // Process captions
206 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
207 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
208 })
209 await Promise.all(videoCaptionsPromises)
210
211 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
212
213 videoCreated.VideoChannel = channelActor.VideoChannel
214 return videoCreated
215 })
216
217 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
218 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
219
220 if (waitThumbnail === true) await p
221
222 return videoCreated
223} 111}
224 112
225type SyncParam = { 113type SyncParam = {
@@ -230,28 +118,7 @@ type SyncParam = {
230 thumbnail: boolean 118 thumbnail: boolean
231 refreshVideo: boolean 119 refreshVideo: boolean
232} 120}
233async function getOrCreateVideoAndAccountAndChannel ( 121async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
234 videoObject: VideoTorrentObject | string,
235 syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
236) {
237 const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
238
239 let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
240 if (videoFromDatabase) {
241 const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
242 if (syncParam.refreshVideo === true) videoFromDatabase = await p
243
244 return { video: videoFromDatabase }
245 }
246
247 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
248 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
249
250 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
251 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
252
253 // Process outside the transaction because we could fetch remote data
254
255 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) 122 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
256 123
257 const jobPayloads: ActivitypubHttpFetcherPayload[] = [] 124 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
@@ -285,64 +152,56 @@ async function getOrCreateVideoAndAccountAndChannel (
285 } 152 }
286 153
287 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) 154 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
288
289 return { video }
290} 155}
291 156
292async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { 157async function getOrCreateVideoAndAccountAndChannel (options: {
293 const options = { 158 videoObject: VideoTorrentObject | string,
294 uri: videoUrl, 159 syncParam?: SyncParam,
295 method: 'GET', 160 fetchType?: VideoFetchByUrlType,
296 json: true, 161 refreshViews?: boolean
297 activityPub: true 162}) {
298 } 163 // Default params
299 164 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
300 logger.info('Fetching remote video %s.', videoUrl) 165 const fetchType = options.fetchType || 'all'
301 166 const refreshViews = options.refreshViews || false
302 const { response, body } = await doRequest(options) 167
168 // Get video url
169 const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
170
171 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
172 if (videoFromDatabase) {
173 const refreshOptions = {
174 video: videoFromDatabase,
175 fetchedType: fetchType,
176 syncParam,
177 refreshViews
178 }
179 const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions)
180 if (syncParam.refreshVideo === true) videoFromDatabase = await p
303 181
304 if (sanitizeAndCheckVideoTorrentObject(body) === false) { 182 return { video: videoFromDatabase }
305 logger.debug('Remote video JSON is not valid.', { body })
306 return { response, videoObject: undefined }
307 } 183 }
308 184
309 return { response, videoObject: body } 185 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
310} 186 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
311
312async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
313 if (!video.isOutdated()) return video
314
315 try {
316 const { response, videoObject } = await fetchRemoteVideo(video.url)
317 if (response.statusCode === 404) {
318 // Video does not exist anymore
319 await video.destroy()
320 return undefined
321 }
322 187
323 if (videoObject === undefined) { 188 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
324 logger.warn('Cannot refresh remote video: invalid body.') 189 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
325 return video
326 }
327 190
328 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) 191 await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
329 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
330 192
331 return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) 193 return { video }
332 } catch (err) {
333 logger.warn('Cannot refresh video.', { err })
334 return video
335 }
336} 194}
337 195
338async function updateVideoFromAP ( 196async function updateVideoFromAP (options: {
339 video: VideoModel, 197 video: VideoModel,
340 videoObject: VideoTorrentObject, 198 videoObject: VideoTorrentObject,
341 account: AccountModel, 199 account: AccountModel,
342 channel: VideoChannelModel, 200 channel: VideoChannelModel,
201 updateViews: boolean,
343 overrideTo?: string[] 202 overrideTo?: string[]
344) { 203}) {
345 logger.debug('Updating remote video "%s".', videoObject.uuid) 204 logger.debug('Updating remote video "%s".', options.videoObject.uuid)
346 let videoFieldsSave: any 205 let videoFieldsSave: any
347 206
348 try { 207 try {
@@ -351,72 +210,72 @@ async function updateVideoFromAP (
351 transaction: t 210 transaction: t
352 } 211 }
353 212
354 videoFieldsSave = video.toJSON() 213 videoFieldsSave = options.video.toJSON()
355 214
356 // Check actor has the right to update the video 215 // Check actor has the right to update the video
357 const videoChannel = video.VideoChannel 216 const videoChannel = options.video.VideoChannel
358 if (videoChannel.Account.id !== account.id) { 217 if (videoChannel.Account.id !== options.account.id) {
359 throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) 218 throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
360 } 219 }
361 220
362 const to = overrideTo ? overrideTo : videoObject.to 221 const to = options.overrideTo ? options.overrideTo : options.videoObject.to
363 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) 222 const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
364 video.set('name', videoData.name) 223 options.video.set('name', videoData.name)
365 video.set('uuid', videoData.uuid) 224 options.video.set('uuid', videoData.uuid)
366 video.set('url', videoData.url) 225 options.video.set('url', videoData.url)
367 video.set('category', videoData.category) 226 options.video.set('category', videoData.category)
368 video.set('licence', videoData.licence) 227 options.video.set('licence', videoData.licence)
369 video.set('language', videoData.language) 228 options.video.set('language', videoData.language)
370 video.set('description', videoData.description) 229 options.video.set('description', videoData.description)
371 video.set('support', videoData.support) 230 options.video.set('support', videoData.support)
372 video.set('nsfw', videoData.nsfw) 231 options.video.set('nsfw', videoData.nsfw)
373 video.set('commentsEnabled', videoData.commentsEnabled) 232 options.video.set('commentsEnabled', videoData.commentsEnabled)
374 video.set('waitTranscoding', videoData.waitTranscoding) 233 options.video.set('waitTranscoding', videoData.waitTranscoding)
375 video.set('state', videoData.state) 234 options.video.set('state', videoData.state)
376 video.set('duration', videoData.duration) 235 options.video.set('duration', videoData.duration)
377 video.set('createdAt', videoData.createdAt) 236 options.video.set('createdAt', videoData.createdAt)
378 video.set('publishedAt', videoData.publishedAt) 237 options.video.set('publishedAt', videoData.publishedAt)
379 video.set('views', videoData.views) 238 options.video.set('privacy', videoData.privacy)
380 video.set('privacy', videoData.privacy) 239 options.video.set('channelId', videoData.channelId)
381 video.set('channelId', videoData.channelId) 240
382 241 if (options.updateViews === true) options.video.set('views', videoData.views)
383 await video.save(sequelizeOptions) 242 await options.video.save(sequelizeOptions)
384 243
385 // Don't block on request 244 // Don't block on request
386 generateThumbnailFromUrl(video, videoObject.icon) 245 generateThumbnailFromUrl(options.video, options.videoObject.icon)
387 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) 246 .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
388 247
389 // Remove old video files 248 // Remove old video files
390 const videoFileDestroyTasks: Bluebird<void>[] = [] 249 const videoFileDestroyTasks: Bluebird<void>[] = []
391 for (const videoFile of video.VideoFiles) { 250 for (const videoFile of options.video.VideoFiles) {
392 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) 251 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
393 } 252 }
394 await Promise.all(videoFileDestroyTasks) 253 await Promise.all(videoFileDestroyTasks)
395 254
396 const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) 255 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
397 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) 256 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
398 await Promise.all(tasks) 257 await Promise.all(tasks)
399 258
400 // Update Tags 259 // Update Tags
401 const tags = videoObject.tag.map(tag => tag.name) 260 const tags = options.videoObject.tag.map(tag => tag.name)
402 const tagInstances = await TagModel.findOrCreateTags(tags, t) 261 const tagInstances = await TagModel.findOrCreateTags(tags, t)
403 await video.$set('Tags', tagInstances, sequelizeOptions) 262 await options.video.$set('Tags', tagInstances, sequelizeOptions)
404 263
405 // Update captions 264 // Update captions
406 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) 265 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
407 266
408 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 267 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
409 return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) 268 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
410 }) 269 })
411 await Promise.all(videoCaptionsPromises) 270 await Promise.all(videoCaptionsPromises)
412 }) 271 })
413 272
414 logger.info('Remote video with uuid %s updated', videoObject.uuid) 273 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
415 274
416 return updatedVideo 275 return updatedVideo
417 } catch (err) { 276 } catch (err) {
418 if (video !== undefined && videoFieldsSave !== undefined) { 277 if (options.video !== undefined && videoFieldsSave !== undefined) {
419 resetSequelizeInstance(video, videoFieldsSave) 278 resetSequelizeInstance(options.video, videoFieldsSave)
420 } 279 }
421 280
422 // This is just a debug because we will retry the insert 281 // This is just a debug because we will retry the insert
@@ -433,12 +292,7 @@ export {
433 fetchRemoteVideoStaticFile, 292 fetchRemoteVideoStaticFile,
434 fetchRemoteVideoDescription, 293 fetchRemoteVideoDescription,
435 generateThumbnailFromUrl, 294 generateThumbnailFromUrl,
436 videoActivityObjectToDBAttributes, 295 getOrCreateVideoChannelFromVideoObject
437 videoFileActivityUrlToDBAttributes,
438 createVideo,
439 getOrCreateVideoChannelFromVideoObject,
440 addVideoShares,
441 createRates
442} 296}
443 297
444// --------------------------------------------------------------------------- 298// ---------------------------------------------------------------------------
@@ -448,3 +302,178 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo
448 302
449 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') 303 return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
450} 304}
305
306async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
307 logger.debug('Adding remote video %s.', videoObject.id)
308
309 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
310 const sequelizeOptions = { transaction: t }
311
312 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
313 const video = VideoModel.build(videoData)
314
315 const videoCreated = await video.save(sequelizeOptions)
316
317 // Process files
318 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
319 if (videoFileAttributes.length === 0) {
320 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
321 }
322
323 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
324 await Promise.all(videoFilePromises)
325
326 // Process tags
327 const tags = videoObject.tag.map(t => t.name)
328 const tagInstances = await TagModel.findOrCreateTags(tags, t)
329 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
330
331 // Process captions
332 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
333 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
334 })
335 await Promise.all(videoCaptionsPromises)
336
337 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
338
339 videoCreated.VideoChannel = channelActor.VideoChannel
340 return videoCreated
341 })
342
343 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon)
344 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
345
346 if (waitThumbnail === true) await p
347
348 return videoCreated
349}
350
351async function refreshVideoIfNeeded (options: {
352 video: VideoModel,
353 fetchedType: VideoFetchByUrlType,
354 syncParam: SyncParam,
355 refreshViews: boolean
356}): Promise<VideoModel> {
357 if (!options.video.isOutdated()) return options.video
358
359 // We need more attributes if the argument video was fetched with not enough joints
360 const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
361
362 try {
363 const { response, videoObject } = await fetchRemoteVideo(video.url)
364 if (response.statusCode === 404) {
365 // Video does not exist anymore
366 await video.destroy()
367 return undefined
368 }
369
370 if (videoObject === undefined) {
371 logger.warn('Cannot refresh remote video: invalid body.')
372 return video
373 }
374
375 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
376 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
377
378 const updateOptions = {
379 video,
380 videoObject,
381 account,
382 channel: channelActor.VideoChannel,
383 updateViews: options.refreshViews
384 }
385 await updateVideoFromAP(updateOptions)
386 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
387 } catch (err) {
388 logger.warn('Cannot refresh video.', { err })
389 return video
390 }
391}
392
393async function videoActivityObjectToDBAttributes (
394 videoChannel: VideoChannelModel,
395 videoObject: VideoTorrentObject,
396 to: string[] = []
397) {
398 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
399 const duration = videoObject.duration.replace(/[^\d]+/, '')
400
401 let language: string | undefined
402 if (videoObject.language) {
403 language = videoObject.language.identifier
404 }
405
406 let category: number | undefined
407 if (videoObject.category) {
408 category = parseInt(videoObject.category.identifier, 10)
409 }
410
411 let licence: number | undefined
412 if (videoObject.licence) {
413 licence = parseInt(videoObject.licence.identifier, 10)
414 }
415
416 const description = videoObject.content || null
417 const support = videoObject.support || null
418
419 return {
420 name: videoObject.name,
421 uuid: videoObject.uuid,
422 url: videoObject.id,
423 category,
424 licence,
425 language,
426 description,
427 support,
428 nsfw: videoObject.sensitive,
429 commentsEnabled: videoObject.commentsEnabled,
430 waitTranscoding: videoObject.waitTranscoding,
431 state: videoObject.state,
432 channelId: videoChannel.id,
433 duration: parseInt(duration, 10),
434 createdAt: new Date(videoObject.published),
435 publishedAt: new Date(videoObject.published),
436 // FIXME: updatedAt does not seems to be considered by Sequelize
437 updatedAt: new Date(videoObject.updated),
438 views: videoObject.views,
439 likes: 0,
440 dislikes: 0,
441 remote: true,
442 privacy
443 }
444}
445
446function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
447 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
448
449 if (fileUrls.length === 0) {
450 throw new Error('Cannot find video files for ' + videoCreated.url)
451 }
452
453 const attributes: VideoFileModel[] = []
454 for (const fileUrl of fileUrls) {
455 // Fetch associated magnet uri
456 const magnet = videoObject.url.find(u => {
457 return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height
458 })
459
460 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
461
462 const parsed = magnetUtil.decode(magnet.href)
463 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
464 throw new Error('Cannot parse magnet URI ' + magnet.href)
465 }
466
467 const attribute = {
468 extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
469 infoHash: parsed.infoHash,
470 resolution: fileUrl.height,
471 size: fileUrl.size,
472 videoId: videoCreated.id,
473 fps: fileUrl.fps
474 } as VideoFileModel
475 attributes.push(attribute)
476 }
477
478 return attributes
479}
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 5cfb81fc7..14f0a05f5 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -3,23 +3,18 @@ import { sendUpdateActor } from './activitypub/send'
3import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' 3import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers'
4import { updateActorAvatarInstance } from './activitypub' 4import { updateActorAvatarInstance } from './activitypub'
5import { processImage } from '../helpers/image-utils' 5import { processImage } from '../helpers/image-utils'
6import { ActorModel } from '../models/activitypub/actor'
7import { AccountModel } from '../models/account/account' 6import { AccountModel } from '../models/account/account'
8import { VideoChannelModel } from '../models/video/video-channel' 7import { VideoChannelModel } from '../models/video/video-channel'
9import { extname, join } from 'path' 8import { extname, join } from 'path'
10 9
11async function updateActorAvatarFile ( 10async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
12 avatarPhysicalFile: Express.Multer.File,
13 actor: ActorModel,
14 accountOrChannel: AccountModel | VideoChannelModel
15) {
16 const extension = extname(avatarPhysicalFile.filename) 11 const extension = extname(avatarPhysicalFile.filename)
17 const avatarName = actor.uuid + extension 12 const avatarName = accountOrChannel.Actor.uuid + extension
18 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) 13 const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
19 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) 14 await processImage(avatarPhysicalFile, destination, AVATARS_SIZE)
20 15
21 return sequelizeTypescript.transaction(async t => { 16 return sequelizeTypescript.transaction(async t => {
22 const updatedActor = await updateActorAvatarInstance(actor, avatarName, t) 17 const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t)
23 await updatedActor.save({ transaction: t }) 18 await updatedActor.save({ transaction: t })
24 19
25 await sendUpdateActor(accountOrChannel, t) 20 await sendUpdateActor(accountOrChannel, t)
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/cache/videos-caption-cache.ts
index 380d42b2c..f240affbc 100644
--- a/server/lib/cache/videos-caption-cache.ts
+++ b/server/lib/cache/videos-caption-cache.ts
@@ -38,7 +38,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
38 if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') 38 if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
39 39
40 // Used to fetch the path 40 // Used to fetch the path
41 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) 41 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
42 if (!video) return undefined 42 if (!video) return undefined
43 43
44 const remoteStaticPath = videoCaption.getCaptionStaticPath() 44 const remoteStaticPath = videoCaption.getCaptionStaticPath()
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts
index 22b6d9cb0..a5d6f5b62 100644
--- a/server/lib/cache/videos-preview-cache.ts
+++ b/server/lib/cache/videos-preview-cache.ts
@@ -16,7 +16,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
16 } 16 }
17 17
18 async getFilePath (videoUUID: string) { 18 async getFilePath (videoUUID: string) {
19 const video = await VideoModel.loadByUUID(videoUUID) 19 const video = await VideoModel.loadByUUIDWithFile(videoUUID)
20 if (!video) return undefined 20 if (!video) return undefined
21 21
22 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) 22 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
@@ -25,7 +25,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
25 } 25 }
26 26
27 protected async loadRemoteFile (key: string) { 27 protected async loadRemoteFile (key: string) {
28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(key) 28 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
29 if (!video) return undefined 29 if (!video) return undefined
30 30
31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 31 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index a69e09c32..fc013e0c3 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -8,6 +8,7 @@ import { VideoModel } from '../models/video/video'
8import * as validator from 'validator' 8import * as validator from 'validator'
9import { VideoPrivacy } from '../../shared/models/videos' 9import { VideoPrivacy } from '../../shared/models/videos'
10import { readFile } from 'fs-extra' 10import { readFile } from 'fs-extra'
11import { getActivityStreamDuration } from '../models/video/video-format-utils'
11 12
12export class ClientHtml { 13export class ClientHtml {
13 14
@@ -38,10 +39,8 @@ export class ClientHtml {
38 let videoPromise: Bluebird<VideoModel> 39 let videoPromise: Bluebird<VideoModel>
39 40
40 // Let Angular application handle errors 41 // Let Angular application handle errors
41 if (validator.isUUID(videoId, 4)) { 42 if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) {
42 videoPromise = VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) 43 videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
43 } else if (validator.isInt(videoId)) {
44 videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(+videoId)
45 } else { 44 } else {
46 return ClientHtml.getIndexHTML(req, res) 45 return ClientHtml.getIndexHTML(req, res)
47 } 46 }
@@ -150,7 +149,7 @@ export class ClientHtml {
150 description: videoDescriptionEscaped, 149 description: videoDescriptionEscaped,
151 thumbnailUrl: previewUrl, 150 thumbnailUrl: previewUrl,
152 uploadDate: video.createdAt.toISOString(), 151 uploadDate: video.createdAt.toISOString(),
153 duration: video.getActivityStreamDuration(), 152 duration: getActivityStreamDuration(video.duration),
154 contentUrl: videoUrl, 153 contentUrl: videoUrl,
155 embedUrl: embedUrl, 154 embedUrl: embedUrl,
156 interactionCount: video.views 155 interactionCount: video.views
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 72d670277..42217c27c 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,10 +1,10 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { processActivities } from '../../activitypub/process' 3import { processActivities } from '../../activitypub/process'
4import { VideoModel } from '../../../models/video/video'
5import { addVideoShares, createRates } from '../../activitypub/videos'
6import { addVideoComments } from '../../activitypub/video-comments' 4import { addVideoComments } from '../../activitypub/video-comments'
7import { crawlCollectionPage } from '../../activitypub/crawl' 5import { crawlCollectionPage } from '../../activitypub/crawl'
6import { VideoModel } from '../../../models/video/video'
7import { addVideoShares, createRates } from '../../activitypub'
8 8
9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' 9type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments'
10 10
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index c6308f7a6..1463c93fc 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -8,6 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding'
11 12
12export type VideoFilePayload = { 13export type VideoFilePayload = {
13 videoUUID: string 14 videoUUID: string
@@ -25,14 +26,14 @@ async function processVideoFileImport (job: Bull.Job) {
25 const payload = job.data as VideoFileImportPayload 26 const payload = job.data as VideoFileImportPayload
26 logger.info('Processing video file import in job %d.', job.id) 27 logger.info('Processing video file import in job %d.', job.id)
27 28
28 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) 29 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
29 // No video, maybe deleted? 30 // No video, maybe deleted?
30 if (!video) { 31 if (!video) {
31 logger.info('Do not process job %d, video does not exist.', job.id) 32 logger.info('Do not process job %d, video does not exist.', job.id)
32 return undefined 33 return undefined
33 } 34 }
34 35
35 await video.importVideoFile(payload.filePath) 36 await importVideoFile(video, payload.filePath)
36 37
37 await onVideoFileTranscoderOrImportSuccess(video) 38 await onVideoFileTranscoderOrImportSuccess(video)
38 return video 39 return video
@@ -42,7 +43,7 @@ async function processVideoFile (job: Bull.Job) {
42 const payload = job.data as VideoFilePayload 43 const payload = job.data as VideoFilePayload
43 logger.info('Processing video file in job %d.', job.id) 44 logger.info('Processing video file in job %d.', job.id)
44 45
45 const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) 46 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID)
46 // No video, maybe deleted? 47 // No video, maybe deleted?
47 if (!video) { 48 if (!video) {
48 logger.info('Do not process job %d, video does not exist.', job.id) 49 logger.info('Do not process job %d, video does not exist.', job.id)
@@ -51,11 +52,11 @@ async function processVideoFile (job: Bull.Job) {
51 52
52 // Transcoding in other resolution 53 // Transcoding in other resolution
53 if (payload.resolution) { 54 if (payload.resolution) {
54 await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode || false) 55 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
55 56
56 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 57 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video)
57 } else { 58 } else {
58 await video.optimizeOriginalVideofile() 59 await optimizeOriginalVideofile(video)
59 60
60 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 61 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo)
61 } 62 }
@@ -68,7 +69,7 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
68 69
69 return sequelizeTypescript.transaction(async t => { 70 return sequelizeTypescript.transaction(async t => {
70 // Maybe the video changed in database, refresh it 71 // Maybe the video changed in database, refresh it
71 let videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) 72 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
72 // Video does not exist anymore 73 // Video does not exist anymore
73 if (!videoDatabase) return undefined 74 if (!videoDatabase) return undefined
74 75
@@ -98,7 +99,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole
98 99
99 return sequelizeTypescript.transaction(async t => { 100 return sequelizeTypescript.transaction(async t => {
100 // Maybe the video changed in database, refresh it 101 // Maybe the video changed in database, refresh it
101 const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) 102 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
102 // Video does not exist anymore 103 // Video does not exist anymore
103 if (!videoDatabase) return undefined 104 if (!videoDatabase) return undefined
104 105
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index ebcb2090c..9e14e57e6 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -183,7 +183,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
183 const videoUpdated = await video.save({ transaction: t }) 183 const videoUpdated = await video.save({ transaction: t })
184 184
185 // Now we can federate the video (reload from database, we need more attributes) 185 // Now we can federate the video (reload from database, we need more attributes)
186 const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) 186 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
187 await federateVideoIfNeeded(videoForFederation, true, t) 187 await federateVideoIfNeeded(videoForFederation, true, t)
188 188
189 // Update video import object 189 // Update video import object
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 2f8667e19..5cbe60b82 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,15 +4,50 @@ import { UserModel } from '../models/account/user'
4import { OAuthClientModel } from '../models/oauth/oauth-client' 4import { OAuthClientModel } from '../models/oauth/oauth-client'
5import { OAuthTokenModel } from '../models/oauth/oauth-token' 5import { OAuthTokenModel } from '../models/oauth/oauth-token'
6import { CONFIG } from '../initializers/constants' 6import { CONFIG } from '../initializers/constants'
7import { Transaction } from 'sequelize'
7 8
8type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } 9type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
10const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
11const userHavingToken: { [ userId: number ]: string } = {}
9 12
10// --------------------------------------------------------------------------- 13// ---------------------------------------------------------------------------
11 14
15function deleteUserToken (userId: number, t?: Transaction) {
16 clearCacheByUserId(userId)
17
18 return OAuthTokenModel.deleteUserToken(userId, t)
19}
20
21function clearCacheByUserId (userId: number) {
22 const token = userHavingToken[userId]
23 if (token !== undefined) {
24 accessTokenCache[ token ] = undefined
25 userHavingToken[ userId ] = undefined
26 }
27}
28
29function clearCacheByToken (token: string) {
30 const tokenModel = accessTokenCache[ token ]
31 if (tokenModel !== undefined) {
32 userHavingToken[tokenModel.userId] = undefined
33 accessTokenCache[ token ] = undefined
34 }
35}
36
12function getAccessToken (bearerToken: string) { 37function getAccessToken (bearerToken: string) {
13 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') 38 logger.debug('Getting access token (bearerToken: ' + bearerToken + ').')
14 39
40 if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken]
41
15 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 42 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
43 .then(tokenModel => {
44 if (tokenModel) {
45 accessTokenCache[ bearerToken ] = tokenModel
46 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
47 }
48
49 return tokenModel
50 })
16} 51}
17 52
18function getClient (clientId: string, clientSecret: string) { 53function getClient (clientId: string, clientSecret: string) {
@@ -48,6 +83,8 @@ async function getUser (usernameOrEmail: string, password: string) {
48async function revokeToken (tokenInfo: TokenInfo) { 83async function revokeToken (tokenInfo: TokenInfo) {
49 const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) 84 const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken)
50 if (token) { 85 if (token) {
86 clearCacheByToken(token.accessToken)
87
51 token.destroy() 88 token.destroy()
52 .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) 89 .catch(err => logger.error('Cannot destroy token when revoking token.', { err }))
53 } 90 }
@@ -85,6 +122,9 @@ async function saveToken (token: TokenInfo, client: OAuthClientModel, user: User
85 122
86// See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications 123// See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications
87export { 124export {
125 deleteUserToken,
126 clearCacheByUserId,
127 clearCacheByToken,
88 getAccessToken, 128 getAccessToken,
89 getClient, 129 getClient,
90 getRefreshToken, 130 getRefreshToken,
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index ee9ba1766..960651712 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,10 +1,9 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 4import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { VideoFileModel } from '../../models/video/video-file' 6import { VideoFileModel } from '../../models/video/video-file'
7import { sortBy } from 'lodash'
8import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 7import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
9import { join } from 'path' 8import { join } from 'path'
10import { rename } from 'fs-extra' 9import { rename } from 'fs-extra'
@@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils'
12import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
13import { VideoModel } from '../../models/video/video' 12import { VideoModel } from '../../models/video/video'
14import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 13import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
15import { removeVideoRedundancy } from '../redundancy'
16import { isTestInstance } from '../../helpers/core-utils' 14import { isTestInstance } from '../../helpers/core-utils'
17 15
18export class VideosRedundancyScheduler extends AbstractScheduler { 16export class VideosRedundancyScheduler extends AbstractScheduler {
@@ -20,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
20 private static instance: AbstractScheduler 18 private static instance: AbstractScheduler
21 private executing = false 19 private executing = false
22 20
23 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy 21 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
24 22
25 private constructor () { 23 private constructor () {
26 super() 24 super()
@@ -31,17 +29,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
31 29
32 this.executing = true 30 this.executing = true
33 31
34 for (const obj of CONFIG.REDUNDANCY.VIDEOS) { 32 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
35
36 try { 33 try {
37 const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) 34 const videoToDuplicate = await this.findVideoToDuplicate(obj)
38 if (!videoToDuplicate) continue 35 if (!videoToDuplicate) continue
39 36
40 const videoFiles = videoToDuplicate.VideoFiles 37 const videoFiles = videoToDuplicate.VideoFiles
41 videoFiles.forEach(f => f.Video = videoToDuplicate) 38 videoFiles.forEach(f => f.Video = videoToDuplicate)
42 39
43 const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) 40 if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) {
44 if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
45 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 41 if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
46 continue 42 continue
47 } 43 }
@@ -54,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
54 } 50 }
55 } 51 }
56 52
53 await this.removeExpired()
54
55 this.executing = false
56 }
57
58 static get Instance () {
59 return this.instance || (this.instance = new this())
60 }
61
62 private async removeExpired () {
57 const expired = await VideoRedundancyModel.listAllExpired() 63 const expired = await VideoRedundancyModel.listAllExpired()
58 64
59 for (const m of expired) { 65 for (const m of expired) {
@@ -65,16 +71,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
65 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) 71 logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
66 } 72 }
67 } 73 }
68
69 this.executing = false
70 } 74 }
71 75
72 static get Instance () { 76 private findVideoToDuplicate (cache: VideosRedundancy) {
73 return this.instance || (this.instance = new this()) 77 if (cache.strategy === 'most-views') {
74 } 78 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
79 }
80
81 if (cache.strategy === 'trending') {
82 return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
83 }
75 84
76 private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { 85 if (cache.strategy === 'recently-added') {
77 if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 86 const minViews = cache.minViews
87 return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
88 }
78 } 89 }
79 90
80 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { 91 private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
@@ -120,27 +131,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
120 } 131 }
121 } 132 }
122 133
123 // Unused, but could be useful in the future, with a custom strategy 134 private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
124 private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
125 const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
126
127 while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
128 const toDelete = sortedVideosRedundancy.shift()
129
130 const videoFile = toDelete.VideoFile
131 logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
132
133 await removeVideoRedundancy(toDelete, undefined)
134 }
135
136 return sortedVideosRedundancy
137 }
138
139 private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
140 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) 135 const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
141 136
142 const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size 137 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy)
143 const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
144 138
145 return totalDuplicated > maxSize 139 return totalDuplicated > maxSize
146 } 140 }
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
index faadb4334..461cd045e 100644
--- a/server/lib/schedulers/youtube-dl-update-scheduler.ts
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -1,13 +1,6 @@
1// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
2// We rewrote it to avoid sync calls
3
4import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers' 2import { SCHEDULER_INTERVALS_MS } from '../../initializers'
6import { logger } from '../../helpers/logger' 3import { updateYoutubeDLBinary } from '../../helpers/youtube-dl'
7import * as request from 'request'
8import { createWriteStream, ensureDir, writeFile } from 'fs-extra'
9import { join } from 'path'
10import { root } from '../../helpers/core-utils'
11 4
12export class YoutubeDlUpdateScheduler extends AbstractScheduler { 5export class YoutubeDlUpdateScheduler extends AbstractScheduler {
13 6
@@ -19,60 +12,8 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler {
19 super() 12 super()
20 } 13 }
21 14
22 async execute () { 15 execute () {
23 logger.info('Updating youtubeDL binary.') 16 return updateYoutubeDLBinary()
24
25 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
26 const bin = join(binDirectory, 'youtube-dl')
27 const detailsPath = join(binDirectory, 'details')
28 const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
29
30 await ensureDir(binDirectory)
31
32 return new Promise(res => {
33 request.get(url, { followRedirect: false }, (err, result) => {
34 if (err) {
35 logger.error('Cannot update youtube-dl.', { err })
36 return res()
37 }
38
39 if (result.statusCode !== 302) {
40 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
41 return res()
42 }
43
44 const url = result.headers.location
45 const downloadFile = request.get(url)
46 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
47
48 downloadFile.on('response', result => {
49 if (result.statusCode !== 200) {
50 logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
51 return res()
52 }
53
54 downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
55 })
56
57 downloadFile.on('error', err => {
58 logger.error('youtube-dl update error.', { err })
59 return res()
60 })
61
62 downloadFile.on('end', () => {
63 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
64 writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
65 if (err) {
66 logger.error('youtube-dl update error: cannot write details.', { err })
67 return res()
68 }
69
70 logger.info('youtube-dl updated to version %s.', newVersion)
71 return res()
72 })
73 })
74 })
75 })
76 } 17 }
77 18
78 static get Instance () { 19 static get Instance () {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
new file mode 100644
index 000000000..bf3ff78c2
--- /dev/null
+++ b/server/lib/video-transcoding.ts
@@ -0,0 +1,130 @@
1import { CONFIG } from '../initializers'
2import { join, extname } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, rename, stat } from 'fs-extra'
5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video'
9
10async function optimizeOriginalVideofile (video: VideoModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
12 const newExtname = '.mp4'
13 const inputVideoFile = video.getOriginalFile()
14 const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile))
15 const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname)
16
17 const transcodeOptions = {
18 inputPath: videoInputPath,
19 outputPath: videoTranscodedPath
20 }
21
22 // Could be very long!
23 await transcode(transcodeOptions)
24
25 try {
26 await remove(videoInputPath)
27
28 // Important to do this before getVideoFilename() to take in account the new file extension
29 inputVideoFile.set('extname', newExtname)
30
31 const videoOutputPath = video.getVideoFilePath(inputVideoFile)
32 await rename(videoTranscodedPath, videoOutputPath)
33 const stats = await stat(videoOutputPath)
34 const fps = await getVideoFileFPS(videoOutputPath)
35
36 inputVideoFile.set('size', stats.size)
37 inputVideoFile.set('fps', fps)
38
39 await video.createTorrentAndSetInfoHash(inputVideoFile)
40 await inputVideoFile.save()
41 } catch (err) {
42 // Auto destruction...
43 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
44
45 throw err
46 }
47}
48
49async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
50 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
51 const extname = '.mp4'
52
53 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
54 const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile()))
55
56 const newVideoFile = new VideoFileModel({
57 resolution,
58 extname,
59 size: 0,
60 videoId: video.id
61 })
62 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile))
63
64 const transcodeOptions = {
65 inputPath: videoInputPath,
66 outputPath: videoOutputPath,
67 resolution,
68 isPortraitMode
69 }
70
71 await transcode(transcodeOptions)
72
73 const stats = await stat(videoOutputPath)
74 const fps = await getVideoFileFPS(videoOutputPath)
75
76 newVideoFile.set('size', stats.size)
77 newVideoFile.set('fps', fps)
78
79 await video.createTorrentAndSetInfoHash(newVideoFile)
80
81 await newVideoFile.save()
82
83 video.VideoFiles.push(newVideoFile)
84}
85
86async function importVideoFile (video: VideoModel, inputFilePath: string) {
87 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
88 const { size } = await stat(inputFilePath)
89 const fps = await getVideoFileFPS(inputFilePath)
90
91 let updatedVideoFile = new VideoFileModel({
92 resolution: videoFileResolution,
93 extname: extname(inputFilePath),
94 size,
95 fps,
96 videoId: video.id
97 })
98
99 const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
100
101 if (currentVideoFile) {
102 // Remove old file and old torrent
103 await video.removeFile(currentVideoFile)
104 await video.removeTorrent(currentVideoFile)
105 // Remove the old video file from the array
106 video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
107
108 // Update the database
109 currentVideoFile.set('extname', updatedVideoFile.extname)
110 currentVideoFile.set('size', updatedVideoFile.size)
111 currentVideoFile.set('fps', updatedVideoFile.fps)
112
113 updatedVideoFile = currentVideoFile
114 }
115
116 const outputPath = video.getVideoFilePath(updatedVideoFile)
117 await copy(inputFilePath, outputPath)
118
119 await video.createTorrentAndSetInfoHash(updatedVideoFile)
120
121 await updatedVideoFile.save()
122
123 video.VideoFiles.push(updatedVideoFile)
124}
125
126export {
127 optimizeOriginalVideofile,
128 transcodeOriginalVideofile,
129 importVideoFile
130}
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index d13c50c84..d3ba1ae23 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -172,7 +172,7 @@ const usersVideoRatingValidator = [
172 logger.debug('Checking usersVideoRating parameters', { parameters: req.params }) 172 logger.debug('Checking usersVideoRating parameters', { parameters: req.params })
173 173
174 if (areValidationErrors(req, res)) return 174 if (areValidationErrors(req, res)) return
175 if (!await isVideoExist(req.params.videoId, res)) return 175 if (!await isVideoExist(req.params.videoId, res, 'id')) return
176 176
177 return next() 177 return next()
178 } 178 }
diff --git a/server/middlewares/validators/video-captions.ts b/server/middlewares/validators/video-captions.ts
index 4f393ea84..51ffd7f3c 100644
--- a/server/middlewares/validators/video-captions.ts
+++ b/server/middlewares/validators/video-captions.ts
@@ -58,7 +58,7 @@ const listVideoCaptionsValidator = [
58 logger.debug('Checking listVideoCaptions parameters', { parameters: req.params }) 58 logger.debug('Checking listVideoCaptions parameters', { parameters: req.params })
59 59
60 if (areValidationErrors(req, res)) return 60 if (areValidationErrors(req, res)) return
61 if (!await isVideoExist(req.params.videoId, res)) return 61 if (!await isVideoExist(req.params.videoId, res, 'id')) return
62 62
63 return next() 63 return next()
64 } 64 }
diff --git a/server/middlewares/validators/video-comments.ts b/server/middlewares/validators/video-comments.ts
index 227bc1fca..693852499 100644
--- a/server/middlewares/validators/video-comments.ts
+++ b/server/middlewares/validators/video-comments.ts
@@ -17,7 +17,7 @@ const listVideoCommentThreadsValidator = [
17 logger.debug('Checking listVideoCommentThreads parameters.', { parameters: req.params }) 17 logger.debug('Checking listVideoCommentThreads parameters.', { parameters: req.params })
18 18
19 if (areValidationErrors(req, res)) return 19 if (areValidationErrors(req, res)) return
20 if (!await isVideoExist(req.params.videoId, res)) return 20 if (!await isVideoExist(req.params.videoId, res, 'only-video')) return
21 21
22 return next() 22 return next()
23 } 23 }
@@ -31,7 +31,7 @@ const listVideoThreadCommentsValidator = [
31 logger.debug('Checking listVideoThreadComments parameters.', { parameters: req.params }) 31 logger.debug('Checking listVideoThreadComments parameters.', { parameters: req.params })
32 32
33 if (areValidationErrors(req, res)) return 33 if (areValidationErrors(req, res)) return
34 if (!await isVideoExist(req.params.videoId, res)) return 34 if (!await isVideoExist(req.params.videoId, res, 'only-video')) return
35 if (!await isVideoCommentThreadExist(req.params.threadId, res.locals.video, res)) return 35 if (!await isVideoCommentThreadExist(req.params.threadId, res.locals.video, res)) return
36 36
37 return next() 37 return next()
@@ -78,7 +78,7 @@ const videoCommentGetValidator = [
78 logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) 78 logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params })
79 79
80 if (areValidationErrors(req, res)) return 80 if (areValidationErrors(req, res)) return
81 if (!await isVideoExist(req.params.videoId, res)) return 81 if (!await isVideoExist(req.params.videoId, res, 'id')) return
82 if (!await isVideoCommentExist(req.params.commentId, res.locals.video, res)) return 82 if (!await isVideoCommentExist(req.params.commentId, res.locals.video, res)) return
83 83
84 return next() 84 return next()
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts
index 9befbc9ee..67eabe468 100644
--- a/server/middlewares/validators/videos.ts
+++ b/server/middlewares/validators/videos.ts
@@ -41,6 +41,7 @@ import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } f
41import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model' 41import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model'
42import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership' 42import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership'
43import { AccountModel } from '../../models/account/account' 43import { AccountModel } from '../../models/account/account'
44import { VideoFetchType } from '../../helpers/video'
44 45
45const videosAddValidator = getCommonVideoAttributes().concat([ 46const videosAddValidator = getCommonVideoAttributes().concat([
46 body('videofile') 47 body('videofile')
@@ -128,47 +129,49 @@ const videosUpdateValidator = getCommonVideoAttributes().concat([
128 } 129 }
129]) 130])
130 131
131const videosGetValidator = [ 132const videosCustomGetValidator = (fetchType: VideoFetchType) => {
132 param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), 133 return [
133 134 param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
134 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
135 logger.debug('Checking videosGet parameters', { parameters: req.params })
136 135
137 if (areValidationErrors(req, res)) return 136 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
138 if (!await isVideoExist(req.params.id, res)) return 137 logger.debug('Checking videosGet parameters', { parameters: req.params })
139 138
140 const video: VideoModel = res.locals.video 139 if (areValidationErrors(req, res)) return
140 if (!await isVideoExist(req.params.id, res, fetchType)) return
141 141
142 // Video private or blacklisted 142 const video: VideoModel = res.locals.video
143 if (video.privacy === VideoPrivacy.PRIVATE || video.VideoBlacklist) {
144 return authenticate(req, res, () => {
145 const user: UserModel = res.locals.oauth.token.User
146 143
147 // Only the owner or a user that have blacklist rights can see the video 144 // Video private or blacklisted
148 if (video.VideoChannel.Account.userId !== user.id && !user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) { 145 if (video.privacy === VideoPrivacy.PRIVATE || video.VideoBlacklist) {
149 return res.status(403) 146 return authenticate(req, res, () => {
150 .json({ error: 'Cannot get this private or blacklisted video.' }) 147 const user: UserModel = res.locals.oauth.token.User
151 .end()
152 }
153 148
154 return next() 149 // Only the owner or a user that have blacklist rights can see the video
155 }) 150 if (video.VideoChannel.Account.userId !== user.id && !user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) {
151 return res.status(403)
152 .json({ error: 'Cannot get this private or blacklisted video.' })
153 .end()
154 }
156 155
157 return 156 return next()
158 } 157 })
158 }
159 159
160 // Video is public, anyone can access it 160 // Video is public, anyone can access it
161 if (video.privacy === VideoPrivacy.PUBLIC) return next() 161 if (video.privacy === VideoPrivacy.PUBLIC) return next()
162 162
163 // Video is unlisted, check we used the uuid to fetch it 163 // Video is unlisted, check we used the uuid to fetch it
164 if (video.privacy === VideoPrivacy.UNLISTED) { 164 if (video.privacy === VideoPrivacy.UNLISTED) {
165 if (isUUIDValid(req.params.id)) return next() 165 if (isUUIDValid(req.params.id)) return next()
166 166
167 // Don't leak this unlisted video 167 // Don't leak this unlisted video
168 return res.status(404).end() 168 return res.status(404).end()
169 }
169 } 170 }
170 } 171 ]
171] 172}
173
174const videosGetValidator = videosCustomGetValidator('all')
172 175
173const videosRemoveValidator = [ 176const videosRemoveValidator = [
174 param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), 177 param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
@@ -366,6 +369,7 @@ export {
366 videosAddValidator, 369 videosAddValidator,
367 videosUpdateValidator, 370 videosUpdateValidator,
368 videosGetValidator, 371 videosGetValidator,
372 videosCustomGetValidator,
369 videosRemoveValidator, 373 videosRemoveValidator,
370 videosShareValidator, 374 videosShareValidator,
371 375
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index 6bbfc6f4e..580d920ce 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -134,8 +134,8 @@ export class AccountModel extends Model<AccountModel> {
134 return undefined 134 return undefined
135 } 135 }
136 136
137 static load (id: number) { 137 static load (id: number, transaction?: Sequelize.Transaction) {
138 return AccountModel.findById(id) 138 return AccountModel.findById(id, { transaction })
139 } 139 }
140 140
141 static loadByUUID (uuid: string) { 141 static loadByUUID (uuid: string) {
diff --git a/server/models/account/user.ts b/server/models/account/user.ts
index 680b1d52d..e56b0bf40 100644
--- a/server/models/account/user.ts
+++ b/server/models/account/user.ts
@@ -1,5 +1,7 @@
1import * as Sequelize from 'sequelize' 1import * as Sequelize from 'sequelize'
2import { 2import {
3 AfterDelete,
4 AfterUpdate,
3 AllowNull, 5 AllowNull,
4 BeforeCreate, 6 BeforeCreate,
5 BeforeUpdate, 7 BeforeUpdate,
@@ -39,6 +41,7 @@ import { AccountModel } from './account'
39import { NSFWPolicyType } from '../../../shared/models/videos/nsfw-policy.type' 41import { NSFWPolicyType } from '../../../shared/models/videos/nsfw-policy.type'
40import { values } from 'lodash' 42import { values } from 'lodash'
41import { NSFW_POLICY_TYPES } from '../../initializers' 43import { NSFW_POLICY_TYPES } from '../../initializers'
44import { clearCacheByUserId } from '../../lib/oauth-model'
42 45
43enum ScopeNames { 46enum ScopeNames {
44 WITH_VIDEO_CHANNEL = 'WITH_VIDEO_CHANNEL' 47 WITH_VIDEO_CHANNEL = 'WITH_VIDEO_CHANNEL'
@@ -168,6 +171,12 @@ export class UserModel extends Model<UserModel> {
168 } 171 }
169 } 172 }
170 173
174 @AfterUpdate
175 @AfterDelete
176 static removeTokenCache (instance: UserModel) {
177 return clearCacheByUserId(instance.id)
178 }
179
171 static countTotal () { 180 static countTotal () {
172 return this.count() 181 return this.count()
173 } 182 }
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index ef8dd9f7c..f8bb59323 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -266,6 +266,18 @@ export class ActorModel extends Model<ActorModel> {
266 return ActorModel.unscoped().findById(id) 266 return ActorModel.unscoped().findById(id)
267 } 267 }
268 268
269 static isActorUrlExist (url: string) {
270 const query = {
271 raw: true,
272 where: {
273 url
274 }
275 }
276
277 return ActorModel.unscoped().findOne(query)
278 .then(a => !!a)
279 }
280
269 static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { 281 static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) {
270 const query = { 282 const query = {
271 where: { 283 where: {
@@ -315,6 +327,29 @@ export class ActorModel extends Model<ActorModel> {
315 where: { 327 where: {
316 url 328 url
317 }, 329 },
330 transaction,
331 include: [
332 {
333 attributes: [ 'id' ],
334 model: AccountModel.unscoped(),
335 required: false
336 },
337 {
338 attributes: [ 'id' ],
339 model: VideoChannelModel.unscoped(),
340 required: false
341 }
342 ]
343 }
344
345 return ActorModel.unscoped().findOne(query)
346 }
347
348 static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Sequelize.Transaction) {
349 const query = {
350 where: {
351 url
352 },
318 transaction 353 transaction
319 } 354 }
320 355
diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts
index 4c53848dc..ef9592c04 100644
--- a/server/models/oauth/oauth-token.ts
+++ b/server/models/oauth/oauth-token.ts
@@ -1,9 +1,23 @@
1import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' 1import {
2 AfterDelete,
3 AfterUpdate,
4 AllowNull,
5 BelongsTo,
6 Column,
7 CreatedAt,
8 ForeignKey,
9 Model,
10 Scopes,
11 Table,
12 UpdatedAt
13} from 'sequelize-typescript'
2import { logger } from '../../helpers/logger' 14import { logger } from '../../helpers/logger'
3import { AccountModel } from '../account/account'
4import { UserModel } from '../account/user' 15import { UserModel } from '../account/user'
5import { OAuthClientModel } from './oauth-client' 16import { OAuthClientModel } from './oauth-client'
6import { Transaction } from 'sequelize' 17import { Transaction } from 'sequelize'
18import { AccountModel } from '../account/account'
19import { ActorModel } from '../activitypub/actor'
20import { clearCacheByToken } from '../../lib/oauth-model'
7 21
8export type OAuthTokenInfo = { 22export type OAuthTokenInfo = {
9 refreshToken: string 23 refreshToken: string
@@ -17,18 +31,27 @@ export type OAuthTokenInfo = {
17} 31}
18 32
19enum ScopeNames { 33enum ScopeNames {
20 WITH_ACCOUNT = 'WITH_ACCOUNT' 34 WITH_USER = 'WITH_USER'
21} 35}
22 36
23@Scopes({ 37@Scopes({
24 [ScopeNames.WITH_ACCOUNT]: { 38 [ScopeNames.WITH_USER]: {
25 include: [ 39 include: [
26 { 40 {
27 model: () => UserModel, 41 model: () => UserModel.unscoped(),
42 required: true,
28 include: [ 43 include: [
29 { 44 {
30 model: () => AccountModel, 45 attributes: [ 'id' ],
31 required: true 46 model: () => AccountModel.unscoped(),
47 required: true,
48 include: [
49 {
50 attributes: [ 'id' ],
51 model: () => ActorModel.unscoped(),
52 required: true
53 }
54 ]
32 } 55 }
33 ] 56 ]
34 } 57 }
@@ -102,6 +125,12 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> {
102 }) 125 })
103 OAuthClients: OAuthClientModel[] 126 OAuthClients: OAuthClientModel[]
104 127
128 @AfterUpdate
129 @AfterDelete
130 static removeTokenCache (token: OAuthTokenModel) {
131 return clearCacheByToken(token.accessToken)
132 }
133
105 static getByRefreshTokenAndPopulateClient (refreshToken: string) { 134 static getByRefreshTokenAndPopulateClient (refreshToken: string) {
106 const query = { 135 const query = {
107 where: { 136 where: {
@@ -138,7 +167,7 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> {
138 } 167 }
139 } 168 }
140 169
141 return OAuthTokenModel.scope(ScopeNames.WITH_ACCOUNT).findOne(query).then(token => { 170 return OAuthTokenModel.scope(ScopeNames.WITH_USER).findOne(query).then(token => {
142 if (token) token['user'] = token.User 171 if (token) token['user'] = token.User
143 172
144 return token 173 return token
@@ -152,7 +181,7 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> {
152 } 181 }
153 } 182 }
154 183
155 return OAuthTokenModel.scope(ScopeNames.WITH_ACCOUNT) 184 return OAuthTokenModel.scope(ScopeNames.WITH_USER)
156 .findOne(query) 185 .findOne(query)
157 .then(token => { 186 .then(token => {
158 if (token) { 187 if (token) {
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 48ec77206..fb07287a8 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -14,11 +14,10 @@ import {
14 UpdatedAt 14 UpdatedAt
15} from 'sequelize-typescript' 15} from 'sequelize-typescript'
16import { ActorModel } from '../activitypub/actor' 16import { ActorModel } from '../activitypub/actor'
17import { throwIfNotValid } from '../utils' 17import { getVideoSort, throwIfNotValid } from '../utils'
18import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' 18import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
19import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' 19import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
20import { VideoFileModel } from '../video/video-file' 20import { VideoFileModel } from '../video/video-file'
21import { isDateValid } from '../../helpers/custom-validators/misc'
22import { getServerActor } from '../../helpers/utils' 21import { getServerActor } from '../../helpers/utils'
23import { VideoModel } from '../video/video' 22import { VideoModel } from '../video/video'
24import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 23import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
@@ -28,6 +27,7 @@ import { VideoChannelModel } from '../video/video-channel'
28import { ServerModel } from '../server/server' 27import { ServerModel } from '../server/server'
29import { sample } from 'lodash' 28import { sample } from 'lodash'
30import { isTestInstance } from '../../helpers/core-utils' 29import { isTestInstance } from '../../helpers/core-utils'
30import * as Bluebird from 'bluebird'
31 31
32export enum ScopeNames { 32export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO' 33 WITH_VIDEO = 'WITH_VIDEO'
@@ -145,65 +145,90 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
145 return VideoRedundancyModel.findOne(query) 145 return VideoRedundancyModel.findOne(query)
146 } 146 }
147 147
148 static async getVideoSample (p: Bluebird<VideoModel[]>) {
149 const rows = await p
150 const ids = rows.map(r => r.id)
151 const id = sample(ids)
152
153 return VideoModel.loadWithFile(id, undefined, !isTestInstance())
154 }
155
148 static async findMostViewToDuplicate (randomizedFactor: number) { 156 static async findMostViewToDuplicate (randomizedFactor: number) {
149 // On VideoModel! 157 // On VideoModel!
150 const query = { 158 const query = {
159 attributes: [ 'id', 'views' ],
151 logging: !isTestInstance(), 160 logging: !isTestInstance(),
152 limit: randomizedFactor, 161 limit: randomizedFactor,
153 order: [ [ 'views', 'DESC' ] ], 162 order: getVideoSort('-views'),
154 include: [ 163 include: [
155 { 164 await VideoRedundancyModel.buildVideoFileForDuplication(),
156 model: VideoFileModel.unscoped(), 165 VideoRedundancyModel.buildServerRedundancyInclude()
157 required: true,
158 where: {
159 id: {
160 [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
161 }
162 }
163 },
164 {
165 attributes: [],
166 model: VideoChannelModel.unscoped(),
167 required: true,
168 include: [
169 {
170 attributes: [],
171 model: ActorModel.unscoped(),
172 required: true,
173 include: [
174 {
175 attributes: [],
176 model: ServerModel.unscoped(),
177 required: true,
178 where: {
179 redundancyAllowed: true
180 }
181 }
182 ]
183 }
184 ]
185 }
186 ] 166 ]
187 } 167 }
188 168
189 const rows = await VideoModel.unscoped().findAll(query) 169 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
170 }
190 171
191 return sample(rows) 172 static async findTrendingToDuplicate (randomizedFactor: number) {
173 // On VideoModel!
174 const query = {
175 attributes: [ 'id', 'views' ],
176 subQuery: false,
177 logging: !isTestInstance(),
178 group: 'VideoModel.id',
179 limit: randomizedFactor,
180 order: getVideoSort('-trending'),
181 include: [
182 await VideoRedundancyModel.buildVideoFileForDuplication(),
183 VideoRedundancyModel.buildServerRedundancyInclude(),
184
185 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
186 ]
187 }
188
189 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
192 } 190 }
193 191
194 static async getVideoFiles (strategy: VideoRedundancyStrategy) { 192 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
193 // On VideoModel!
194 const query = {
195 attributes: [ 'id', 'publishedAt' ],
196 logging: !isTestInstance(),
197 limit: randomizedFactor,
198 order: getVideoSort('-publishedAt'),
199 where: {
200 views: {
201 [ Sequelize.Op.gte ]: minViews
202 }
203 },
204 include: [
205 await VideoRedundancyModel.buildVideoFileForDuplication(),
206 VideoRedundancyModel.buildServerRedundancyInclude()
207 ]
208 }
209
210 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
211 }
212
213 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
195 const actor = await getServerActor() 214 const actor = await getServerActor()
196 215
197 const queryVideoFiles = { 216 const options = {
198 logging: !isTestInstance(), 217 logging: !isTestInstance(),
199 where: { 218 include: [
200 actorId: actor.id, 219 {
201 strategy 220 attributes: [],
202 } 221 model: VideoRedundancyModel,
222 required: true,
223 where: {
224 actorId: actor.id,
225 strategy
226 }
227 }
228 ]
203 } 229 }
204 230
205 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) 231 return VideoFileModel.sum('size', options)
206 .findAll(queryVideoFiles)
207 } 232 }
208 233
209 static listAllExpired () { 234 static listAllExpired () {
@@ -211,7 +236,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
211 logging: !isTestInstance(), 236 logging: !isTestInstance(),
212 where: { 237 where: {
213 expiresOn: { 238 expiresOn: {
214 [Sequelize.Op.lt]: new Date() 239 [ Sequelize.Op.lt ]: new Date()
215 } 240 }
216 } 241 }
217 } 242 }
@@ -220,6 +245,37 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
220 .findAll(query) 245 .findAll(query)
221 } 246 }
222 247
248 static async getStats (strategy: VideoRedundancyStrategy) {
249 const actor = await getServerActor()
250
251 const query = {
252 raw: true,
253 attributes: [
254 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
255 [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ],
256 [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ]
257 ],
258 where: {
259 strategy,
260 actorId: actor.id
261 },
262 include: [
263 {
264 attributes: [],
265 model: VideoFileModel,
266 required: true
267 }
268 ]
269 }
270
271 return VideoRedundancyModel.find(query as any) // FIXME: typings
272 .then((r: any) => ({
273 totalUsed: parseInt(r.totalUsed.toString(), 10),
274 totalVideos: r.totalVideos,
275 totalVideoFiles: r.totalVideoFiles
276 }))
277 }
278
223 toActivityPubObject (): CacheFileObject { 279 toActivityPubObject (): CacheFileObject {
224 return { 280 return {
225 id: this.url, 281 id: this.url,
@@ -237,13 +293,50 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
237 } 293 }
238 } 294 }
239 295
240 private static async buildExcludeIn () { 296 // Don't include video files we already duplicated
297 private static async buildVideoFileForDuplication () {
241 const actor = await getServerActor() 298 const actor = await getServerActor()
242 299
243 return Sequelize.literal( 300 const notIn = Sequelize.literal(
244 '(' + 301 '(' +
245 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + 302 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
246 ')' 303 ')'
247 ) 304 )
305
306 return {
307 attributes: [],
308 model: VideoFileModel.unscoped(),
309 required: true,
310 where: {
311 id: {
312 [ Sequelize.Op.notIn ]: notIn
313 }
314 }
315 }
316 }
317
318 private static buildServerRedundancyInclude () {
319 return {
320 attributes: [],
321 model: VideoChannelModel.unscoped(),
322 required: true,
323 include: [
324 {
325 attributes: [],
326 model: ActorModel.unscoped(),
327 required: true,
328 include: [
329 {
330 attributes: [],
331 model: ServerModel.unscoped(),
332 required: true,
333 where: {
334 redundancyAllowed: true
335 }
336 }
337 ]
338 }
339 ]
340 }
248 } 341 }
249} 342}
diff --git a/server/models/video/tag.ts b/server/models/video/tag.ts
index e39a418cd..b39621eaf 100644
--- a/server/models/video/tag.ts
+++ b/server/models/video/tag.ts
@@ -48,11 +48,10 @@ export class TagModel extends Model<TagModel> {
48 }, 48 },
49 defaults: { 49 defaults: {
50 name: tag 50 name: tag
51 } 51 },
52 transaction
52 } 53 }
53 54
54 if (transaction) query['transaction'] = transaction
55
56 const promise = TagModel.findOrCreate(query) 55 const promise = TagModel.findOrCreate(query)
57 .then(([ tagInstance ]) => tagInstance) 56 .then(([ tagInstance ]) => tagInstance)
58 tasks.push(promise) 57 tasks.push(promise)
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
new file mode 100644
index 000000000..a9a58624d
--- /dev/null
+++ b/server/models/video/video-format-utils.ts
@@ -0,0 +1,296 @@
1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
2import { VideoModel } from './video'
3import { VideoFileModel } from './video-file'
4import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects'
5import { CONFIG, THUMBNAILS_SIZE, VIDEO_EXT_MIMETYPE } from '../../initializers'
6import { VideoCaptionModel } from './video-caption'
7import {
8 getVideoCommentsActivityPubUrl,
9 getVideoDislikesActivityPubUrl,
10 getVideoLikesActivityPubUrl,
11 getVideoSharesActivityPubUrl
12} from '../../lib/activitypub'
13
14export type VideoFormattingJSONOptions = {
15 additionalAttributes: {
16 state?: boolean,
17 waitTranscoding?: boolean,
18 scheduledUpdate?: boolean,
19 blacklistInfo?: boolean
20 }
21}
22function videoModelToFormattedJSON (video: VideoModel, options?: VideoFormattingJSONOptions): Video {
23 const formattedAccount = video.VideoChannel.Account.toFormattedJSON()
24 const formattedVideoChannel = video.VideoChannel.toFormattedJSON()
25
26 const videoObject: Video = {
27 id: video.id,
28 uuid: video.uuid,
29 name: video.name,
30 category: {
31 id: video.category,
32 label: VideoModel.getCategoryLabel(video.category)
33 },
34 licence: {
35 id: video.licence,
36 label: VideoModel.getLicenceLabel(video.licence)
37 },
38 language: {
39 id: video.language,
40 label: VideoModel.getLanguageLabel(video.language)
41 },
42 privacy: {
43 id: video.privacy,
44 label: VideoModel.getPrivacyLabel(video.privacy)
45 },
46 nsfw: video.nsfw,
47 description: video.getTruncatedDescription(),
48 isLocal: video.isOwned(),
49 duration: video.duration,
50 views: video.views,
51 likes: video.likes,
52 dislikes: video.dislikes,
53 thumbnailPath: video.getThumbnailStaticPath(),
54 previewPath: video.getPreviewStaticPath(),
55 embedPath: video.getEmbedStaticPath(),
56 createdAt: video.createdAt,
57 updatedAt: video.updatedAt,
58 publishedAt: video.publishedAt,
59 account: {
60 id: formattedAccount.id,
61 uuid: formattedAccount.uuid,
62 name: formattedAccount.name,
63 displayName: formattedAccount.displayName,
64 url: formattedAccount.url,
65 host: formattedAccount.host,
66 avatar: formattedAccount.avatar
67 },
68 channel: {
69 id: formattedVideoChannel.id,
70 uuid: formattedVideoChannel.uuid,
71 name: formattedVideoChannel.name,
72 displayName: formattedVideoChannel.displayName,
73 url: formattedVideoChannel.url,
74 host: formattedVideoChannel.host,
75 avatar: formattedVideoChannel.avatar
76 }
77 }
78
79 if (options) {
80 if (options.additionalAttributes.state === true) {
81 videoObject.state = {
82 id: video.state,
83 label: VideoModel.getStateLabel(video.state)
84 }
85 }
86
87 if (options.additionalAttributes.waitTranscoding === true) {
88 videoObject.waitTranscoding = video.waitTranscoding
89 }
90
91 if (options.additionalAttributes.scheduledUpdate === true && video.ScheduleVideoUpdate) {
92 videoObject.scheduledUpdate = {
93 updateAt: video.ScheduleVideoUpdate.updateAt,
94 privacy: video.ScheduleVideoUpdate.privacy || undefined
95 }
96 }
97
98 if (options.additionalAttributes.blacklistInfo === true) {
99 videoObject.blacklisted = !!video.VideoBlacklist
100 videoObject.blacklistedReason = video.VideoBlacklist ? video.VideoBlacklist.reason : null
101 }
102 }
103
104 return videoObject
105}
106
107function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
108 const formattedJson = video.toFormattedJSON({
109 additionalAttributes: {
110 scheduledUpdate: true,
111 blacklistInfo: true
112 }
113 })
114
115 const tags = video.Tags ? video.Tags.map(t => t.name) : []
116 const detailsJson = {
117 support: video.support,
118 descriptionPath: video.getDescriptionAPIPath(),
119 channel: video.VideoChannel.toFormattedJSON(),
120 account: video.VideoChannel.Account.toFormattedJSON(),
121 tags,
122 commentsEnabled: video.commentsEnabled,
123 waitTranscoding: video.waitTranscoding,
124 state: {
125 id: video.state,
126 label: VideoModel.getStateLabel(video.state)
127 },
128 files: []
129 }
130
131 // Format and sort video files
132 detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles)
133
134 return Object.assign(formattedJson, detailsJson)
135}
136
137function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] {
138 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
139
140 return videoFiles
141 .map(videoFile => {
142 let resolutionLabel = videoFile.resolution + 'p'
143
144 return {
145 resolution: {
146 id: videoFile.resolution,
147 label: resolutionLabel
148 },
149 magnetUri: video.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs),
150 size: videoFile.size,
151 fps: videoFile.fps,
152 torrentUrl: video.getTorrentUrl(videoFile, baseUrlHttp),
153 torrentDownloadUrl: video.getTorrentDownloadUrl(videoFile, baseUrlHttp),
154 fileUrl: video.getVideoFileUrl(videoFile, baseUrlHttp),
155 fileDownloadUrl: video.getVideoFileDownloadUrl(videoFile, baseUrlHttp)
156 } as VideoFile
157 })
158 .sort((a, b) => {
159 if (a.resolution.id < b.resolution.id) return 1
160 if (a.resolution.id === b.resolution.id) return 0
161 return -1
162 })
163}
164
165function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
166 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
167 if (!video.Tags) video.Tags = []
168
169 const tag = video.Tags.map(t => ({
170 type: 'Hashtag' as 'Hashtag',
171 name: t.name
172 }))
173
174 let language
175 if (video.language) {
176 language = {
177 identifier: video.language,
178 name: VideoModel.getLanguageLabel(video.language)
179 }
180 }
181
182 let category
183 if (video.category) {
184 category = {
185 identifier: video.category + '',
186 name: VideoModel.getCategoryLabel(video.category)
187 }
188 }
189
190 let licence
191 if (video.licence) {
192 licence = {
193 identifier: video.licence + '',
194 name: VideoModel.getLicenceLabel(video.licence)
195 }
196 }
197
198 const url: ActivityUrlObject[] = []
199 for (const file of video.VideoFiles) {
200 url.push({
201 type: 'Link',
202 mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
203 href: video.getVideoFileUrl(file, baseUrlHttp),
204 height: file.resolution,
205 size: file.size,
206 fps: file.fps
207 })
208
209 url.push({
210 type: 'Link',
211 mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
212 href: video.getTorrentUrl(file, baseUrlHttp),
213 height: file.resolution
214 })
215
216 url.push({
217 type: 'Link',
218 mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
219 href: video.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
220 height: file.resolution
221 })
222 }
223
224 // Add video url too
225 url.push({
226 type: 'Link',
227 mimeType: 'text/html',
228 href: CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
229 })
230
231 const subtitleLanguage = []
232 for (const caption of video.VideoCaptions) {
233 subtitleLanguage.push({
234 identifier: caption.language,
235 name: VideoCaptionModel.getLanguageLabel(caption.language)
236 })
237 }
238
239 return {
240 type: 'Video' as 'Video',
241 id: video.url,
242 name: video.name,
243 duration: getActivityStreamDuration(video.duration),
244 uuid: video.uuid,
245 tag,
246 category,
247 licence,
248 language,
249 views: video.views,
250 sensitive: video.nsfw,
251 waitTranscoding: video.waitTranscoding,
252 state: video.state,
253 commentsEnabled: video.commentsEnabled,
254 published: video.publishedAt.toISOString(),
255 updated: video.updatedAt.toISOString(),
256 mediaType: 'text/markdown',
257 content: video.getTruncatedDescription(),
258 support: video.support,
259 subtitleLanguage,
260 icon: {
261 type: 'Image',
262 url: video.getThumbnailUrl(baseUrlHttp),
263 mediaType: 'image/jpeg',
264 width: THUMBNAILS_SIZE.width,
265 height: THUMBNAILS_SIZE.height
266 },
267 url,
268 likes: getVideoLikesActivityPubUrl(video),
269 dislikes: getVideoDislikesActivityPubUrl(video),
270 shares: getVideoSharesActivityPubUrl(video),
271 comments: getVideoCommentsActivityPubUrl(video),
272 attributedTo: [
273 {
274 type: 'Person',
275 id: video.VideoChannel.Account.Actor.url
276 },
277 {
278 type: 'Group',
279 id: video.VideoChannel.Actor.url
280 }
281 ]
282 }
283}
284
285function getActivityStreamDuration (duration: number) {
286 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
287 return 'PT' + duration + 'S'
288}
289
290export {
291 videoModelToFormattedJSON,
292 videoModelToFormattedDetailsJSON,
293 videoFilesModelToFormattedJSON,
294 videoModelToActivityPubObject,
295 getActivityStreamDuration
296}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 27c631dcd..6c89c16bf 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1,8 +1,8 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { map, maxBy } from 'lodash' 2import { maxBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as parseTorrent from 'parse-torrent' 4import * as parseTorrent from 'parse-torrent'
5import { extname, join } from 'path' 5import { join } from 'path'
6import * as Sequelize from 'sequelize' 6import * as Sequelize from 'sequelize'
7import { 7import {
8 AllowNull, 8 AllowNull,
@@ -27,7 +27,7 @@ import {
27 Table, 27 Table,
28 UpdatedAt 28 UpdatedAt
29} from 'sequelize-typescript' 29} from 'sequelize-typescript'
30import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' 30import { VideoPrivacy, VideoState } from '../../../shared'
31import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 31import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
32import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 32import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
33import { VideoFilter } from '../../../shared/models/videos/video-query.type' 33import { VideoFilter } from '../../../shared/models/videos/video-query.type'
@@ -45,7 +45,7 @@ import {
45 isVideoStateValid, 45 isVideoStateValid,
46 isVideoSupportValid 46 isVideoSupportValid
47} from '../../helpers/custom-validators/videos' 47} from '../../helpers/custom-validators/videos'
48import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, transcode } from '../../helpers/ffmpeg-utils' 48import { generateImageFromVideoFile, getVideoFileResolution } from '../../helpers/ffmpeg-utils'
49import { logger } from '../../helpers/logger' 49import { logger } from '../../helpers/logger'
50import { getServerActor } from '../../helpers/utils' 50import { getServerActor } from '../../helpers/utils'
51import { 51import {
@@ -59,18 +59,11 @@ import {
59 STATIC_PATHS, 59 STATIC_PATHS,
60 THUMBNAILS_SIZE, 60 THUMBNAILS_SIZE,
61 VIDEO_CATEGORIES, 61 VIDEO_CATEGORIES,
62 VIDEO_EXT_MIMETYPE,
63 VIDEO_LANGUAGES, 62 VIDEO_LANGUAGES,
64 VIDEO_LICENCES, 63 VIDEO_LICENCES,
65 VIDEO_PRIVACIES, 64 VIDEO_PRIVACIES,
66 VIDEO_STATES 65 VIDEO_STATES
67} from '../../initializers' 66} from '../../initializers'
68import {
69 getVideoCommentsActivityPubUrl,
70 getVideoDislikesActivityPubUrl,
71 getVideoLikesActivityPubUrl,
72 getVideoSharesActivityPubUrl
73} from '../../lib/activitypub'
74import { sendDeleteVideo } from '../../lib/activitypub/send' 67import { sendDeleteVideo } from '../../lib/activitypub/send'
75import { AccountModel } from '../account/account' 68import { AccountModel } from '../account/account'
76import { AccountVideoRateModel } from '../account/account-video-rate' 69import { AccountVideoRateModel } from '../account/account-video-rate'
@@ -88,9 +81,17 @@ import { VideoTagModel } from './video-tag'
88import { ScheduleVideoUpdateModel } from './schedule-video-update' 81import { ScheduleVideoUpdateModel } from './schedule-video-update'
89import { VideoCaptionModel } from './video-caption' 82import { VideoCaptionModel } from './video-caption'
90import { VideoBlacklistModel } from './video-blacklist' 83import { VideoBlacklistModel } from './video-blacklist'
91import { copy, remove, rename, stat, writeFile } from 'fs-extra' 84import { remove, writeFile } from 'fs-extra'
92import { VideoViewModel } from './video-views' 85import { VideoViewModel } from './video-views'
93import { VideoRedundancyModel } from '../redundancy/video-redundancy' 86import { VideoRedundancyModel } from '../redundancy/video-redundancy'
87import {
88 videoFilesModelToFormattedJSON,
89 VideoFormattingJSONOptions,
90 videoModelToActivityPubObject,
91 videoModelToFormattedDetailsJSON,
92 videoModelToFormattedJSON
93} from './video-format-utils'
94import * as validator from 'validator'
94 95
95// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 96// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
96const indexes: Sequelize.DefineIndexesOptions[] = [ 97const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -221,6 +222,7 @@ type AvailableForListIDsOptions = {
221 }, 222 },
222 [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { 223 [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => {
223 const query: IFindOptions<VideoModel> = { 224 const query: IFindOptions<VideoModel> = {
225 raw: true,
224 attributes: [ 'id' ], 226 attributes: [ 'id' ],
225 where: { 227 where: {
226 id: { 228 id: {
@@ -387,16 +389,7 @@ type AvailableForListIDsOptions = {
387 } 389 }
388 390
389 if (options.trendingDays) { 391 if (options.trendingDays) {
390 query.include.push({ 392 query.include.push(VideoModel.buildTrendingQuery(options.trendingDays))
391 attributes: [],
392 model: VideoViewModel,
393 required: false,
394 where: {
395 startDate: {
396 [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays)
397 }
398 }
399 })
400 393
401 query.subQuery = false 394 query.subQuery = false
402 } 395 }
@@ -474,6 +467,7 @@ type AvailableForListIDsOptions = {
474 required: false, 467 required: false,
475 include: [ 468 include: [
476 { 469 {
470 attributes: [ 'fileUrl' ],
477 model: () => VideoRedundancyModel.unscoped(), 471 model: () => VideoRedundancyModel.unscoped(),
478 required: false 472 required: false
479 } 473 }
@@ -937,7 +931,7 @@ export class VideoModel extends Model<VideoModel> {
937 videoChannelId?: number, 931 videoChannelId?: number,
938 actorId?: number 932 actorId?: number
939 trendingDays?: number 933 trendingDays?: number
940 }) { 934 }, countVideos = true) {
941 const query: IFindOptions<VideoModel> = { 935 const query: IFindOptions<VideoModel> = {
942 offset: options.start, 936 offset: options.start,
943 limit: options.count, 937 limit: options.count,
@@ -970,7 +964,7 @@ export class VideoModel extends Model<VideoModel> {
970 trendingDays 964 trendingDays
971 } 965 }
972 966
973 return VideoModel.getAvailableForApi(query, queryOptions) 967 return VideoModel.getAvailableForApi(query, queryOptions, countVideos)
974 } 968 }
975 969
976 static async searchAndPopulateAccountAndServer (options: { 970 static async searchAndPopulateAccountAndServer (options: {
@@ -1070,41 +1064,34 @@ export class VideoModel extends Model<VideoModel> {
1070 return VideoModel.getAvailableForApi(query, queryOptions) 1064 return VideoModel.getAvailableForApi(query, queryOptions)
1071 } 1065 }
1072 1066
1073 static load (id: number, t?: Sequelize.Transaction) { 1067 static load (id: number | string, t?: Sequelize.Transaction) {
1074 const options = t ? { transaction: t } : undefined 1068 const where = VideoModel.buildWhereIdOrUUID(id)
1075 1069 const options = {
1076 return VideoModel.findById(id, options) 1070 where,
1077 } 1071 transaction: t
1078
1079 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
1080 const query: IFindOptions<VideoModel> = {
1081 where: {
1082 url
1083 }
1084 } 1072 }
1085 1073
1086 if (t !== undefined) query.transaction = t 1074 return VideoModel.findOne(options)
1087
1088 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
1089 } 1075 }
1090 1076
1091 static loadAndPopulateAccountAndServerAndTags (id: number) { 1077 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) {
1078 const where = VideoModel.buildWhereIdOrUUID(id)
1079
1092 const options = { 1080 const options = {
1093 order: [ [ 'Tags', 'name', 'ASC' ] ] 1081 attributes: [ 'id' ],
1082 where,
1083 transaction: t
1094 } 1084 }
1095 1085
1096 return VideoModel 1086 return VideoModel.findOne(options)
1097 .scope([
1098 ScopeNames.WITH_TAGS,
1099 ScopeNames.WITH_BLACKLISTED,
1100 ScopeNames.WITH_FILES,
1101 ScopeNames.WITH_ACCOUNT_DETAILS,
1102 ScopeNames.WITH_SCHEDULED_UPDATE
1103 ])
1104 .findById(id, options)
1105 } 1087 }
1106 1088
1107 static loadByUUID (uuid: string) { 1089 static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) {
1090 return VideoModel.scope(ScopeNames.WITH_FILES)
1091 .findById(id, { transaction: t, logging })
1092 }
1093
1094 static loadByUUIDWithFile (uuid: string) {
1108 const options = { 1095 const options = {
1109 where: { 1096 where: {
1110 uuid 1097 uuid
@@ -1116,12 +1103,34 @@ export class VideoModel extends Model<VideoModel> {
1116 .findOne(options) 1103 .findOne(options)
1117 } 1104 }
1118 1105
1119 static loadByUUIDAndPopulateAccountAndServerAndTags (uuid: string, t?: Sequelize.Transaction) { 1106 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
1120 const options = { 1107 const query: IFindOptions<VideoModel> = {
1121 order: [ [ 'Tags', 'name', 'ASC' ] ],
1122 where: { 1108 where: {
1123 uuid 1109 url
1110 },
1111 transaction
1112 }
1113
1114 return VideoModel.findOne(query)
1115 }
1116
1117 static loadByUrlAndPopulateAccount (url: string, transaction?: Sequelize.Transaction) {
1118 const query: IFindOptions<VideoModel> = {
1119 where: {
1120 url
1124 }, 1121 },
1122 transaction
1123 }
1124
1125 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
1126 }
1127
1128 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction) {
1129 const where = VideoModel.buildWhereIdOrUUID(id)
1130
1131 const options = {
1132 order: [ [ 'Tags', 'name', 'ASC' ] ],
1133 where,
1125 transaction: t 1134 transaction: t
1126 } 1135 }
1127 1136
@@ -1169,7 +1178,14 @@ export class VideoModel extends Model<VideoModel> {
1169 } 1178 }
1170 1179
1171 // threshold corresponds to how many video the field should have to be returned 1180 // threshold corresponds to how many video the field should have to be returned
1172 static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { 1181 static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
1182 const actorId = (await getServerActor()).id
1183
1184 const scopeOptions = {
1185 actorId,
1186 includeLocalVideos: true
1187 }
1188
1173 const query: IFindOptions<VideoModel> = { 1189 const query: IFindOptions<VideoModel> = {
1174 attributes: [ field ], 1190 attributes: [ field ],
1175 limit: count, 1191 limit: count,
@@ -1177,20 +1193,28 @@ export class VideoModel extends Model<VideoModel> {
1177 having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { 1193 having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), {
1178 [ Sequelize.Op.gte ]: threshold 1194 [ Sequelize.Op.gte ]: threshold
1179 }) as any, // FIXME: typings 1195 }) as any, // FIXME: typings
1180 where: {
1181 [ field ]: {
1182 [ Sequelize.Op.not ]: null
1183 },
1184 privacy: VideoPrivacy.PUBLIC,
1185 state: VideoState.PUBLISHED
1186 },
1187 order: [ this.sequelize.random() ] 1196 order: [ this.sequelize.random() ]
1188 } 1197 }
1189 1198
1190 return VideoModel.findAll(query) 1199 return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] })
1200 .findAll(query)
1191 .then(rows => rows.map(r => r[ field ])) 1201 .then(rows => rows.map(r => r[ field ]))
1192 } 1202 }
1193 1203
1204 static buildTrendingQuery (trendingDays: number) {
1205 return {
1206 attributes: [],
1207 subQuery: false,
1208 model: VideoViewModel,
1209 required: false,
1210 where: {
1211 startDate: {
1212 [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays)
1213 }
1214 }
1215 }
1216 }
1217
1194 private static buildActorWhereWithFilter (filter?: VideoFilter) { 1218 private static buildActorWhereWithFilter (filter?: VideoFilter) {
1195 if (filter && filter === 'local') { 1219 if (filter && filter === 'local') {
1196 return { 1220 return {
@@ -1201,7 +1225,7 @@ export class VideoModel extends Model<VideoModel> {
1201 return {} 1225 return {}
1202 } 1226 }
1203 1227
1204 private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions) { 1228 private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions, countVideos = true) {
1205 const idsScope = { 1229 const idsScope = {
1206 method: [ 1230 method: [
1207 ScopeNames.AVAILABLE_FOR_LIST_IDS, options 1231 ScopeNames.AVAILABLE_FOR_LIST_IDS, options
@@ -1218,7 +1242,7 @@ export class VideoModel extends Model<VideoModel> {
1218 } 1242 }
1219 1243
1220 const [ count, rowsId ] = await Promise.all([ 1244 const [ count, rowsId ] = await Promise.all([
1221 VideoModel.scope(countScope).count(countQuery), 1245 countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
1222 VideoModel.scope(idsScope).findAll(query) 1246 VideoModel.scope(idsScope).findAll(query)
1223 ]) 1247 ])
1224 const ids = rowsId.map(r => r.id) 1248 const ids = rowsId.map(r => r.id)
@@ -1247,26 +1271,30 @@ export class VideoModel extends Model<VideoModel> {
1247 } 1271 }
1248 } 1272 }
1249 1273
1250 private static getCategoryLabel (id: number) { 1274 static getCategoryLabel (id: number) {
1251 return VIDEO_CATEGORIES[ id ] || 'Misc' 1275 return VIDEO_CATEGORIES[ id ] || 'Misc'
1252 } 1276 }
1253 1277
1254 private static getLicenceLabel (id: number) { 1278 static getLicenceLabel (id: number) {
1255 return VIDEO_LICENCES[ id ] || 'Unknown' 1279 return VIDEO_LICENCES[ id ] || 'Unknown'
1256 } 1280 }
1257 1281
1258 private static getLanguageLabel (id: string) { 1282 static getLanguageLabel (id: string) {
1259 return VIDEO_LANGUAGES[ id ] || 'Unknown' 1283 return VIDEO_LANGUAGES[ id ] || 'Unknown'
1260 } 1284 }
1261 1285
1262 private static getPrivacyLabel (id: number) { 1286 static getPrivacyLabel (id: number) {
1263 return VIDEO_PRIVACIES[ id ] || 'Unknown' 1287 return VIDEO_PRIVACIES[ id ] || 'Unknown'
1264 } 1288 }
1265 1289
1266 private static getStateLabel (id: number) { 1290 static getStateLabel (id: number) {
1267 return VIDEO_STATES[ id ] || 'Unknown' 1291 return VIDEO_STATES[ id ] || 'Unknown'
1268 } 1292 }
1269 1293
1294 static buildWhereIdOrUUID (id: number | string) {
1295 return validator.isInt('' + id) ? { id } : { uuid: id }
1296 }
1297
1270 getOriginalFile () { 1298 getOriginalFile () {
1271 if (Array.isArray(this.VideoFiles) === false) return undefined 1299 if (Array.isArray(this.VideoFiles) === false) return undefined
1272 1300
@@ -1359,273 +1387,20 @@ export class VideoModel extends Model<VideoModel> {
1359 return join(STATIC_PATHS.PREVIEWS, this.getPreviewName()) 1387 return join(STATIC_PATHS.PREVIEWS, this.getPreviewName())
1360 } 1388 }
1361 1389
1362 toFormattedJSON (options?: { 1390 toFormattedJSON (options?: VideoFormattingJSONOptions): Video {
1363 additionalAttributes: { 1391 return videoModelToFormattedJSON(this, options)
1364 state?: boolean,
1365 waitTranscoding?: boolean,
1366 scheduledUpdate?: boolean,
1367 blacklistInfo?: boolean
1368 }
1369 }): Video {
1370 const formattedAccount = this.VideoChannel.Account.toFormattedJSON()
1371 const formattedVideoChannel = this.VideoChannel.toFormattedJSON()
1372
1373 const videoObject: Video = {
1374 id: this.id,
1375 uuid: this.uuid,
1376 name: this.name,
1377 category: {
1378 id: this.category,
1379 label: VideoModel.getCategoryLabel(this.category)
1380 },
1381 licence: {
1382 id: this.licence,
1383 label: VideoModel.getLicenceLabel(this.licence)
1384 },
1385 language: {
1386 id: this.language,
1387 label: VideoModel.getLanguageLabel(this.language)
1388 },
1389 privacy: {
1390 id: this.privacy,
1391 label: VideoModel.getPrivacyLabel(this.privacy)
1392 },
1393 nsfw: this.nsfw,
1394 description: this.getTruncatedDescription(),
1395 isLocal: this.isOwned(),
1396 duration: this.duration,
1397 views: this.views,
1398 likes: this.likes,
1399 dislikes: this.dislikes,
1400 thumbnailPath: this.getThumbnailStaticPath(),
1401 previewPath: this.getPreviewStaticPath(),
1402 embedPath: this.getEmbedStaticPath(),
1403 createdAt: this.createdAt,
1404 updatedAt: this.updatedAt,
1405 publishedAt: this.publishedAt,
1406 account: {
1407 id: formattedAccount.id,
1408 uuid: formattedAccount.uuid,
1409 name: formattedAccount.name,
1410 displayName: formattedAccount.displayName,
1411 url: formattedAccount.url,
1412 host: formattedAccount.host,
1413 avatar: formattedAccount.avatar
1414 },
1415 channel: {
1416 id: formattedVideoChannel.id,
1417 uuid: formattedVideoChannel.uuid,
1418 name: formattedVideoChannel.name,
1419 displayName: formattedVideoChannel.displayName,
1420 url: formattedVideoChannel.url,
1421 host: formattedVideoChannel.host,
1422 avatar: formattedVideoChannel.avatar
1423 }
1424 }
1425
1426 if (options) {
1427 if (options.additionalAttributes.state === true) {
1428 videoObject.state = {
1429 id: this.state,
1430 label: VideoModel.getStateLabel(this.state)
1431 }
1432 }
1433
1434 if (options.additionalAttributes.waitTranscoding === true) {
1435 videoObject.waitTranscoding = this.waitTranscoding
1436 }
1437
1438 if (options.additionalAttributes.scheduledUpdate === true && this.ScheduleVideoUpdate) {
1439 videoObject.scheduledUpdate = {
1440 updateAt: this.ScheduleVideoUpdate.updateAt,
1441 privacy: this.ScheduleVideoUpdate.privacy || undefined
1442 }
1443 }
1444
1445 if (options.additionalAttributes.blacklistInfo === true) {
1446 videoObject.blacklisted = !!this.VideoBlacklist
1447 videoObject.blacklistedReason = this.VideoBlacklist ? this.VideoBlacklist.reason : null
1448 }
1449 }
1450
1451 return videoObject
1452 } 1392 }
1453 1393
1454 toFormattedDetailsJSON (): VideoDetails { 1394 toFormattedDetailsJSON (): VideoDetails {
1455 const formattedJson = this.toFormattedJSON({ 1395 return videoModelToFormattedDetailsJSON(this)
1456 additionalAttributes: {
1457 scheduledUpdate: true,
1458 blacklistInfo: true
1459 }
1460 })
1461
1462 const detailsJson = {
1463 support: this.support,
1464 descriptionPath: this.getDescriptionPath(),
1465 channel: this.VideoChannel.toFormattedJSON(),
1466 account: this.VideoChannel.Account.toFormattedJSON(),
1467 tags: map(this.Tags, 'name'),
1468 commentsEnabled: this.commentsEnabled,
1469 waitTranscoding: this.waitTranscoding,
1470 state: {
1471 id: this.state,
1472 label: VideoModel.getStateLabel(this.state)
1473 },
1474 files: []
1475 }
1476
1477 // Format and sort video files
1478 detailsJson.files = this.getFormattedVideoFilesJSON()
1479
1480 return Object.assign(formattedJson, detailsJson)
1481 } 1396 }
1482 1397
1483 getFormattedVideoFilesJSON (): VideoFile[] { 1398 getFormattedVideoFilesJSON (): VideoFile[] {
1484 const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() 1399 return videoFilesModelToFormattedJSON(this, this.VideoFiles)
1485
1486 return this.VideoFiles
1487 .map(videoFile => {
1488 let resolutionLabel = videoFile.resolution + 'p'
1489
1490 return {
1491 resolution: {
1492 id: videoFile.resolution,
1493 label: resolutionLabel
1494 },
1495 magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs),
1496 size: videoFile.size,
1497 fps: videoFile.fps,
1498 torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp),
1499 torrentDownloadUrl: this.getTorrentDownloadUrl(videoFile, baseUrlHttp),
1500 fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp),
1501 fileDownloadUrl: this.getVideoFileDownloadUrl(videoFile, baseUrlHttp)
1502 } as VideoFile
1503 })
1504 .sort((a, b) => {
1505 if (a.resolution.id < b.resolution.id) return 1
1506 if (a.resolution.id === b.resolution.id) return 0
1507 return -1
1508 })
1509 } 1400 }
1510 1401
1511 toActivityPubObject (): VideoTorrentObject { 1402 toActivityPubObject (): VideoTorrentObject {
1512 const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() 1403 return videoModelToActivityPubObject(this)
1513 if (!this.Tags) this.Tags = []
1514
1515 const tag = this.Tags.map(t => ({
1516 type: 'Hashtag' as 'Hashtag',
1517 name: t.name
1518 }))
1519
1520 let language
1521 if (this.language) {
1522 language = {
1523 identifier: this.language,
1524 name: VideoModel.getLanguageLabel(this.language)
1525 }
1526 }
1527
1528 let category
1529 if (this.category) {
1530 category = {
1531 identifier: this.category + '',
1532 name: VideoModel.getCategoryLabel(this.category)
1533 }
1534 }
1535
1536 let licence
1537 if (this.licence) {
1538 licence = {
1539 identifier: this.licence + '',
1540 name: VideoModel.getLicenceLabel(this.licence)
1541 }
1542 }
1543
1544 const url: ActivityUrlObject[] = []
1545 for (const file of this.VideoFiles) {
1546 url.push({
1547 type: 'Link',
1548 mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
1549 href: this.getVideoFileUrl(file, baseUrlHttp),
1550 height: file.resolution,
1551 size: file.size,
1552 fps: file.fps
1553 })
1554
1555 url.push({
1556 type: 'Link',
1557 mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
1558 href: this.getTorrentUrl(file, baseUrlHttp),
1559 height: file.resolution
1560 })
1561
1562 url.push({
1563 type: 'Link',
1564 mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
1565 href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
1566 height: file.resolution
1567 })
1568 }
1569
1570 // Add video url too
1571 url.push({
1572 type: 'Link',
1573 mimeType: 'text/html',
1574 href: CONFIG.WEBSERVER.URL + '/videos/watch/' + this.uuid
1575 })
1576
1577 const subtitleLanguage = []
1578 for (const caption of this.VideoCaptions) {
1579 subtitleLanguage.push({
1580 identifier: caption.language,
1581 name: VideoCaptionModel.getLanguageLabel(caption.language)
1582 })
1583 }
1584
1585 return {
1586 type: 'Video' as 'Video',
1587 id: this.url,
1588 name: this.name,
1589 duration: this.getActivityStreamDuration(),
1590 uuid: this.uuid,
1591 tag,
1592 category,
1593 licence,
1594 language,
1595 views: this.views,
1596 sensitive: this.nsfw,
1597 waitTranscoding: this.waitTranscoding,
1598 state: this.state,
1599 commentsEnabled: this.commentsEnabled,
1600 published: this.publishedAt.toISOString(),
1601 updated: this.updatedAt.toISOString(),
1602 mediaType: 'text/markdown',
1603 content: this.getTruncatedDescription(),
1604 support: this.support,
1605 subtitleLanguage,
1606 icon: {
1607 type: 'Image',
1608 url: this.getThumbnailUrl(baseUrlHttp),
1609 mediaType: 'image/jpeg',
1610 width: THUMBNAILS_SIZE.width,
1611 height: THUMBNAILS_SIZE.height
1612 },
1613 url,
1614 likes: getVideoLikesActivityPubUrl(this),
1615 dislikes: getVideoDislikesActivityPubUrl(this),
1616 shares: getVideoSharesActivityPubUrl(this),
1617 comments: getVideoCommentsActivityPubUrl(this),
1618 attributedTo: [
1619 {
1620 type: 'Person',
1621 id: this.VideoChannel.Account.Actor.url
1622 },
1623 {
1624 type: 'Group',
1625 id: this.VideoChannel.Actor.url
1626 }
1627 ]
1628 }
1629 } 1404 }
1630 1405
1631 getTruncatedDescription () { 1406 getTruncatedDescription () {
@@ -1635,130 +1410,13 @@ export class VideoModel extends Model<VideoModel> {
1635 return peertubeTruncate(this.description, maxLength) 1410 return peertubeTruncate(this.description, maxLength)
1636 } 1411 }
1637 1412
1638 async optimizeOriginalVideofile () {
1639 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
1640 const newExtname = '.mp4'
1641 const inputVideoFile = this.getOriginalFile()
1642 const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile))
1643 const videoTranscodedPath = join(videosDirectory, this.id + '-transcoded' + newExtname)
1644
1645 const transcodeOptions = {
1646 inputPath: videoInputPath,
1647 outputPath: videoTranscodedPath
1648 }
1649
1650 // Could be very long!
1651 await transcode(transcodeOptions)
1652
1653 try {
1654 await remove(videoInputPath)
1655
1656 // Important to do this before getVideoFilename() to take in account the new file extension
1657 inputVideoFile.set('extname', newExtname)
1658
1659 const videoOutputPath = this.getVideoFilePath(inputVideoFile)
1660 await rename(videoTranscodedPath, videoOutputPath)
1661 const stats = await stat(videoOutputPath)
1662 const fps = await getVideoFileFPS(videoOutputPath)
1663
1664 inputVideoFile.set('size', stats.size)
1665 inputVideoFile.set('fps', fps)
1666
1667 await this.createTorrentAndSetInfoHash(inputVideoFile)
1668 await inputVideoFile.save()
1669
1670 } catch (err) {
1671 // Auto destruction...
1672 this.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
1673
1674 throw err
1675 }
1676 }
1677
1678 async transcodeOriginalVideofile (resolution: VideoResolution, isPortraitMode: boolean) {
1679 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
1680 const extname = '.mp4'
1681
1682 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
1683 const videoInputPath = join(videosDirectory, this.getVideoFilename(this.getOriginalFile()))
1684
1685 const newVideoFile = new VideoFileModel({
1686 resolution,
1687 extname,
1688 size: 0,
1689 videoId: this.id
1690 })
1691 const videoOutputPath = join(videosDirectory, this.getVideoFilename(newVideoFile))
1692
1693 const transcodeOptions = {
1694 inputPath: videoInputPath,
1695 outputPath: videoOutputPath,
1696 resolution,
1697 isPortraitMode
1698 }
1699
1700 await transcode(transcodeOptions)
1701
1702 const stats = await stat(videoOutputPath)
1703 const fps = await getVideoFileFPS(videoOutputPath)
1704
1705 newVideoFile.set('size', stats.size)
1706 newVideoFile.set('fps', fps)
1707
1708 await this.createTorrentAndSetInfoHash(newVideoFile)
1709
1710 await newVideoFile.save()
1711
1712 this.VideoFiles.push(newVideoFile)
1713 }
1714
1715 async importVideoFile (inputFilePath: string) {
1716 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
1717 const { size } = await stat(inputFilePath)
1718 const fps = await getVideoFileFPS(inputFilePath)
1719
1720 let updatedVideoFile = new VideoFileModel({
1721 resolution: videoFileResolution,
1722 extname: extname(inputFilePath),
1723 size,
1724 fps,
1725 videoId: this.id
1726 })
1727
1728 const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution)
1729
1730 if (currentVideoFile) {
1731 // Remove old file and old torrent
1732 await this.removeFile(currentVideoFile)
1733 await this.removeTorrent(currentVideoFile)
1734 // Remove the old video file from the array
1735 this.VideoFiles = this.VideoFiles.filter(f => f !== currentVideoFile)
1736
1737 // Update the database
1738 currentVideoFile.set('extname', updatedVideoFile.extname)
1739 currentVideoFile.set('size', updatedVideoFile.size)
1740 currentVideoFile.set('fps', updatedVideoFile.fps)
1741
1742 updatedVideoFile = currentVideoFile
1743 }
1744
1745 const outputPath = this.getVideoFilePath(updatedVideoFile)
1746 await copy(inputFilePath, outputPath)
1747
1748 await this.createTorrentAndSetInfoHash(updatedVideoFile)
1749
1750 await updatedVideoFile.save()
1751
1752 this.VideoFiles.push(updatedVideoFile)
1753 }
1754
1755 getOriginalFileResolution () { 1413 getOriginalFileResolution () {
1756 const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) 1414 const originalFilePath = this.getVideoFilePath(this.getOriginalFile())
1757 1415
1758 return getVideoFileResolution(originalFilePath) 1416 return getVideoFileResolution(originalFilePath)
1759 } 1417 }
1760 1418
1761 getDescriptionPath () { 1419 getDescriptionAPIPath () {
1762 return `/api/${API_VERSION}/videos/${this.uuid}/description` 1420 return `/api/${API_VERSION}/videos/${this.uuid}/description`
1763 } 1421 }
1764 1422
@@ -1786,11 +1444,6 @@ export class VideoModel extends Model<VideoModel> {
1786 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) 1444 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
1787 } 1445 }
1788 1446
1789 getActivityStreamDuration () {
1790 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
1791 return 'PT' + this.duration + 'S'
1792 }
1793
1794 isOutdated () { 1447 isOutdated () {
1795 if (this.isOwned()) return false 1448 if (this.isOwned()) return false
1796 1449
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts
index b2922c5da..f5a19c5ea 100644
--- a/server/tests/api/server/jobs.ts
+++ b/server/tests/api/server/jobs.ts
@@ -45,7 +45,9 @@ describe('Test jobs', function () {
45 expect(res.body.total).to.be.above(2) 45 expect(res.body.total).to.be.above(2)
46 expect(res.body.data).to.have.lengthOf(1) 46 expect(res.body.data).to.have.lengthOf(1)
47 47
48 const job = res.body.data[0] 48 let job = res.body.data[0]
49 // Skip repeat jobs
50 if (job.type === 'videos-views') job = res.body.data[1]
49 51
50 expect(job.state).to.equal('completed') 52 expect(job.state).to.equal('completed')
51 expect(job.type).to.equal('activitypub-follow') 53 expect(job.type).to.equal('activitypub-follow')
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index c0ec75a45..6ce4b9dd1 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -6,15 +6,16 @@ import { VideoDetails } from '../../../../shared/models/videos'
6import { 6import {
7 doubleFollow, 7 doubleFollow,
8 flushAndRunMultipleServers, 8 flushAndRunMultipleServers,
9 flushTests,
10 getFollowingListPaginationAndSort, 9 getFollowingListPaginationAndSort,
11 getVideo, 10 getVideo,
11 immutableAssign,
12 killallServers, 12 killallServers,
13 root,
13 ServerInfo, 14 ServerInfo,
14 setAccessTokensToServers, 15 setAccessTokensToServers,
15 uploadVideo, 16 uploadVideo,
16 wait, 17 viewVideo,
17 root, viewVideo 18 wait
18} from '../../utils' 19} from '../../utils'
19import { waitJobs } from '../../utils/server/jobs' 20import { waitJobs } from '../../utils/server/jobs'
20import * as magnetUtil from 'magnet-uri' 21import * as magnetUtil from 'magnet-uri'
@@ -22,9 +23,16 @@ import { updateRedundancy } from '../../utils/server/redundancy'
22import { ActorFollow } from '../../../../shared/models/actors' 23import { ActorFollow } from '../../../../shared/models/actors'
23import { readdir } from 'fs-extra' 24import { readdir } from 'fs-extra'
24import { join } from 'path' 25import { join } from 'path'
26import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
27import { getStats } from '../../utils/server/stats'
28import { ServerStats } from '../../../../shared/models/server/server-stats.model'
25 29
26const expect = chai.expect 30const expect = chai.expect
27 31
32let servers: ServerInfo[] = []
33let video1Server2UUID: string
34let video2Server2UUID: string
35
28function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { 36function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
29 const parsed = magnetUtil.decode(file.magnetUri) 37 const parsed = magnetUtil.decode(file.magnetUri)
30 38
@@ -34,84 +42,105 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
34 } 42 }
35} 43}
36 44
37describe('Test videos redundancy', function () { 45async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
38 let servers: ServerInfo[] = [] 46 const config = {
39 let video1Server2UUID: string 47 redundancy: {
40 let video2Server2UUID: string 48 videos: {
41 49 check_interval: '5 seconds',
42 before(async function () { 50 strategies: [
43 this.timeout(120000) 51 immutableAssign({
44 52 strategy: strategy,
45 servers = await flushAndRunMultipleServers(3) 53 size: '100KB'
54 }, additionalParams)
55 ]
56 }
57 }
58 }
59 servers = await flushAndRunMultipleServers(3, config)
46 60
47 // Get the access tokens 61 // Get the access tokens
48 await setAccessTokensToServers(servers) 62 await setAccessTokensToServers(servers)
49 63
50 { 64 {
51 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) 65 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
52 video1Server2UUID = res.body.video.uuid 66 video1Server2UUID = res.body.video.uuid
53 67
54 await viewVideo(servers[1].url, video1Server2UUID) 68 await viewVideo(servers[ 1 ].url, video1Server2UUID)
55 } 69 }
56 70
57 { 71 {
58 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) 72 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
59 video2Server2UUID = res.body.video.uuid 73 video2Server2UUID = res.body.video.uuid
60 } 74 }
61 75
62 await waitJobs(servers) 76 await waitJobs(servers)
63 77
64 // Server 1 and server 2 follow each other 78 // Server 1 and server 2 follow each other
65 await doubleFollow(servers[0], servers[1]) 79 await doubleFollow(servers[ 0 ], servers[ 1 ])
66 // Server 1 and server 3 follow each other 80 // Server 1 and server 3 follow each other
67 await doubleFollow(servers[0], servers[2]) 81 await doubleFollow(servers[ 0 ], servers[ 2 ])
68 // Server 2 and server 3 follow each other 82 // Server 2 and server 3 follow each other
69 await doubleFollow(servers[1], servers[2]) 83 await doubleFollow(servers[ 1 ], servers[ 2 ])
70 84
71 await waitJobs(servers) 85 await waitJobs(servers)
72 }) 86}
73 87
74 it('Should have 1 webseed on the first video', async function () { 88async function check1WebSeed (strategy: VideoRedundancyStrategy) {
75 const webseeds = [ 89 const webseeds = [
76 'http://localhost:9002/static/webseed/' + video1Server2UUID 90 'http://localhost:9002/static/webseed/' + video1Server2UUID
77 ] 91 ]
78 92
79 for (const server of servers) { 93 for (const server of servers) {
94 {
80 const res = await getVideo(server.url, video1Server2UUID) 95 const res = await getVideo(server.url, video1Server2UUID)
81 96
82 const video: VideoDetails = res.body 97 const video: VideoDetails = res.body
83 video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) 98 video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
84 } 99 }
85 })
86 100
87 it('Should enable redundancy on server 1', async function () { 101 {
88 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) 102 const res = await getStats(server.url)
103 const data: ServerStats = res.body
89 104
90 const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') 105 expect(data.videosRedundancy).to.have.lengthOf(1)
91 const follows: ActorFollow[] = res.body.data
92 const server2 = follows.find(f => f.following.host === 'localhost:9002')
93 const server3 = follows.find(f => f.following.host === 'localhost:9003')
94 106
95 expect(server3).to.not.be.undefined 107 const stat = data.videosRedundancy[0]
96 expect(server3.following.hostRedundancyAllowed).to.be.false 108 expect(stat.strategy).to.equal(strategy)
109 expect(stat.totalSize).to.equal(102400)
110 expect(stat.totalUsed).to.equal(0)
111 expect(stat.totalVideoFiles).to.equal(0)
112 expect(stat.totalVideos).to.equal(0)
113 }
114 }
115}
97 116
98 expect(server2).to.not.be.undefined 117async function enableRedundancy () {
99 expect(server2.following.hostRedundancyAllowed).to.be.true 118 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
100 })
101 119
102 it('Should have 2 webseed on the first video', async function () { 120 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
103 this.timeout(40000) 121 const follows: ActorFollow[] = res.body.data
122 const server2 = follows.find(f => f.following.host === 'localhost:9002')
123 const server3 = follows.find(f => f.following.host === 'localhost:9003')
104 124
105 await waitJobs(servers) 125 expect(server3).to.not.be.undefined
106 await wait(15000) 126 expect(server3.following.hostRedundancyAllowed).to.be.false
107 await waitJobs(servers)
108 127
109 const webseeds = [ 128 expect(server2).to.not.be.undefined
110 'http://localhost:9001/static/webseed/' + video1Server2UUID, 129 expect(server2.following.hostRedundancyAllowed).to.be.true
111 'http://localhost:9002/static/webseed/' + video1Server2UUID 130}
112 ] 131
132async function check2Webseeds (strategy: VideoRedundancyStrategy) {
133 await waitJobs(servers)
134 await wait(15000)
135 await waitJobs(servers)
113 136
114 for (const server of servers) { 137 const webseeds = [
138 'http://localhost:9001/static/webseed/' + video1Server2UUID,
139 'http://localhost:9002/static/webseed/' + video1Server2UUID
140 ]
141
142 for (const server of servers) {
143 {
115 const res = await getVideo(server.url, video1Server2UUID) 144 const res = await getVideo(server.url, video1Server2UUID)
116 145
117 const video: VideoDetails = res.body 146 const video: VideoDetails = res.body
@@ -120,21 +149,137 @@ describe('Test videos redundancy', function () {
120 checkMagnetWebseeds(file, webseeds) 149 checkMagnetWebseeds(file, webseeds)
121 } 150 }
122 } 151 }
152 }
123 153
124 const files = await readdir(join(root(), 'test1', 'videos')) 154 const files = await readdir(join(root(), 'test1', 'videos'))
125 expect(files).to.have.lengthOf(4) 155 expect(files).to.have.lengthOf(4)
126 156
127 for (const resolution of [ 240, 360, 480, 720 ]) { 157 for (const resolution of [ 240, 360, 480, 720 ]) {
128 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined 158 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
129 } 159 }
160
161 {
162 const res = await getStats(servers[0].url)
163 const data: ServerStats = res.body
164
165 expect(data.videosRedundancy).to.have.lengthOf(1)
166 const stat = data.videosRedundancy[0]
167
168 expect(stat.strategy).to.equal(strategy)
169 expect(stat.totalSize).to.equal(102400)
170 expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
171 expect(stat.totalVideoFiles).to.equal(4)
172 expect(stat.totalVideos).to.equal(1)
173 }
174}
175
176async function cleanServers () {
177 killallServers(servers)
178}
179
180describe('Test videos redundancy', function () {
181
182 describe('With most-views strategy', function () {
183 const strategy = 'most-views'
184
185 before(function () {
186 this.timeout(120000)
187
188 return runServers(strategy)
189 })
190
191 it('Should have 1 webseed on the first video', function () {
192 return check1WebSeed(strategy)
193 })
194
195 it('Should enable redundancy on server 1', function () {
196 return enableRedundancy()
197 })
198
199 it('Should have 2 webseed on the first video', function () {
200 this.timeout(40000)
201
202 return check2Webseeds(strategy)
203 })
204
205 after(function () {
206 return cleanServers()
207 })
130 }) 208 })
131 209
132 after(async function () { 210 describe('With trending strategy', function () {
133 killallServers(servers) 211 const strategy = 'trending'
134 212
135 // Keep the logs if the test failed 213 before(function () {
136 if (this['ok']) { 214 this.timeout(120000)
137 await flushTests() 215
138 } 216 return runServers(strategy)
217 })
218
219 it('Should have 1 webseed on the first video', function () {
220 return check1WebSeed(strategy)
221 })
222
223 it('Should enable redundancy on server 1', function () {
224 return enableRedundancy()
225 })
226
227 it('Should have 2 webseed on the first video', function () {
228 this.timeout(40000)
229
230 return check2Webseeds(strategy)
231 })
232
233 after(function () {
234 return cleanServers()
235 })
236 })
237
238 describe('With recently added strategy', function () {
239 const strategy = 'recently-added'
240
241 before(function () {
242 this.timeout(120000)
243
244 return runServers(strategy, { minViews: 3 })
245 })
246
247 it('Should have 1 webseed on the first video', function () {
248 return check1WebSeed(strategy)
249 })
250
251 it('Should enable redundancy on server 1', function () {
252 return enableRedundancy()
253 })
254
255 it('Should still have 1 webseed on the first video', async function () {
256 this.timeout(40000)
257
258 await waitJobs(servers)
259 await wait(15000)
260 await waitJobs(servers)
261
262 return check1WebSeed(strategy)
263 })
264
265 it('Should view 2 times the first video', async function () {
266 this.timeout(40000)
267
268 await viewVideo(servers[ 0 ].url, video1Server2UUID)
269 await viewVideo(servers[ 2 ].url, video1Server2UUID)
270
271 await wait(10000)
272 await waitJobs(servers)
273 })
274
275 it('Should have 2 webseed on the first video', function () {
276 this.timeout(40000)
277
278 return check2Webseeds(strategy)
279 })
280
281 after(function () {
282 return cleanServers()
283 })
139 }) 284 })
140}) 285})
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts
index fc9b88805..cb229e876 100644
--- a/server/tests/api/server/stats.ts
+++ b/server/tests/api/server/stats.ts
@@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs'
21 21
22const expect = chai.expect 22const expect = chai.expect
23 23
24describe('Test stats', function () { 24describe('Test stats (excluding redundancy)', function () {
25 let servers: ServerInfo[] = [] 25 let servers: ServerInfo[] = []
26 26
27 before(async function () { 27 before(async function () {
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 1372c03c3..26ab4e1bb 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -35,7 +35,7 @@ interface ServerInfo {
35 } 35 }
36} 36}
37 37
38function flushAndRunMultipleServers (totalServers) { 38function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) {
39 let apps = [] 39 let apps = []
40 let i = 0 40 let i = 0
41 41
@@ -51,10 +51,7 @@ function flushAndRunMultipleServers (totalServers) {
51 flushTests() 51 flushTests()
52 .then(() => { 52 .then(() => {
53 for (let j = 1; j <= totalServers; j++) { 53 for (let j = 1; j <= totalServers; j++) {
54 // For the virtual buffer 54 runServer(j, configOverride).then(app => anotherServerDone(j, app))
55 setTimeout(() => {
56 runServer(j).then(app => anotherServerDone(j, app))
57 }, 1000 * (j - 1))
58 } 55 }
59 }) 56 })
60 }) 57 })
diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts
index 9cdec6cff..01989d952 100644
--- a/server/tests/utils/server/stats.ts
+++ b/server/tests/utils/server/stats.ts
@@ -1,11 +1,16 @@
1import { makeGetRequest } from '../' 1import { makeGetRequest } from '../'
2 2
3function getStats (url: string) { 3function getStats (url: string, useCache = false) {
4 const path = '/api/v1/server/stats' 4 const path = '/api/v1/server/stats'
5 5
6 const query = {
7 t: useCache ? undefined : new Date().getTime()
8 }
9
6 return makeGetRequest({ 10 return makeGetRequest({
7 url, 11 url,
8 path, 12 path,
13 query,
9 statusCodeExpected: 200 14 statusCodeExpected: 200
10 }) 15 })
11} 16}