diff options
Diffstat (limited to 'server')
78 files changed, 1984 insertions, 1457 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 2e168ea78..6229c44aa 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -6,7 +6,13 @@ import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' | |||
6 | import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send' | 6 | import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send' |
7 | import { audiencify, getAudience } from '../../lib/activitypub/audience' | 7 | import { audiencify, getAudience } from '../../lib/activitypub/audience' |
8 | import { buildCreateActivity } from '../../lib/activitypub/send/send-create' | 8 | import { buildCreateActivity } from '../../lib/activitypub/send/send-create' |
9 | import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' | 9 | import { |
10 | asyncMiddleware, | ||
11 | executeIfActivityPub, | ||
12 | localAccountValidator, | ||
13 | localVideoChannelValidator, | ||
14 | videosCustomGetValidator | ||
15 | } from '../../middlewares' | ||
10 | import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' | 16 | import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' |
11 | import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' | 17 | import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' |
12 | import { AccountModel } from '../../models/account/account' | 18 | import { AccountModel } from '../../models/account/account' |
@@ -54,7 +60,7 @@ activityPubClientRouter.get('/videos/watch/:id/activity', | |||
54 | executeIfActivityPub(asyncMiddleware(videoController)) | 60 | executeIfActivityPub(asyncMiddleware(videoController)) |
55 | ) | 61 | ) |
56 | activityPubClientRouter.get('/videos/watch/:id/announces', | 62 | activityPubClientRouter.get('/videos/watch/:id/announces', |
57 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 63 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), |
58 | executeIfActivityPub(asyncMiddleware(videoAnnouncesController)) | 64 | executeIfActivityPub(asyncMiddleware(videoAnnouncesController)) |
59 | ) | 65 | ) |
60 | activityPubClientRouter.get('/videos/watch/:id/announces/:accountId', | 66 | activityPubClientRouter.get('/videos/watch/:id/announces/:accountId', |
@@ -62,15 +68,15 @@ activityPubClientRouter.get('/videos/watch/:id/announces/:accountId', | |||
62 | executeIfActivityPub(asyncMiddleware(videoAnnounceController)) | 68 | executeIfActivityPub(asyncMiddleware(videoAnnounceController)) |
63 | ) | 69 | ) |
64 | activityPubClientRouter.get('/videos/watch/:id/likes', | 70 | activityPubClientRouter.get('/videos/watch/:id/likes', |
65 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 71 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), |
66 | executeIfActivityPub(asyncMiddleware(videoLikesController)) | 72 | executeIfActivityPub(asyncMiddleware(videoLikesController)) |
67 | ) | 73 | ) |
68 | activityPubClientRouter.get('/videos/watch/:id/dislikes', | 74 | activityPubClientRouter.get('/videos/watch/:id/dislikes', |
69 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 75 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), |
70 | executeIfActivityPub(asyncMiddleware(videoDislikesController)) | 76 | executeIfActivityPub(asyncMiddleware(videoDislikesController)) |
71 | ) | 77 | ) |
72 | activityPubClientRouter.get('/videos/watch/:id/comments', | 78 | activityPubClientRouter.get('/videos/watch/:id/comments', |
73 | executeIfActivityPub(asyncMiddleware(videosGetValidator)), | 79 | executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video'))), |
74 | executeIfActivityPub(asyncMiddleware(videoCommentsController)) | 80 | executeIfActivityPub(asyncMiddleware(videoCommentsController)) |
75 | ) | 81 | ) |
76 | activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', | 82 | activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', |
diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts index 20bd20ed4..738d155eb 100644 --- a/server/controllers/activitypub/inbox.ts +++ b/server/controllers/activitypub/inbox.ts | |||
@@ -7,6 +7,8 @@ import { asyncMiddleware, checkSignature, localAccountValidator, localVideoChann | |||
7 | import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' | 7 | import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' |
8 | import { VideoChannelModel } from '../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../models/video/video-channel' |
9 | import { AccountModel } from '../../models/account/account' | 9 | import { AccountModel } from '../../models/account/account' |
10 | import { queue } from 'async' | ||
11 | import { ActorModel } from '../../models/activitypub/actor' | ||
10 | 12 | ||
11 | const inboxRouter = express.Router() | 13 | const inboxRouter = express.Router() |
12 | 14 | ||
@@ -14,7 +16,7 @@ inboxRouter.post('/inbox', | |||
14 | signatureValidator, | 16 | signatureValidator, |
15 | asyncMiddleware(checkSignature), | 17 | asyncMiddleware(checkSignature), |
16 | asyncMiddleware(activityPubValidator), | 18 | asyncMiddleware(activityPubValidator), |
17 | asyncMiddleware(inboxController) | 19 | inboxController |
18 | ) | 20 | ) |
19 | 21 | ||
20 | inboxRouter.post('/accounts/:name/inbox', | 22 | inboxRouter.post('/accounts/:name/inbox', |
@@ -22,14 +24,14 @@ inboxRouter.post('/accounts/:name/inbox', | |||
22 | asyncMiddleware(checkSignature), | 24 | asyncMiddleware(checkSignature), |
23 | asyncMiddleware(localAccountValidator), | 25 | asyncMiddleware(localAccountValidator), |
24 | asyncMiddleware(activityPubValidator), | 26 | asyncMiddleware(activityPubValidator), |
25 | asyncMiddleware(inboxController) | 27 | inboxController |
26 | ) | 28 | ) |
27 | inboxRouter.post('/video-channels/:name/inbox', | 29 | inboxRouter.post('/video-channels/:name/inbox', |
28 | signatureValidator, | 30 | signatureValidator, |
29 | asyncMiddleware(checkSignature), | 31 | asyncMiddleware(checkSignature), |
30 | asyncMiddleware(localVideoChannelValidator), | 32 | asyncMiddleware(localVideoChannelValidator), |
31 | asyncMiddleware(activityPubValidator), | 33 | asyncMiddleware(activityPubValidator), |
32 | asyncMiddleware(inboxController) | 34 | inboxController |
33 | ) | 35 | ) |
34 | 36 | ||
35 | // --------------------------------------------------------------------------- | 37 | // --------------------------------------------------------------------------- |
@@ -40,7 +42,12 @@ export { | |||
40 | 42 | ||
41 | // --------------------------------------------------------------------------- | 43 | // --------------------------------------------------------------------------- |
42 | 44 | ||
43 | async function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) { | 45 | const inboxQueue = queue<{ activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel }, Error>((task, cb) => { |
46 | processActivities(task.activities, task.signatureActor, task.inboxActor) | ||
47 | .then(() => cb()) | ||
48 | }) | ||
49 | |||
50 | function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
44 | const rootActivity: RootActivity = req.body | 51 | const rootActivity: RootActivity = req.body |
45 | let activities: Activity[] = [] | 52 | let activities: Activity[] = [] |
46 | 53 | ||
@@ -66,7 +73,11 @@ async function inboxController (req: express.Request, res: express.Response, nex | |||
66 | 73 | ||
67 | logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url) | 74 | logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url) |
68 | 75 | ||
69 | await processActivities(activities, res.locals.signature.actor, accountOrChannel ? accountOrChannel.Actor : undefined) | 76 | inboxQueue.push({ |
77 | activities, | ||
78 | signatureActor: res.locals.signature.actor, | ||
79 | inboxActor: accountOrChannel ? accountOrChannel.Actor : undefined | ||
80 | }) | ||
70 | 81 | ||
71 | res.status(204).end() | 82 | return res.status(204).end() |
72 | } | 83 | } |
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 6edbe4820..95549b724 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -8,7 +8,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers' | |||
8 | import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' | 8 | import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' |
9 | import { customConfigUpdateValidator } from '../../middlewares/validators/config' | 9 | import { customConfigUpdateValidator } from '../../middlewares/validators/config' |
10 | import { ClientHtml } from '../../lib/client-html' | 10 | import { ClientHtml } from '../../lib/client-html' |
11 | import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger' | 11 | import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger' |
12 | import { remove, writeJSON } from 'fs-extra' | 12 | import { remove, writeJSON } from 'fs-extra' |
13 | 13 | ||
14 | const packageJSON = require('../../../../package.json') | 14 | const packageJSON = require('../../../../package.json') |
@@ -134,10 +134,7 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex | |||
134 | async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { | 134 | async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { |
135 | await remove(CONFIG.CUSTOM_FILE) | 135 | await remove(CONFIG.CUSTOM_FILE) |
136 | 136 | ||
137 | auditLogger.delete( | 137 | auditLogger.delete(getAuditIdFromRes(res), new CustomConfigAuditView(customConfig())) |
138 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
139 | new CustomConfigAuditView(customConfig()) | ||
140 | ) | ||
141 | 138 | ||
142 | reloadConfig() | 139 | reloadConfig() |
143 | ClientHtml.invalidCache() | 140 | ClientHtml.invalidCache() |
@@ -183,7 +180,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response, | |||
183 | const data = customConfig() | 180 | const data = customConfig() |
184 | 181 | ||
185 | auditLogger.update( | 182 | auditLogger.update( |
186 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | 183 | getAuditIdFromRes(res), |
187 | new CustomConfigAuditView(data), | 184 | new CustomConfigAuditView(data), |
188 | oldCustomConfigAuditKeys | 185 | oldCustomConfigAuditKeys |
189 | ) | 186 | ) |
diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts index da941c0ac..8b6773056 100644 --- a/server/controllers/api/overviews.ts +++ b/server/controllers/api/overviews.ts | |||
@@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video' | |||
4 | import { asyncMiddleware } from '../../middlewares' | 4 | import { asyncMiddleware } from '../../middlewares' |
5 | import { TagModel } from '../../models/video/tag' | 5 | import { TagModel } from '../../models/video/tag' |
6 | import { VideosOverview } from '../../../shared/models/overviews' | 6 | import { VideosOverview } from '../../../shared/models/overviews' |
7 | import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' | 7 | import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' |
8 | import { cacheRoute } from '../../middlewares/cache' | 8 | import { cacheRoute } from '../../middlewares/cache' |
9 | import * as memoizee from 'memoizee' | ||
9 | 10 | ||
10 | const overviewsRouter = express.Router() | 11 | const overviewsRouter = express.Router() |
11 | 12 | ||
@@ -20,13 +21,30 @@ export { overviewsRouter } | |||
20 | 21 | ||
21 | // --------------------------------------------------------------------------- | 22 | // --------------------------------------------------------------------------- |
22 | 23 | ||
24 | const buildSamples = memoizee(async function () { | ||
25 | const [ categories, channels, tags ] = await Promise.all([ | ||
26 | VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), | ||
27 | VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), | ||
28 | TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) | ||
29 | ]) | ||
30 | |||
31 | return { categories, channels, tags } | ||
32 | }, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) | ||
33 | |||
23 | // This endpoint could be quite long, but we cache it | 34 | // This endpoint could be quite long, but we cache it |
24 | async function getVideosOverview (req: express.Request, res: express.Response) { | 35 | async function getVideosOverview (req: express.Request, res: express.Response) { |
25 | const attributes = await buildSamples() | 36 | const attributes = await buildSamples() |
37 | |||
38 | const [ categories, channels, tags ] = await Promise.all([ | ||
39 | Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), | ||
40 | Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), | ||
41 | Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) | ||
42 | ]) | ||
43 | |||
26 | const result: VideosOverview = { | 44 | const result: VideosOverview = { |
27 | categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), | 45 | categories, |
28 | channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), | 46 | channels, |
29 | tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) | 47 | tags |
30 | } | 48 | } |
31 | 49 | ||
32 | // Cleanup our object | 50 | // Cleanup our object |
@@ -37,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) { | |||
37 | return res.json(result) | 55 | return res.json(result) |
38 | } | 56 | } |
39 | 57 | ||
40 | async function buildSamples () { | ||
41 | const [ categories, channels, tags ] = await Promise.all([ | ||
42 | VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), | ||
43 | VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), | ||
44 | TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) | ||
45 | ]) | ||
46 | |||
47 | return { categories, channels, tags } | ||
48 | } | ||
49 | |||
50 | async function getVideosByTag (tag: string, res: express.Response) { | 58 | async function getVideosByTag (tag: string, res: express.Response) { |
51 | const videos = await getVideos(res, { tagsOneOf: [ tag ] }) | 59 | const videos = await getVideos(res, { tagsOneOf: [ tag ] }) |
52 | 60 | ||
@@ -84,14 +92,16 @@ async function getVideos ( | |||
84 | res: express.Response, | 92 | res: express.Response, |
85 | where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } | 93 | where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } |
86 | ) { | 94 | ) { |
87 | const { data } = await VideoModel.listForApi(Object.assign({ | 95 | const query = Object.assign({ |
88 | start: 0, | 96 | start: 0, |
89 | count: 10, | 97 | count: 10, |
90 | sort: '-createdAt', | 98 | sort: '-createdAt', |
91 | includeLocalVideos: true, | 99 | includeLocalVideos: true, |
92 | nsfw: buildNSFWFilter(res), | 100 | nsfw: buildNSFWFilter(res), |
93 | withFiles: false | 101 | withFiles: false |
94 | }, where)) | 102 | }, where) |
103 | |||
104 | const { data } = await VideoModel.listForApi(query, false) | ||
95 | 105 | ||
96 | return data.map(d => d.toFormattedJSON()) | 106 | return data.map(d => d.toFormattedJSON()) |
97 | } | 107 | } |
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 28a7a04ca..fd4db7a54 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts | |||
@@ -56,6 +56,9 @@ function searchVideoChannels (req: express.Request, res: express.Response) { | |||
56 | const isURISearch = search.startsWith('http://') || search.startsWith('https://') | 56 | const isURISearch = search.startsWith('http://') || search.startsWith('https://') |
57 | 57 | ||
58 | const parts = search.split('@') | 58 | const parts = search.split('@') |
59 | |||
60 | // Handle strings like @toto@example.com | ||
61 | if (parts.length === 3 && parts[0].length === 0) parts.shift() | ||
59 | const isWebfingerSearch = parts.length === 2 && parts.every(p => p.indexOf(' ') === -1) | 62 | const isWebfingerSearch = parts.length === 2 && parts.every(p => p.indexOf(' ') === -1) |
60 | 63 | ||
61 | if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res) | 64 | if (isURISearch || isWebfingerSearch) return searchVideoChannelURI(search, isWebfingerSearch, res) |
@@ -86,7 +89,7 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean | |||
86 | 89 | ||
87 | if (isUserAbleToSearchRemoteURI(res)) { | 90 | if (isUserAbleToSearchRemoteURI(res)) { |
88 | try { | 91 | try { |
89 | const actor = await getOrCreateActorAndServerAndModel(uri, true, true) | 92 | const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true) |
90 | videoChannel = actor.VideoChannel | 93 | videoChannel = actor.VideoChannel |
91 | } catch (err) { | 94 | } catch (err) { |
92 | logger.info('Cannot search remote video channel %s.', uri, { err }) | 95 | logger.info('Cannot search remote video channel %s.', uri, { err }) |
@@ -136,7 +139,7 @@ async function searchVideoURI (url: string, res: express.Response) { | |||
136 | refreshVideo: false | 139 | refreshVideo: false |
137 | } | 140 | } |
138 | 141 | ||
139 | const result = await getOrCreateVideoAndAccountAndChannel(url, syncParam) | 142 | const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) |
140 | video = result ? result.video : undefined | 143 | video = result ? result.video : undefined |
141 | } catch (err) { | 144 | } catch (err) { |
142 | logger.info('Cannot search remote video %s.', url, { err }) | 145 | logger.info('Cannot search remote video %s.', url, { err }) |
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 6f4fe938c..85803f69e 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts | |||
@@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user' | |||
5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
6 | import { VideoModel } from '../../../models/video/video' | 6 | import { VideoModel } from '../../../models/video/video' |
7 | import { VideoCommentModel } from '../../../models/video/video-comment' | 7 | import { VideoCommentModel } from '../../../models/video/video-comment' |
8 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | ||
9 | import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' | ||
10 | import { cacheRoute } from '../../../middlewares/cache' | ||
8 | 11 | ||
9 | const statsRouter = express.Router() | 12 | const statsRouter = express.Router() |
10 | 13 | ||
11 | statsRouter.get('/stats', | 14 | statsRouter.get('/stats', |
15 | asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)), | ||
12 | asyncMiddleware(getStats) | 16 | asyncMiddleware(getStats) |
13 | ) | 17 | ) |
14 | 18 | ||
@@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr | |||
18 | const { totalUsers } = await UserModel.getStats() | 22 | const { totalUsers } = await UserModel.getStats() |
19 | const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() | 23 | const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() |
20 | 24 | ||
25 | const videosRedundancyStats = await Promise.all( | ||
26 | CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { | ||
27 | return VideoRedundancyModel.getStats(r.strategy) | ||
28 | .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) | ||
29 | }) | ||
30 | ) | ||
31 | |||
21 | const data: ServerStats = { | 32 | const data: ServerStats = { |
22 | totalLocalVideos, | 33 | totalLocalVideos, |
23 | totalLocalVideoViews, | 34 | totalLocalVideoViews, |
@@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr | |||
26 | totalVideoComments, | 37 | totalVideoComments, |
27 | totalUsers, | 38 | totalUsers, |
28 | totalInstanceFollowers, | 39 | totalInstanceFollowers, |
29 | totalInstanceFollowing | 40 | totalInstanceFollowing, |
41 | videosRedundancy: videosRedundancyStats | ||
30 | } | 42 | } |
31 | 43 | ||
32 | return res.json(data).end() | 44 | return res.json(data).end() |
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 07edf3727..8b8ebcd23 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts | |||
@@ -27,13 +27,17 @@ import { | |||
27 | usersUpdateValidator | 27 | usersUpdateValidator |
28 | } from '../../../middlewares' | 28 | } from '../../../middlewares' |
29 | import { | 29 | import { |
30 | usersAskResetPasswordValidator, usersBlockingValidator, usersResetPasswordValidator, | 30 | usersAskResetPasswordValidator, |
31 | usersAskSendVerifyEmailValidator, usersVerifyEmailValidator | 31 | usersAskSendVerifyEmailValidator, |
32 | usersBlockingValidator, | ||
33 | usersResetPasswordValidator, | ||
34 | usersVerifyEmailValidator | ||
32 | } from '../../../middlewares/validators' | 35 | } from '../../../middlewares/validators' |
33 | import { UserModel } from '../../../models/account/user' | 36 | import { UserModel } from '../../../models/account/user' |
34 | import { OAuthTokenModel } from '../../../models/oauth/oauth-token' | 37 | import { OAuthTokenModel } from '../../../models/oauth/oauth-token' |
35 | import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' | 38 | import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' |
36 | import { meRouter } from './me' | 39 | import { meRouter } from './me' |
40 | import { deleteUserToken } from '../../../lib/oauth-model' | ||
37 | 41 | ||
38 | const auditLogger = auditLoggerFactory('users') | 42 | const auditLogger = auditLoggerFactory('users') |
39 | 43 | ||
@@ -166,7 +170,7 @@ async function createUser (req: express.Request, res: express.Response) { | |||
166 | 170 | ||
167 | const { user, account } = await createUserAccountAndChannel(userToCreate) | 171 | const { user, account } = await createUserAccountAndChannel(userToCreate) |
168 | 172 | ||
169 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) | 173 | auditLogger.create(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) |
170 | logger.info('User %s with its channel and account created.', body.username) | 174 | logger.info('User %s with its channel and account created.', body.username) |
171 | 175 | ||
172 | return res.json({ | 176 | return res.json({ |
@@ -245,7 +249,7 @@ async function removeUser (req: express.Request, res: express.Response, next: ex | |||
245 | 249 | ||
246 | await user.destroy() | 250 | await user.destroy() |
247 | 251 | ||
248 | auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) | 252 | auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) |
249 | 253 | ||
250 | return res.sendStatus(204) | 254 | return res.sendStatus(204) |
251 | } | 255 | } |
@@ -264,15 +268,9 @@ async function updateUser (req: express.Request, res: express.Response, next: ex | |||
264 | const user = await userToUpdate.save() | 268 | const user = await userToUpdate.save() |
265 | 269 | ||
266 | // Destroy user token to refresh rights | 270 | // Destroy user token to refresh rights |
267 | if (roleChanged) { | 271 | if (roleChanged) await deleteUserToken(userToUpdate.id) |
268 | await OAuthTokenModel.deleteUserToken(userToUpdate.id) | ||
269 | } | ||
270 | 272 | ||
271 | auditLogger.update( | 273 | auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) |
272 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
273 | new UserAuditView(user.toFormattedJSON()), | ||
274 | oldUserAuditView | ||
275 | ) | ||
276 | 274 | ||
277 | // Don't need to send this update to followers, these attributes are not propagated | 275 | // Don't need to send this update to followers, these attributes are not propagated |
278 | 276 | ||
@@ -333,16 +331,12 @@ async function changeUserBlock (res: express.Response, user: UserModel, block: b | |||
333 | user.blockedReason = reason || null | 331 | user.blockedReason = reason || null |
334 | 332 | ||
335 | await sequelizeTypescript.transaction(async t => { | 333 | await sequelizeTypescript.transaction(async t => { |
336 | await OAuthTokenModel.deleteUserToken(user.id, t) | 334 | await deleteUserToken(user.id, t) |
337 | 335 | ||
338 | await user.save({ transaction: t }) | 336 | await user.save({ transaction: t }) |
339 | }) | 337 | }) |
340 | 338 | ||
341 | await Emailer.Instance.addUserBlockJob(user, block, reason) | 339 | await Emailer.Instance.addUserBlockJob(user, block, reason) |
342 | 340 | ||
343 | auditLogger.update( | 341 | auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) |
344 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
345 | new UserAuditView(user.toFormattedJSON()), | ||
346 | oldUserAuditView | ||
347 | ) | ||
348 | } | 342 | } |
diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index e886d4b2a..ff3a87b7f 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts | |||
@@ -5,7 +5,8 @@ import { getFormattedObjects } from '../../../helpers/utils' | |||
5 | import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers' | 5 | import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers' |
6 | import { sendUpdateActor } from '../../../lib/activitypub/send' | 6 | import { sendUpdateActor } from '../../../lib/activitypub/send' |
7 | import { | 7 | import { |
8 | asyncMiddleware, asyncRetryTransactionMiddleware, | 8 | asyncMiddleware, |
9 | asyncRetryTransactionMiddleware, | ||
9 | authenticate, | 10 | authenticate, |
10 | commonVideosFiltersValidator, | 11 | commonVideosFiltersValidator, |
11 | paginationValidator, | 12 | paginationValidator, |
@@ -17,11 +18,11 @@ import { | |||
17 | usersVideoRatingValidator | 18 | usersVideoRatingValidator |
18 | } from '../../../middlewares' | 19 | } from '../../../middlewares' |
19 | import { | 20 | import { |
21 | areSubscriptionsExistValidator, | ||
20 | deleteMeValidator, | 22 | deleteMeValidator, |
21 | userSubscriptionsSortValidator, | 23 | userSubscriptionsSortValidator, |
22 | videoImportsSortValidator, | 24 | videoImportsSortValidator, |
23 | videosSortValidator, | 25 | videosSortValidator |
24 | areSubscriptionsExistValidator | ||
25 | } from '../../../middlewares/validators' | 26 | } from '../../../middlewares/validators' |
26 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 27 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
27 | import { UserModel } from '../../../models/account/user' | 28 | import { UserModel } from '../../../models/account/user' |
@@ -31,12 +32,13 @@ import { buildNSFWFilter, createReqFiles } from '../../../helpers/express-utils' | |||
31 | import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model' | 32 | import { UserVideoQuota } from '../../../../shared/models/users/user-video-quota.model' |
32 | import { updateAvatarValidator } from '../../../middlewares/validators/avatar' | 33 | import { updateAvatarValidator } from '../../../middlewares/validators/avatar' |
33 | import { updateActorAvatarFile } from '../../../lib/avatar' | 34 | import { updateActorAvatarFile } from '../../../lib/avatar' |
34 | import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' | 35 | import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' |
35 | import { VideoImportModel } from '../../../models/video/video-import' | 36 | import { VideoImportModel } from '../../../models/video/video-import' |
36 | import { VideoFilter } from '../../../../shared/models/videos/video-query.type' | 37 | import { VideoFilter } from '../../../../shared/models/videos/video-query.type' |
37 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 38 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
38 | import { JobQueue } from '../../../lib/job-queue' | 39 | import { JobQueue } from '../../../lib/job-queue' |
39 | import { logger } from '../../../helpers/logger' | 40 | import { logger } from '../../../helpers/logger' |
41 | import { AccountModel } from '../../../models/account/account' | ||
40 | 42 | ||
41 | const auditLogger = auditLoggerFactory('users-me') | 43 | const auditLogger = auditLoggerFactory('users-me') |
42 | 44 | ||
@@ -293,7 +295,7 @@ async function getUserVideoQuotaUsed (req: express.Request, res: express.Respons | |||
293 | } | 295 | } |
294 | 296 | ||
295 | async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { | 297 | async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { |
296 | const videoId = +req.params.videoId | 298 | const videoId = res.locals.video.id |
297 | const accountId = +res.locals.oauth.token.User.Account.id | 299 | const accountId = +res.locals.oauth.token.User.Account.id |
298 | 300 | ||
299 | const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null) | 301 | const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null) |
@@ -311,7 +313,7 @@ async function deleteMe (req: express.Request, res: express.Response) { | |||
311 | 313 | ||
312 | await user.destroy() | 314 | await user.destroy() |
313 | 315 | ||
314 | auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) | 316 | auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) |
315 | 317 | ||
316 | return res.sendStatus(204) | 318 | return res.sendStatus(204) |
317 | } | 319 | } |
@@ -328,19 +330,17 @@ async function updateMe (req: express.Request, res: express.Response, next: expr | |||
328 | if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo | 330 | if (body.autoPlayVideo !== undefined) user.autoPlayVideo = body.autoPlayVideo |
329 | 331 | ||
330 | await sequelizeTypescript.transaction(async t => { | 332 | await sequelizeTypescript.transaction(async t => { |
333 | const userAccount = await AccountModel.load(user.Account.id) | ||
334 | |||
331 | await user.save({ transaction: t }) | 335 | await user.save({ transaction: t }) |
332 | 336 | ||
333 | if (body.displayName !== undefined) user.Account.name = body.displayName | 337 | if (body.displayName !== undefined) userAccount.name = body.displayName |
334 | if (body.description !== undefined) user.Account.description = body.description | 338 | if (body.description !== undefined) userAccount.description = body.description |
335 | await user.Account.save({ transaction: t }) | 339 | await userAccount.save({ transaction: t }) |
336 | 340 | ||
337 | await sendUpdateActor(user.Account, t) | 341 | await sendUpdateActor(userAccount, t) |
338 | 342 | ||
339 | auditLogger.update( | 343 | auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) |
340 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
341 | new UserAuditView(user.toFormattedJSON()), | ||
342 | oldUserAuditView | ||
343 | ) | ||
344 | }) | 344 | }) |
345 | 345 | ||
346 | return res.sendStatus(204) | 346 | return res.sendStatus(204) |
@@ -350,15 +350,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next | |||
350 | const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] | 350 | const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] |
351 | const user: UserModel = res.locals.oauth.token.user | 351 | const user: UserModel = res.locals.oauth.token.user |
352 | const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) | 352 | const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) |
353 | const account = user.Account | ||
354 | 353 | ||
355 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) | 354 | const userAccount = await AccountModel.load(user.Account.id) |
356 | 355 | ||
357 | auditLogger.update( | 356 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, userAccount) |
358 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | 357 | |
359 | new UserAuditView(user.toFormattedJSON()), | 358 | auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) |
360 | oldUserAuditView | ||
361 | ) | ||
362 | 359 | ||
363 | return res.json({ avatar: avatar.toFormattedJSON() }) | 360 | return res.json({ avatar: avatar.toFormattedJSON() }) |
364 | } | 361 | } |
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index a7a36080b..ff6bbe44c 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts | |||
@@ -27,8 +27,9 @@ import { logger } from '../../helpers/logger' | |||
27 | import { VideoModel } from '../../models/video/video' | 27 | import { VideoModel } from '../../models/video/video' |
28 | import { updateAvatarValidator } from '../../middlewares/validators/avatar' | 28 | import { updateAvatarValidator } from '../../middlewares/validators/avatar' |
29 | import { updateActorAvatarFile } from '../../lib/avatar' | 29 | import { updateActorAvatarFile } from '../../lib/avatar' |
30 | import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger' | 30 | import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' |
31 | import { resetSequelizeInstance } from '../../helpers/database-utils' | 31 | import { resetSequelizeInstance } from '../../helpers/database-utils' |
32 | import { UserModel } from '../../models/account/user' | ||
32 | 33 | ||
33 | const auditLogger = auditLoggerFactory('channels') | 34 | const auditLogger = auditLoggerFactory('channels') |
34 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) | 35 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) |
@@ -55,7 +56,7 @@ videoChannelRouter.post('/:nameWithHost/avatar/pick', | |||
55 | // Check the rights | 56 | // Check the rights |
56 | asyncMiddleware(videoChannelsUpdateValidator), | 57 | asyncMiddleware(videoChannelsUpdateValidator), |
57 | updateAvatarValidator, | 58 | updateAvatarValidator, |
58 | asyncMiddleware(updateVideoChannelAvatar) | 59 | asyncRetryTransactionMiddleware(updateVideoChannelAvatar) |
59 | ) | 60 | ) |
60 | 61 | ||
61 | videoChannelRouter.put('/:nameWithHost', | 62 | videoChannelRouter.put('/:nameWithHost', |
@@ -106,13 +107,9 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp | |||
106 | const videoChannel = res.locals.videoChannel as VideoChannelModel | 107 | const videoChannel = res.locals.videoChannel as VideoChannelModel |
107 | const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) | 108 | const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) |
108 | 109 | ||
109 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) | 110 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel) |
110 | 111 | ||
111 | auditLogger.update( | 112 | auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) |
112 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
113 | new VideoChannelAuditView(videoChannel.toFormattedJSON()), | ||
114 | oldVideoChannelAuditKeys | ||
115 | ) | ||
116 | 113 | ||
117 | return res | 114 | return res |
118 | .json({ | 115 | .json({ |
@@ -123,19 +120,17 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp | |||
123 | 120 | ||
124 | async function addVideoChannel (req: express.Request, res: express.Response) { | 121 | async function addVideoChannel (req: express.Request, res: express.Response) { |
125 | const videoChannelInfo: VideoChannelCreate = req.body | 122 | const videoChannelInfo: VideoChannelCreate = req.body |
126 | const account: AccountModel = res.locals.oauth.token.User.Account | ||
127 | 123 | ||
128 | const videoChannelCreated: VideoChannelModel = await sequelizeTypescript.transaction(async t => { | 124 | const videoChannelCreated: VideoChannelModel = await sequelizeTypescript.transaction(async t => { |
125 | const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) | ||
126 | |||
129 | return createVideoChannel(videoChannelInfo, account, t) | 127 | return createVideoChannel(videoChannelInfo, account, t) |
130 | }) | 128 | }) |
131 | 129 | ||
132 | setAsyncActorKeys(videoChannelCreated.Actor) | 130 | setAsyncActorKeys(videoChannelCreated.Actor) |
133 | .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) | 131 | .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) |
134 | 132 | ||
135 | auditLogger.create( | 133 | auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())) |
136 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
137 | new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()) | ||
138 | ) | ||
139 | logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) | 134 | logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) |
140 | 135 | ||
141 | return res.json({ | 136 | return res.json({ |
@@ -166,7 +161,7 @@ async function updateVideoChannel (req: express.Request, res: express.Response) | |||
166 | await sendUpdateActor(videoChannelInstanceUpdated, t) | 161 | await sendUpdateActor(videoChannelInstanceUpdated, t) |
167 | 162 | ||
168 | auditLogger.update( | 163 | auditLogger.update( |
169 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | 164 | getAuditIdFromRes(res), |
170 | new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), | 165 | new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), |
171 | oldVideoChannelAuditKeys | 166 | oldVideoChannelAuditKeys |
172 | ) | 167 | ) |
@@ -192,10 +187,7 @@ async function removeVideoChannel (req: express.Request, res: express.Response) | |||
192 | await sequelizeTypescript.transaction(async t => { | 187 | await sequelizeTypescript.transaction(async t => { |
193 | await videoChannelInstance.destroy({ transaction: t }) | 188 | await videoChannelInstance.destroy({ transaction: t }) |
194 | 189 | ||
195 | auditLogger.delete( | 190 | auditLogger.delete(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())) |
196 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
197 | new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) | ||
198 | ) | ||
199 | logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) | 191 | logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) |
200 | }) | 192 | }) |
201 | 193 | ||
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 08e11b00b..d0c81804b 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts | |||
@@ -21,6 +21,7 @@ import { AccountModel } from '../../../models/account/account' | |||
21 | import { VideoModel } from '../../../models/video/video' | 21 | import { VideoModel } from '../../../models/video/video' |
22 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | 22 | import { VideoAbuseModel } from '../../../models/video/video-abuse' |
23 | import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger' | 23 | import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger' |
24 | import { UserModel } from '../../../models/account/user' | ||
24 | 25 | ||
25 | const auditLogger = auditLoggerFactory('abuse') | 26 | const auditLogger = auditLoggerFactory('abuse') |
26 | const abuseVideoRouter = express.Router() | 27 | const abuseVideoRouter = express.Router() |
@@ -95,17 +96,18 @@ async function deleteVideoAbuse (req: express.Request, res: express.Response) { | |||
95 | 96 | ||
96 | async function reportVideoAbuse (req: express.Request, res: express.Response) { | 97 | async function reportVideoAbuse (req: express.Request, res: express.Response) { |
97 | const videoInstance = res.locals.video as VideoModel | 98 | const videoInstance = res.locals.video as VideoModel |
98 | const reporterAccount = res.locals.oauth.token.User.Account as AccountModel | ||
99 | const body: VideoAbuseCreate = req.body | 99 | const body: VideoAbuseCreate = req.body |
100 | 100 | ||
101 | const abuseToCreate = { | ||
102 | reporterAccountId: reporterAccount.id, | ||
103 | reason: body.reason, | ||
104 | videoId: videoInstance.id, | ||
105 | state: VideoAbuseState.PENDING | ||
106 | } | ||
107 | |||
108 | const videoAbuse: VideoAbuseModel = await sequelizeTypescript.transaction(async t => { | 101 | const videoAbuse: VideoAbuseModel = await sequelizeTypescript.transaction(async t => { |
102 | const reporterAccount = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) | ||
103 | |||
104 | const abuseToCreate = { | ||
105 | reporterAccountId: reporterAccount.id, | ||
106 | reason: body.reason, | ||
107 | videoId: videoInstance.id, | ||
108 | state: VideoAbuseState.PENDING | ||
109 | } | ||
110 | |||
109 | const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) | 111 | const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) |
110 | videoAbuseInstance.Video = videoInstance | 112 | videoAbuseInstance.Video = videoInstance |
111 | videoAbuseInstance.Account = reporterAccount | 113 | videoAbuseInstance.Account = reporterAccount |
@@ -121,7 +123,6 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { | |||
121 | }) | 123 | }) |
122 | 124 | ||
123 | logger.info('Abuse report for video %s created.', videoInstance.name) | 125 | logger.info('Abuse report for video %s created.', videoInstance.name) |
124 | return res.json({ | 126 | |
125 | videoAbuse: videoAbuse.toFormattedJSON() | 127 | return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end() |
126 | }).end() | ||
127 | } | 128 | } |
diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts index e35247829..dc25e1e85 100644 --- a/server/controllers/api/videos/comment.ts +++ b/server/controllers/api/videos/comment.ts | |||
@@ -23,7 +23,9 @@ import { | |||
23 | } from '../../../middlewares/validators/video-comments' | 23 | } from '../../../middlewares/validators/video-comments' |
24 | import { VideoModel } from '../../../models/video/video' | 24 | import { VideoModel } from '../../../models/video/video' |
25 | import { VideoCommentModel } from '../../../models/video/video-comment' | 25 | import { VideoCommentModel } from '../../../models/video/video-comment' |
26 | import { auditLoggerFactory, CommentAuditView } from '../../../helpers/audit-logger' | 26 | import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger' |
27 | import { AccountModel } from '../../../models/account/account' | ||
28 | import { UserModel } from '../../../models/account/user' | ||
27 | 29 | ||
28 | const auditLogger = auditLoggerFactory('comments') | 30 | const auditLogger = auditLoggerFactory('comments') |
29 | const videoCommentRouter = express.Router() | 31 | const videoCommentRouter = express.Router() |
@@ -86,7 +88,7 @@ async function listVideoThreadComments (req: express.Request, res: express.Respo | |||
86 | let resultList: ResultList<VideoCommentModel> | 88 | let resultList: ResultList<VideoCommentModel> |
87 | 89 | ||
88 | if (video.commentsEnabled === true) { | 90 | if (video.commentsEnabled === true) { |
89 | resultList = await VideoCommentModel.listThreadCommentsForApi(res.locals.video.id, res.locals.videoCommentThread.id) | 91 | resultList = await VideoCommentModel.listThreadCommentsForApi(video.id, res.locals.videoCommentThread.id) |
90 | } else { | 92 | } else { |
91 | resultList = { | 93 | resultList = { |
92 | total: 0, | 94 | total: 0, |
@@ -101,15 +103,17 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons | |||
101 | const videoCommentInfo: VideoCommentCreate = req.body | 103 | const videoCommentInfo: VideoCommentCreate = req.body |
102 | 104 | ||
103 | const comment = await sequelizeTypescript.transaction(async t => { | 105 | const comment = await sequelizeTypescript.transaction(async t => { |
106 | const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) | ||
107 | |||
104 | return createVideoComment({ | 108 | return createVideoComment({ |
105 | text: videoCommentInfo.text, | 109 | text: videoCommentInfo.text, |
106 | inReplyToComment: null, | 110 | inReplyToComment: null, |
107 | video: res.locals.video, | 111 | video: res.locals.video, |
108 | account: res.locals.oauth.token.User.Account | 112 | account |
109 | }, t) | 113 | }, t) |
110 | }) | 114 | }) |
111 | 115 | ||
112 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) | 116 | auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) |
113 | 117 | ||
114 | return res.json({ | 118 | return res.json({ |
115 | comment: comment.toFormattedJSON() | 119 | comment: comment.toFormattedJSON() |
@@ -120,19 +124,19 @@ async function addVideoCommentReply (req: express.Request, res: express.Response | |||
120 | const videoCommentInfo: VideoCommentCreate = req.body | 124 | const videoCommentInfo: VideoCommentCreate = req.body |
121 | 125 | ||
122 | const comment = await sequelizeTypescript.transaction(async t => { | 126 | const comment = await sequelizeTypescript.transaction(async t => { |
127 | const account = await AccountModel.load((res.locals.oauth.token.User as UserModel).Account.id, t) | ||
128 | |||
123 | return createVideoComment({ | 129 | return createVideoComment({ |
124 | text: videoCommentInfo.text, | 130 | text: videoCommentInfo.text, |
125 | inReplyToComment: res.locals.videoComment, | 131 | inReplyToComment: res.locals.videoComment, |
126 | video: res.locals.video, | 132 | video: res.locals.video, |
127 | account: res.locals.oauth.token.User.Account | 133 | account |
128 | }, t) | 134 | }, t) |
129 | }) | 135 | }) |
130 | 136 | ||
131 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) | 137 | auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) |
132 | 138 | ||
133 | return res.json({ | 139 | return res.json({ comment: comment.toFormattedJSON() }).end() |
134 | comment: comment.toFormattedJSON() | ||
135 | }).end() | ||
136 | } | 140 | } |
137 | 141 | ||
138 | async function removeVideoComment (req: express.Request, res: express.Response) { | 142 | async function removeVideoComment (req: express.Request, res: express.Response) { |
@@ -143,7 +147,7 @@ async function removeVideoComment (req: express.Request, res: express.Response) | |||
143 | }) | 147 | }) |
144 | 148 | ||
145 | auditLogger.delete( | 149 | auditLogger.delete( |
146 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | 150 | getAuditIdFromRes(res), |
147 | new CommentAuditView(videoCommentInstance.toFormattedJSON()) | 151 | new CommentAuditView(videoCommentInstance.toFormattedJSON()) |
148 | ) | 152 | ) |
149 | logger.info('Video comment %d deleted.', videoCommentInstance.id) | 153 | logger.info('Video comment %d deleted.', videoCommentInstance.id) |
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 44f15ef74..398fd5a7f 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import * as magnetUtil from 'magnet-uri' | 2 | import * as magnetUtil from 'magnet-uri' |
3 | import 'multer' | 3 | import 'multer' |
4 | import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' | 4 | import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' |
5 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' | 5 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' |
6 | import { | 6 | import { |
7 | CONFIG, | 7 | CONFIG, |
@@ -114,7 +114,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to | |||
114 | } | 114 | } |
115 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | 115 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) |
116 | 116 | ||
117 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) | 117 | auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) |
118 | 118 | ||
119 | return res.json(videoImport.toFormattedJSON()).end() | 119 | return res.json(videoImport.toFormattedJSON()).end() |
120 | } | 120 | } |
@@ -158,7 +158,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) | |||
158 | } | 158 | } |
159 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | 159 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) |
160 | 160 | ||
161 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) | 161 | auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) |
162 | 162 | ||
163 | return res.json(videoImport.toFormattedJSON()).end() | 163 | return res.json(videoImport.toFormattedJSON()).end() |
164 | } | 164 | } |
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 0c9e6c2d1..581046782 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -4,7 +4,7 @@ import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../ | |||
4 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' | 4 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' |
5 | import { processImage } from '../../../helpers/image-utils' | 5 | import { processImage } from '../../../helpers/image-utils' |
6 | import { logger } from '../../../helpers/logger' | 6 | import { logger } from '../../../helpers/logger' |
7 | import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger' | 7 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' |
8 | import { getFormattedObjects, getServerActor } from '../../../helpers/utils' | 8 | import { getFormattedObjects, getServerActor } from '../../../helpers/utils' |
9 | import { | 9 | import { |
10 | CONFIG, | 10 | CONFIG, |
@@ -253,7 +253,7 @@ async function addVideo (req: express.Request, res: express.Response) { | |||
253 | 253 | ||
254 | await federateVideoIfNeeded(video, true, t) | 254 | await federateVideoIfNeeded(video, true, t) |
255 | 255 | ||
256 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) | 256 | auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) |
257 | logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) | 257 | logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) |
258 | 258 | ||
259 | return videoCreated | 259 | return videoCreated |
@@ -354,7 +354,7 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
354 | await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) | 354 | await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) |
355 | 355 | ||
356 | auditLogger.update( | 356 | auditLogger.update( |
357 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | 357 | getAuditIdFromRes(res), |
358 | new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), | 358 | new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), |
359 | oldVideoAuditView | 359 | oldVideoAuditView |
360 | ) | 360 | ) |
@@ -393,9 +393,9 @@ async function viewVideo (req: express.Request, res: express.Response) { | |||
393 | Redis.Instance.setIPVideoView(ip, videoInstance.uuid) | 393 | Redis.Instance.setIPVideoView(ip, videoInstance.uuid) |
394 | ]) | 394 | ]) |
395 | 395 | ||
396 | const serverAccount = await getServerActor() | 396 | const serverActor = await getServerActor() |
397 | 397 | ||
398 | await sendCreateView(serverAccount, videoInstance, undefined) | 398 | await sendCreateView(serverActor, videoInstance, undefined) |
399 | 399 | ||
400 | return res.status(204).end() | 400 | return res.status(204).end() |
401 | } | 401 | } |
@@ -439,7 +439,7 @@ async function removeVideo (req: express.Request, res: express.Response) { | |||
439 | await videoInstance.destroy({ transaction: t }) | 439 | await videoInstance.destroy({ transaction: t }) |
440 | }) | 440 | }) |
441 | 441 | ||
442 | auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) | 442 | auditLogger.delete(getAuditIdFromRes(res), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) |
443 | logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) | 443 | logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) |
444 | 444 | ||
445 | return res.type('json').status(204).end() | 445 | return res.type('json').status(204).end() |
diff --git a/server/controllers/api/videos/ownership.ts b/server/controllers/api/videos/ownership.ts index d26ed6cfc..5ea7d7c6a 100644 --- a/server/controllers/api/videos/ownership.ts +++ b/server/controllers/api/videos/ownership.ts | |||
@@ -19,6 +19,7 @@ import { VideoChannelModel } from '../../../models/video/video-channel' | |||
19 | import { getFormattedObjects } from '../../../helpers/utils' | 19 | import { getFormattedObjects } from '../../../helpers/utils' |
20 | import { changeVideoChannelShare } from '../../../lib/activitypub' | 20 | import { changeVideoChannelShare } from '../../../lib/activitypub' |
21 | import { sendUpdateVideo } from '../../../lib/activitypub/send' | 21 | import { sendUpdateVideo } from '../../../lib/activitypub/send' |
22 | import { UserModel } from '../../../models/account/user' | ||
22 | 23 | ||
23 | const ownershipVideoRouter = express.Router() | 24 | const ownershipVideoRouter = express.Router() |
24 | 25 | ||
@@ -58,26 +59,25 @@ export { | |||
58 | 59 | ||
59 | async function giveVideoOwnership (req: express.Request, res: express.Response) { | 60 | async function giveVideoOwnership (req: express.Request, res: express.Response) { |
60 | const videoInstance = res.locals.video as VideoModel | 61 | const videoInstance = res.locals.video as VideoModel |
61 | const initiatorAccount = res.locals.oauth.token.User.Account as AccountModel | 62 | const initiatorAccountId = (res.locals.oauth.token.User as UserModel).Account.id |
62 | const nextOwner = res.locals.nextOwner as AccountModel | 63 | const nextOwner = res.locals.nextOwner as AccountModel |
63 | 64 | ||
64 | await sequelizeTypescript.transaction(t => { | 65 | await sequelizeTypescript.transaction(t => { |
65 | return VideoChangeOwnershipModel.findOrCreate({ | 66 | return VideoChangeOwnershipModel.findOrCreate({ |
66 | where: { | 67 | where: { |
67 | initiatorAccountId: initiatorAccount.id, | 68 | initiatorAccountId, |
68 | nextOwnerAccountId: nextOwner.id, | 69 | nextOwnerAccountId: nextOwner.id, |
69 | videoId: videoInstance.id, | 70 | videoId: videoInstance.id, |
70 | status: VideoChangeOwnershipStatus.WAITING | 71 | status: VideoChangeOwnershipStatus.WAITING |
71 | }, | 72 | }, |
72 | defaults: { | 73 | defaults: { |
73 | initiatorAccountId: initiatorAccount.id, | 74 | initiatorAccountId, |
74 | nextOwnerAccountId: nextOwner.id, | 75 | nextOwnerAccountId: nextOwner.id, |
75 | videoId: videoInstance.id, | 76 | videoId: videoInstance.id, |
76 | status: VideoChangeOwnershipStatus.WAITING | 77 | status: VideoChangeOwnershipStatus.WAITING |
77 | }, | 78 | }, |
78 | transaction: t | 79 | transaction: t |
79 | }) | 80 | }) |
80 | |||
81 | }) | 81 | }) |
82 | 82 | ||
83 | logger.info('Ownership change for video %s created.', videoInstance.name) | 83 | logger.info('Ownership change for video %s created.', videoInstance.name) |
@@ -85,9 +85,10 @@ async function giveVideoOwnership (req: express.Request, res: express.Response) | |||
85 | } | 85 | } |
86 | 86 | ||
87 | async function listVideoOwnership (req: express.Request, res: express.Response) { | 87 | async function listVideoOwnership (req: express.Request, res: express.Response) { |
88 | const currentAccount = res.locals.oauth.token.User.Account as AccountModel | 88 | const currentAccountId = (res.locals.oauth.token.User as UserModel).Account.id |
89 | |||
89 | const resultList = await VideoChangeOwnershipModel.listForApi( | 90 | const resultList = await VideoChangeOwnershipModel.listForApi( |
90 | currentAccount.id, | 91 | currentAccountId, |
91 | req.query.start || 0, | 92 | req.query.start || 0, |
92 | req.query.count || 10, | 93 | req.query.count || 10, |
93 | req.query.sort || 'createdAt' | 94 | req.query.sort || 'createdAt' |
diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts index b1732837d..dc322bb0c 100644 --- a/server/controllers/api/videos/rate.ts +++ b/server/controllers/api/videos/rate.ts | |||
@@ -28,10 +28,11 @@ async function rateVideo (req: express.Request, res: express.Response) { | |||
28 | const body: UserVideoRateUpdate = req.body | 28 | const body: UserVideoRateUpdate = req.body |
29 | const rateType = body.rating | 29 | const rateType = body.rating |
30 | const videoInstance: VideoModel = res.locals.video | 30 | const videoInstance: VideoModel = res.locals.video |
31 | const accountInstance: AccountModel = res.locals.oauth.token.User.Account | ||
32 | 31 | ||
33 | await sequelizeTypescript.transaction(async t => { | 32 | await sequelizeTypescript.transaction(async t => { |
34 | const sequelizeOptions = { transaction: t } | 33 | const sequelizeOptions = { transaction: t } |
34 | |||
35 | const accountInstance = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) | ||
35 | const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) | 36 | const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) |
36 | 37 | ||
37 | let likesToIncrement = 0 | 38 | let likesToIncrement = 0 |
@@ -47,10 +48,10 @@ async function rateVideo (req: express.Request, res: express.Response) { | |||
47 | else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement-- | 48 | else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement-- |
48 | 49 | ||
49 | if (rateType === 'none') { // Destroy previous rate | 50 | if (rateType === 'none') { // Destroy previous rate |
50 | await previousRate.destroy({ transaction: t }) | 51 | await previousRate.destroy(sequelizeOptions) |
51 | } else { // Update previous rate | 52 | } else { // Update previous rate |
52 | previousRate.type = rateType | 53 | previousRate.type = rateType |
53 | await previousRate.save({ transaction: t }) | 54 | await previousRate.save(sequelizeOptions) |
54 | } | 55 | } |
55 | } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate | 56 | } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate |
56 | const query = { | 57 | const query = { |
@@ -70,9 +71,9 @@ async function rateVideo (req: express.Request, res: express.Response) { | |||
70 | await videoInstance.increment(incrementQuery, sequelizeOptions) | 71 | await videoInstance.increment(incrementQuery, sequelizeOptions) |
71 | 72 | ||
72 | await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) | 73 | await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) |
73 | }) | ||
74 | 74 | ||
75 | logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) | 75 | logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) |
76 | }) | ||
76 | 77 | ||
77 | return res.type('json').status(204).end() | 78 | return res.type('json').status(204).end() |
78 | } | 79 | } |
diff --git a/server/controllers/client.ts b/server/controllers/client.ts index c33061289..73b40cf65 100644 --- a/server/controllers/client.ts +++ b/server/controllers/client.ts | |||
@@ -35,7 +35,7 @@ clientsRouter.use('' + | |||
35 | // Static HTML/CSS/JS client files | 35 | // Static HTML/CSS/JS client files |
36 | 36 | ||
37 | const staticClientFiles = [ | 37 | const staticClientFiles = [ |
38 | 'manifest.json', | 38 | 'manifest.webmanifest', |
39 | 'ngsw-worker.js', | 39 | 'ngsw-worker.js', |
40 | 'ngsw.json' | 40 | 'ngsw.json' |
41 | ] | 41 | ] |
diff --git a/server/helpers/actor.ts b/server/helpers/actor.ts new file mode 100644 index 000000000..12a7ace9f --- /dev/null +++ b/server/helpers/actor.ts | |||
@@ -0,0 +1,13 @@ | |||
1 | import { ActorModel } from '../models/activitypub/actor' | ||
2 | |||
3 | type ActorFetchByUrlType = 'all' | 'actor-and-association-ids' | ||
4 | function fetchActorByUrl (url: string, fetchType: ActorFetchByUrlType) { | ||
5 | if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) | ||
6 | |||
7 | if (fetchType === 'actor-and-association-ids') return ActorModel.loadByUrl(url) | ||
8 | } | ||
9 | |||
10 | export { | ||
11 | ActorFetchByUrlType, | ||
12 | fetchActorByUrl | ||
13 | } | ||
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index 7db72b69c..00311fce1 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | import * as path from 'path' | 1 | import * as path from 'path' |
2 | import * as express from 'express' | ||
2 | import { diff } from 'deep-object-diff' | 3 | import { diff } from 'deep-object-diff' |
3 | import { chain } from 'lodash' | 4 | import { chain } from 'lodash' |
4 | import * as flatten from 'flat' | 5 | import * as flatten from 'flat' |
@@ -8,6 +9,11 @@ import { jsonLoggerFormat, labelFormatter } from './logger' | |||
8 | import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared' | 9 | import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared' |
9 | import { VideoComment } from '../../shared/models/videos/video-comment.model' | 10 | import { VideoComment } from '../../shared/models/videos/video-comment.model' |
10 | import { CustomConfig } from '../../shared/models/server/custom-config.model' | 11 | import { CustomConfig } from '../../shared/models/server/custom-config.model' |
12 | import { UserModel } from '../models/account/user' | ||
13 | |||
14 | function getAuditIdFromRes (res: express.Response) { | ||
15 | return (res.locals.oauth.token.User as UserModel).username | ||
16 | } | ||
11 | 17 | ||
12 | enum AUDIT_TYPE { | 18 | enum AUDIT_TYPE { |
13 | CREATE = 'create', | 19 | CREATE = 'create', |
@@ -255,6 +261,8 @@ class CustomConfigAuditView extends EntityAuditView { | |||
255 | } | 261 | } |
256 | 262 | ||
257 | export { | 263 | export { |
264 | getAuditIdFromRes, | ||
265 | |||
258 | auditLoggerFactory, | 266 | auditLoggerFactory, |
259 | VideoImportAuditView, | 267 | VideoImportAuditView, |
260 | VideoChannelAuditView, | 268 | VideoChannelAuditView, |
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index f76eba474..8772e74cf 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -171,5 +171,3 @@ function setRemoteVideoTruncatedContent (video: any) { | |||
171 | 171 | ||
172 | return true | 172 | return true |
173 | } | 173 | } |
174 | |||
175 | |||
diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts index aaa0c736b..a7771e07b 100644 --- a/server/helpers/custom-validators/video-ownership.ts +++ b/server/helpers/custom-validators/video-ownership.ts | |||
@@ -31,7 +31,7 @@ export function checkUserCanTerminateOwnershipChange ( | |||
31 | videoChangeOwnership: VideoChangeOwnershipModel, | 31 | videoChangeOwnership: VideoChangeOwnershipModel, |
32 | res: Response | 32 | res: Response |
33 | ): boolean { | 33 | ): boolean { |
34 | if (videoChangeOwnership.NextOwner.userId === user.Account.userId) { | 34 | if (videoChangeOwnership.NextOwner.userId === user.id) { |
35 | return true | 35 | return true |
36 | } | 36 | } |
37 | 37 | ||
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index edafba6e2..9875c68bd 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts | |||
@@ -18,6 +18,7 @@ import { exists, isArray, isFileValid } from './misc' | |||
18 | import { VideoChannelModel } from '../../models/video/video-channel' | 18 | import { VideoChannelModel } from '../../models/video/video-channel' |
19 | import { UserModel } from '../../models/account/user' | 19 | import { UserModel } from '../../models/account/user' |
20 | import * as magnetUtil from 'magnet-uri' | 20 | import * as magnetUtil from 'magnet-uri' |
21 | import { fetchVideo, VideoFetchType } from '../video' | ||
21 | 22 | ||
22 | const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS | 23 | const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS |
23 | 24 | ||
@@ -152,14 +153,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use | |||
152 | return true | 153 | return true |
153 | } | 154 | } |
154 | 155 | ||
155 | async function isVideoExist (id: string, res: Response) { | 156 | async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') { |
156 | let video: VideoModel | null | 157 | const video = await fetchVideo(id, fetchType) |
157 | |||
158 | if (validator.isInt(id)) { | ||
159 | video = await VideoModel.loadAndPopulateAccountAndServerAndTags(+id) | ||
160 | } else { // UUID | ||
161 | video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(id) | ||
162 | } | ||
163 | 158 | ||
164 | if (video === null) { | 159 | if (video === null) { |
165 | res.status(404) | 160 | res.status(404) |
@@ -169,7 +164,7 @@ async function isVideoExist (id: string, res: Response) { | |||
169 | return false | 164 | return false |
170 | } | 165 | } |
171 | 166 | ||
172 | res.locals.video = video | 167 | if (fetchType !== 'none') res.locals.video = video |
173 | return true | 168 | return true |
174 | } | 169 | } |
175 | 170 | ||
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index a1ed8e72d..a42474417 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | import { ResultList } from '../../shared' | 1 | import { ResultList } from '../../shared' |
2 | import { CONFIG } from '../initializers' | 2 | import { CONFIG } from '../initializers' |
3 | import { ActorModel } from '../models/activitypub/actor' | ||
4 | import { ApplicationModel } from '../models/application/application' | 3 | import { ApplicationModel } from '../models/application/application' |
5 | import { pseudoRandomBytesPromise, sha256 } from './core-utils' | 4 | import { pseudoRandomBytesPromise, sha256 } from './core-utils' |
6 | import { logger } from './logger' | 5 | import { logger } from './logger' |
7 | import { join } from 'path' | 6 | import { join } from 'path' |
8 | import { Instance as ParseTorrent } from 'parse-torrent' | 7 | import { Instance as ParseTorrent } from 'parse-torrent' |
9 | import { remove } from 'fs-extra' | 8 | import { remove } from 'fs-extra' |
9 | import * as memoizee from 'memoizee' | ||
10 | 10 | ||
11 | function deleteFileAsync (path: string) { | 11 | function deleteFileAsync (path: string) { |
12 | remove(path) | 12 | remove(path) |
@@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje | |||
36 | } as ResultList<U> | 36 | } as ResultList<U> |
37 | } | 37 | } |
38 | 38 | ||
39 | async function getServerActor () { | 39 | const getServerActor = memoizee(async function () { |
40 | if (getServerActor.serverActor === undefined) { | 40 | const application = await ApplicationModel.load() |
41 | const application = await ApplicationModel.load() | 41 | if (!application) throw Error('Could not load Application from database.') |
42 | if (!application) throw Error('Could not load Application from database.') | ||
43 | 42 | ||
44 | getServerActor.serverActor = application.Account.Actor | 43 | return application.Account.Actor |
45 | } | 44 | }) |
46 | |||
47 | if (!getServerActor.serverActor) { | ||
48 | logger.error('Cannot load server actor.') | ||
49 | process.exit(0) | ||
50 | } | ||
51 | |||
52 | return Promise.resolve(getServerActor.serverActor) | ||
53 | } | ||
54 | namespace getServerActor { | ||
55 | export let serverActor: ActorModel | ||
56 | } | ||
57 | 45 | ||
58 | function generateVideoTmpPath (target: string | ParseTorrent) { | 46 | function generateVideoTmpPath (target: string | ParseTorrent) { |
59 | const id = typeof target === 'string' ? target : target.infoHash | 47 | const id = typeof target === 'string' ? target : target.infoHash |
diff --git a/server/helpers/video.ts b/server/helpers/video.ts new file mode 100644 index 000000000..b1577a6b0 --- /dev/null +++ b/server/helpers/video.ts | |||
@@ -0,0 +1,25 @@ | |||
1 | import { VideoModel } from '../models/video/video' | ||
2 | |||
3 | type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' | ||
4 | |||
5 | function fetchVideo (id: number | string, fetchType: VideoFetchType) { | ||
6 | if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id) | ||
7 | |||
8 | if (fetchType === 'only-video') return VideoModel.load(id) | ||
9 | |||
10 | if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) | ||
11 | } | ||
12 | |||
13 | type VideoFetchByUrlType = 'all' | 'only-video' | ||
14 | function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType) { | ||
15 | if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) | ||
16 | |||
17 | if (fetchType === 'only-video') return VideoModel.loadByUrl(url) | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | VideoFetchType, | ||
22 | VideoFetchByUrlType, | ||
23 | fetchVideo, | ||
24 | fetchVideoByUrl | ||
25 | } | ||
diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts index 10fcec462..156376943 100644 --- a/server/helpers/webfinger.ts +++ b/server/helpers/webfinger.ts | |||
@@ -12,7 +12,10 @@ const webfinger = new WebFinger({ | |||
12 | request_timeout: 3000 | 12 | request_timeout: 3000 |
13 | }) | 13 | }) |
14 | 14 | ||
15 | async function loadActorUrlOrGetFromWebfinger (uri: string) { | 15 | async function loadActorUrlOrGetFromWebfinger (uriArg: string) { |
16 | // Handle strings like @toto@example.com | ||
17 | const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg | ||
18 | |||
16 | const [ name, host ] = uri.split('@') | 19 | const [ name, host ] = uri.split('@') |
17 | let actor: ActorModel | 20 | let actor: ActorModel |
18 | 21 | ||
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 2fdfd1876..f4b44bc4f 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -24,7 +24,7 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: str | |||
24 | if (timer) clearTimeout(timer) | 24 | if (timer) clearTimeout(timer) |
25 | 25 | ||
26 | return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) | 26 | return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) |
27 | .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId))) | 27 | .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) |
28 | } | 28 | } |
29 | 29 | ||
30 | file = torrent.files[ 0 ] | 30 | file = torrent.files[ 0 ] |
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 8b2bc1782..25e719cc3 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts | |||
@@ -2,7 +2,11 @@ import { truncate } from 'lodash' | |||
2 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' | 2 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' |
3 | import { logger } from './logger' | 3 | import { logger } from './logger' |
4 | import { generateVideoTmpPath } from './utils' | 4 | import { generateVideoTmpPath } from './utils' |
5 | import { YoutubeDlUpdateScheduler } from '../lib/schedulers/youtube-dl-update-scheduler' | 5 | import { join } from 'path' |
6 | import { root } from './core-utils' | ||
7 | import { ensureDir, writeFile } from 'fs-extra' | ||
8 | import * as request from 'request' | ||
9 | import { createWriteStream } from 'fs' | ||
6 | 10 | ||
7 | export type YoutubeDLInfo = { | 11 | export type YoutubeDLInfo = { |
8 | name?: string | 12 | name?: string |
@@ -40,7 +44,7 @@ function downloadYoutubeDLVideo (url: string) { | |||
40 | 44 | ||
41 | return new Promise<string>(async (res, rej) => { | 45 | return new Promise<string>(async (res, rej) => { |
42 | const youtubeDL = await safeGetYoutubeDL() | 46 | const youtubeDL = await safeGetYoutubeDL() |
43 | youtubeDL.exec(url, options, async (err, output) => { | 47 | youtubeDL.exec(url, options, err => { |
44 | if (err) return rej(err) | 48 | if (err) return rej(err) |
45 | 49 | ||
46 | return res(path) | 50 | return res(path) |
@@ -48,6 +52,64 @@ function downloadYoutubeDLVideo (url: string) { | |||
48 | }) | 52 | }) |
49 | } | 53 | } |
50 | 54 | ||
55 | // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js | ||
56 | // We rewrote it to avoid sync calls | ||
57 | async function updateYoutubeDLBinary () { | ||
58 | logger.info('Updating youtubeDL binary.') | ||
59 | |||
60 | const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') | ||
61 | const bin = join(binDirectory, 'youtube-dl') | ||
62 | const detailsPath = join(binDirectory, 'details') | ||
63 | const url = 'https://yt-dl.org/downloads/latest/youtube-dl' | ||
64 | |||
65 | await ensureDir(binDirectory) | ||
66 | |||
67 | return new Promise(res => { | ||
68 | request.get(url, { followRedirect: false }, (err, result) => { | ||
69 | if (err) { | ||
70 | logger.error('Cannot update youtube-dl.', { err }) | ||
71 | return res() | ||
72 | } | ||
73 | |||
74 | if (result.statusCode !== 302) { | ||
75 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) | ||
76 | return res() | ||
77 | } | ||
78 | |||
79 | const url = result.headers.location | ||
80 | const downloadFile = request.get(url) | ||
81 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] | ||
82 | |||
83 | downloadFile.on('response', result => { | ||
84 | if (result.statusCode !== 200) { | ||
85 | logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) | ||
86 | return res() | ||
87 | } | ||
88 | |||
89 | downloadFile.pipe(createWriteStream(bin, { mode: 493 })) | ||
90 | }) | ||
91 | |||
92 | downloadFile.on('error', err => { | ||
93 | logger.error('youtube-dl update error.', { err }) | ||
94 | return res() | ||
95 | }) | ||
96 | |||
97 | downloadFile.on('end', () => { | ||
98 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) | ||
99 | writeFile(detailsPath, details, { encoding: 'utf8' }, err => { | ||
100 | if (err) { | ||
101 | logger.error('youtube-dl update error: cannot write details.', { err }) | ||
102 | return res() | ||
103 | } | ||
104 | |||
105 | logger.info('youtube-dl updated to version %s.', newVersion) | ||
106 | return res() | ||
107 | }) | ||
108 | }) | ||
109 | }) | ||
110 | }) | ||
111 | } | ||
112 | |||
51 | async function safeGetYoutubeDL () { | 113 | async function safeGetYoutubeDL () { |
52 | let youtubeDL | 114 | let youtubeDL |
53 | 115 | ||
@@ -55,7 +117,7 @@ async function safeGetYoutubeDL () { | |||
55 | youtubeDL = require('youtube-dl') | 117 | youtubeDL = require('youtube-dl') |
56 | } catch (e) { | 118 | } catch (e) { |
57 | // Download binary | 119 | // Download binary |
58 | await YoutubeDlUpdateScheduler.Instance.execute() | 120 | await updateYoutubeDLBinary() |
59 | youtubeDL = require('youtube-dl') | 121 | youtubeDL = require('youtube-dl') |
60 | } | 122 | } |
61 | 123 | ||
@@ -65,6 +127,7 @@ async function safeGetYoutubeDL () { | |||
65 | // --------------------------------------------------------------------------- | 127 | // --------------------------------------------------------------------------- |
66 | 128 | ||
67 | export { | 129 | export { |
130 | updateYoutubeDLBinary, | ||
68 | downloadYoutubeDLVideo, | 131 | downloadYoutubeDLVideo, |
69 | getYoutubeDLInfo, | 132 | getYoutubeDLInfo, |
70 | safeGetYoutubeDL | 133 | safeGetYoutubeDL |
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 6a2badd35..a54f6155b 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts | |||
@@ -7,7 +7,7 @@ import { parse } from 'url' | |||
7 | import { CONFIG } from './constants' | 7 | import { CONFIG } from './constants' |
8 | import { logger } from '../helpers/logger' | 8 | import { logger } from '../helpers/logger' |
9 | import { getServerActor } from '../helpers/utils' | 9 | import { getServerActor } from '../helpers/utils' |
10 | import { VideosRedundancy } from '../../shared/models/redundancy' | 10 | import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy' |
11 | import { isArray } from '../helpers/custom-validators/misc' | 11 | import { isArray } from '../helpers/custom-validators/misc' |
12 | import { uniq } from 'lodash' | 12 | import { uniq } from 'lodash' |
13 | 13 | ||
@@ -34,21 +34,28 @@ async function checkActivityPubUrls () { | |||
34 | function checkConfig () { | 34 | function checkConfig () { |
35 | const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy') | 35 | const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy') |
36 | 36 | ||
37 | // NSFW policy | ||
37 | if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { | 38 | if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) { |
38 | return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy | 39 | return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy |
39 | } | 40 | } |
40 | 41 | ||
41 | const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos') | 42 | // Redundancies |
43 | const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies') | ||
42 | if (isArray(redundancyVideos)) { | 44 | if (isArray(redundancyVideos)) { |
43 | for (const r of redundancyVideos) { | 45 | for (const r of redundancyVideos) { |
44 | if ([ 'most-views' ].indexOf(r.strategy) === -1) { | 46 | if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) { |
45 | return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy | 47 | return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy |
46 | } | 48 | } |
47 | } | 49 | } |
48 | 50 | ||
49 | const filtered = uniq(redundancyVideos.map(r => r.strategy)) | 51 | const filtered = uniq(redundancyVideos.map(r => r.strategy)) |
50 | if (filtered.length !== redundancyVideos.length) { | 52 | if (filtered.length !== redundancyVideos.length) { |
51 | return 'Redundancy video entries should have uniq strategies' | 53 | return 'Redundancy video entries should have unique strategies' |
54 | } | ||
55 | |||
56 | const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy | ||
57 | if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { | ||
58 | return 'Min views in recently added strategy is not a number' | ||
52 | } | 59 | } |
53 | } | 60 | } |
54 | 61 | ||
@@ -68,6 +75,7 @@ function checkMissedConfig () { | |||
68 | 'cache.previews.size', 'admin.email', | 75 | 'cache.previews.size', 'admin.email', |
69 | 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', | 76 | 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', |
70 | 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', | 77 | 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', |
78 | 'redundancy.videos.strategies', 'redundancy.videos.check_interval', | ||
71 | 'transcoding.enabled', 'transcoding.threads', | 79 | 'transcoding.enabled', 'transcoding.threads', |
72 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', | 80 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', |
73 | 'trending.videos.interval_days', | 81 | 'trending.videos.interval_days', |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6b4afbfd8..03424ffb8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -1,11 +1,11 @@ | |||
1 | import { IConfig } from 'config' | 1 | import { IConfig } from 'config' |
2 | import { dirname, join } from 'path' | 2 | import { dirname, join } from 'path' |
3 | import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' | 3 | import { JobType, VideoRateType, VideoState, VideosRedundancy } from '../../shared/models' |
4 | import { ActivityPubActorType } from '../../shared/models/activitypub' | 4 | import { ActivityPubActorType } from '../../shared/models/activitypub' |
5 | import { FollowState } from '../../shared/models/actors' | 5 | import { FollowState } from '../../shared/models/actors' |
6 | import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' | 6 | import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' |
7 | // Do not use barrels, remain constants as independent as possible | 7 | // Do not use barrels, remain constants as independent as possible |
8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' | 8 | import { buildPath, isTestInstance, parseDuration, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' |
9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' | 9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' |
10 | import { invert } from 'lodash' | 10 | import { invert } from 'lodash' |
11 | import { CronRepeatOptions, EveryRepeatOptions } from 'bull' | 11 | import { CronRepeatOptions, EveryRepeatOptions } from 'bull' |
@@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = { | |||
66 | }, | 66 | }, |
67 | ACTIVITY_PUB: { | 67 | ACTIVITY_PUB: { |
68 | VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example | 68 | VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example |
69 | } | 69 | }, |
70 | STATS: '4 hours' | ||
70 | } | 71 | } |
71 | 72 | ||
72 | // --------------------------------------------------------------------------- | 73 | // --------------------------------------------------------------------------- |
@@ -138,8 +139,7 @@ let SCHEDULER_INTERVALS_MS = { | |||
138 | badActorFollow: 60000 * 60, // 1 hour | 139 | badActorFollow: 60000 * 60, // 1 hour |
139 | removeOldJobs: 60000 * 60, // 1 hour | 140 | removeOldJobs: 60000 * 60, // 1 hour |
140 | updateVideos: 60000, // 1 minute | 141 | updateVideos: 60000, // 1 minute |
141 | youtubeDLUpdate: 60000 * 60 * 24, // 1 day | 142 | youtubeDLUpdate: 60000 * 60 * 24 // 1 day |
142 | videosRedundancy: 60000 * 2 // 2 hours | ||
143 | } | 143 | } |
144 | 144 | ||
145 | // --------------------------------------------------------------------------- | 145 | // --------------------------------------------------------------------------- |
@@ -211,7 +211,10 @@ const CONFIG = { | |||
211 | } | 211 | } |
212 | }, | 212 | }, |
213 | REDUNDANCY: { | 213 | REDUNDANCY: { |
214 | VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos')) | 214 | VIDEOS: { |
215 | CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')), | ||
216 | STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies')) | ||
217 | } | ||
215 | }, | 218 | }, |
216 | ADMIN: { | 219 | ADMIN: { |
217 | get EMAIL () { return config.get<string>('admin.email') } | 220 | get EMAIL () { return config.get<string>('admin.email') } |
@@ -592,6 +595,10 @@ const CACHE = { | |||
592 | } | 595 | } |
593 | } | 596 | } |
594 | 597 | ||
598 | const MEMOIZE_TTL = { | ||
599 | OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours | ||
600 | } | ||
601 | |||
595 | const REDUNDANCY = { | 602 | const REDUNDANCY = { |
596 | VIDEOS: { | 603 | VIDEOS: { |
597 | EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days | 604 | EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days |
@@ -644,7 +651,6 @@ if (isTestInstance() === true) { | |||
644 | SCHEDULER_INTERVALS_MS.badActorFollow = 10000 | 651 | SCHEDULER_INTERVALS_MS.badActorFollow = 10000 |
645 | SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 | 652 | SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 |
646 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 | 653 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 |
647 | SCHEDULER_INTERVALS_MS.videosRedundancy = 5000 | ||
648 | REPEAT_JOBS['videos-views'] = { every: 5000 } | 654 | REPEAT_JOBS['videos-views'] = { every: 5000 } |
649 | 655 | ||
650 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 | 656 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 |
@@ -654,6 +660,8 @@ if (isTestInstance() === true) { | |||
654 | JOB_ATTEMPTS['email'] = 1 | 660 | JOB_ATTEMPTS['email'] = 1 |
655 | 661 | ||
656 | CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 | 662 | CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 |
663 | MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 | ||
664 | ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' | ||
657 | } | 665 | } |
658 | 666 | ||
659 | updateWebserverConfig() | 667 | updateWebserverConfig() |
@@ -708,6 +716,7 @@ export { | |||
708 | VIDEO_ABUSE_STATES, | 716 | VIDEO_ABUSE_STATES, |
709 | JOB_REQUEST_TIMEOUT, | 717 | JOB_REQUEST_TIMEOUT, |
710 | USER_PASSWORD_RESET_LIFETIME, | 718 | USER_PASSWORD_RESET_LIFETIME, |
719 | MEMOIZE_TTL, | ||
711 | USER_EMAIL_VERIFY_LIFETIME, | 720 | USER_EMAIL_VERIFY_LIFETIME, |
712 | IMAGE_MIMETYPE_EXT, | 721 | IMAGE_MIMETYPE_EXT, |
713 | OVERVIEWS, | 722 | OVERVIEWS, |
@@ -741,15 +750,10 @@ function updateWebserverConfig () { | |||
741 | CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) | 750 | CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) |
742 | } | 751 | } |
743 | 752 | ||
744 | function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { | 753 | function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] { |
745 | if (!objs) return [] | 754 | if (!objs) return [] |
746 | 755 | ||
747 | return objs.map(obj => { | 756 | return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) })) |
748 | return { | ||
749 | strategy: obj.strategy, | ||
750 | size: bytes.parse(obj.size) | ||
751 | } | ||
752 | }) | ||
753 | } | 757 | } |
754 | 758 | ||
755 | function buildLanguages () { | 759 | function buildLanguages () { |
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 3464add03..d37a695a7 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -21,6 +21,7 @@ import { ServerModel } from '../../models/server/server' | |||
21 | import { VideoChannelModel } from '../../models/video/video-channel' | 21 | import { VideoChannelModel } from '../../models/video/video-channel' |
22 | import { JobQueue } from '../job-queue' | 22 | import { JobQueue } from '../job-queue' |
23 | import { getServerActor } from '../../helpers/utils' | 23 | import { getServerActor } from '../../helpers/utils' |
24 | import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' | ||
24 | 25 | ||
25 | // Set account keys, this could be long so process after the account creation and do not block the client | 26 | // Set account keys, this could be long so process after the account creation and do not block the client |
26 | function setAsyncActorKeys (actor: ActorModel) { | 27 | function setAsyncActorKeys (actor: ActorModel) { |
@@ -38,13 +39,14 @@ function setAsyncActorKeys (actor: ActorModel) { | |||
38 | 39 | ||
39 | async function getOrCreateActorAndServerAndModel ( | 40 | async function getOrCreateActorAndServerAndModel ( |
40 | activityActor: string | ActivityPubActor, | 41 | activityActor: string | ActivityPubActor, |
42 | fetchType: ActorFetchByUrlType = 'actor-and-association-ids', | ||
41 | recurseIfNeeded = true, | 43 | recurseIfNeeded = true, |
42 | updateCollections = false | 44 | updateCollections = false |
43 | ) { | 45 | ) { |
44 | const actorUrl = getActorUrl(activityActor) | 46 | const actorUrl = getActorUrl(activityActor) |
45 | let created = false | 47 | let created = false |
46 | 48 | ||
47 | let actor = await ActorModel.loadByUrl(actorUrl) | 49 | let actor = await fetchActorByUrl(actorUrl, fetchType) |
48 | // Orphan actor (not associated to an account of channel) so recreate it | 50 | // Orphan actor (not associated to an account of channel) so recreate it |
49 | if (actor && (!actor.Account && !actor.VideoChannel)) { | 51 | if (actor && (!actor.Account && !actor.VideoChannel)) { |
50 | await actor.destroy() | 52 | await actor.destroy() |
@@ -65,7 +67,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
65 | 67 | ||
66 | try { | 68 | try { |
67 | // Assert we don't recurse another time | 69 | // Assert we don't recurse another time |
68 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, false) | 70 | ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false) |
69 | } catch (err) { | 71 | } catch (err) { |
70 | logger.error('Cannot get or create account attributed to video channel ' + actor.url) | 72 | logger.error('Cannot get or create account attributed to video channel ' + actor.url) |
71 | throw new Error(err) | 73 | throw new Error(err) |
@@ -79,7 +81,7 @@ async function getOrCreateActorAndServerAndModel ( | |||
79 | if (actor.Account) actor.Account.Actor = actor | 81 | if (actor.Account) actor.Account.Actor = actor |
80 | if (actor.VideoChannel) actor.VideoChannel.Actor = actor | 82 | if (actor.VideoChannel) actor.VideoChannel.Actor = actor |
81 | 83 | ||
82 | const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor) | 84 | const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) |
83 | if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') | 85 | if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') |
84 | 86 | ||
85 | if ((created === true || refreshed === true) && updateCollections === true) { | 87 | if ((created === true || refreshed === true) && updateCollections === true) { |
@@ -370,8 +372,14 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu | |||
370 | return videoChannelCreated | 372 | return videoChannelCreated |
371 | } | 373 | } |
372 | 374 | ||
373 | async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorModel, refreshed: boolean }> { | 375 | async function refreshActorIfNeeded ( |
374 | if (!actor.isOutdated()) return { actor, refreshed: false } | 376 | actorArg: ActorModel, |
377 | fetchedType: ActorFetchByUrlType | ||
378 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
379 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
380 | |||
381 | // We need more attributes | ||
382 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
375 | 383 | ||
376 | try { | 384 | try { |
377 | const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | 385 | const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) |
diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts index 7b4067c11..a86428461 100644 --- a/server/lib/activitypub/audience.ts +++ b/server/lib/activitypub/audience.ts | |||
@@ -6,7 +6,7 @@ import { VideoModel } from '../../models/video/video' | |||
6 | import { VideoCommentModel } from '../../models/video/video-comment' | 6 | import { VideoCommentModel } from '../../models/video/video-comment' |
7 | import { VideoShareModel } from '../../models/video/video-share' | 7 | import { VideoShareModel } from '../../models/video/video-share' |
8 | 8 | ||
9 | function getVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]) { | 9 | function getRemoteVideoAudience (video: VideoModel, actorsInvolvedInVideo: ActorModel[]): ActivityAudience { |
10 | return { | 10 | return { |
11 | to: [ video.VideoChannel.Account.Actor.url ], | 11 | to: [ video.VideoChannel.Account.Actor.url ], |
12 | cc: actorsInvolvedInVideo.map(a => a.followersUrl) | 12 | cc: actorsInvolvedInVideo.map(a => a.followersUrl) |
@@ -18,7 +18,7 @@ function getVideoCommentAudience ( | |||
18 | threadParentComments: VideoCommentModel[], | 18 | threadParentComments: VideoCommentModel[], |
19 | actorsInvolvedInVideo: ActorModel[], | 19 | actorsInvolvedInVideo: ActorModel[], |
20 | isOrigin = false | 20 | isOrigin = false |
21 | ) { | 21 | ): ActivityAudience { |
22 | const to = [ ACTIVITY_PUB.PUBLIC ] | 22 | const to = [ ACTIVITY_PUB.PUBLIC ] |
23 | const cc: string[] = [] | 23 | const cc: string[] = [] |
24 | 24 | ||
@@ -41,7 +41,7 @@ function getVideoCommentAudience ( | |||
41 | } | 41 | } |
42 | } | 42 | } |
43 | 43 | ||
44 | function getObjectFollowersAudience (actorsInvolvedInObject: ActorModel[]) { | 44 | function getAudienceFromFollowersOf (actorsInvolvedInObject: ActorModel[]): ActivityAudience { |
45 | return { | 45 | return { |
46 | to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), | 46 | to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), |
47 | cc: [] | 47 | cc: [] |
@@ -83,9 +83,9 @@ function audiencify<T> (object: T, audience: ActivityAudience) { | |||
83 | export { | 83 | export { |
84 | buildAudience, | 84 | buildAudience, |
85 | getAudience, | 85 | getAudience, |
86 | getVideoAudience, | 86 | getRemoteVideoAudience, |
87 | getActorsInvolvedInVideo, | 87 | getActorsInvolvedInVideo, |
88 | getObjectFollowersAudience, | 88 | getAudienceFromFollowersOf, |
89 | audiencify, | 89 | audiencify, |
90 | getVideoCommentAudience | 90 | getVideoCommentAudience |
91 | } | 91 | } |
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 7325ddcb6..87f8a4162 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -1,10 +1,9 @@ | |||
1 | import { CacheFileObject } from '../../../shared/index' | 1 | import { CacheFileObject } from '../../../shared/index' |
2 | import { VideoModel } from '../../models/video/video' | 2 | import { VideoModel } from '../../models/video/video' |
3 | import { ActorModel } from '../../models/activitypub/actor' | ||
4 | import { sequelizeTypescript } from '../../initializers' | 3 | import { sequelizeTypescript } from '../../initializers' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 4 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | 5 | ||
7 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { | 6 | function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
8 | const url = cacheFileObject.url | 7 | const url = cacheFileObject.url |
9 | 8 | ||
10 | const videoFile = video.VideoFiles.find(f => { | 9 | const videoFile = video.VideoFiles.find(f => { |
@@ -23,7 +22,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
23 | } | 22 | } |
24 | } | 23 | } |
25 | 24 | ||
26 | function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { | 25 | function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { |
27 | return sequelizeTypescript.transaction(async t => { | 26 | return sequelizeTypescript.transaction(async t => { |
28 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) | 27 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) |
29 | 28 | ||
@@ -31,7 +30,11 @@ function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, b | |||
31 | }) | 30 | }) |
32 | } | 31 | } |
33 | 32 | ||
34 | function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) { | 33 | function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: { id?: number }) { |
34 | if (redundancyModel.actorId !== byActor.id) { | ||
35 | throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') | ||
36 | } | ||
37 | |||
35 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) | 38 | const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) |
36 | 39 | ||
37 | redundancyModel.set('expires', attributes.expiresOn) | 40 | redundancyModel.set('expires', attributes.expiresOn) |
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 046370b79..89bda9c32 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts | |||
@@ -1,15 +1,11 @@ | |||
1 | import { ActivityAccept } from '../../../../shared/models/activitypub' | 1 | import { ActivityAccept } from '../../../../shared/models/activitypub' |
2 | import { getActorUrl } from '../../../helpers/activitypub' | ||
3 | import { ActorModel } from '../../../models/activitypub/actor' | 2 | import { ActorModel } from '../../../models/activitypub/actor' |
4 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 3 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
5 | import { addFetchOutboxJob } from '../actor' | 4 | import { addFetchOutboxJob } from '../actor' |
6 | 5 | ||
7 | async function processAcceptActivity (activity: ActivityAccept, inboxActor?: ActorModel) { | 6 | async function processAcceptActivity (activity: ActivityAccept, targetActor: ActorModel, inboxActor?: ActorModel) { |
8 | if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') | 7 | if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') |
9 | 8 | ||
10 | const actorUrl = getActorUrl(activity.actor) | ||
11 | const targetActor = await ActorModel.loadByUrl(actorUrl) | ||
12 | |||
13 | return processAccept(inboxActor, targetActor) | 9 | return processAccept(inboxActor, targetActor) |
14 | } | 10 | } |
15 | 11 | ||
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 814556817..cc88b5423 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts | |||
@@ -2,15 +2,11 @@ import { ActivityAnnounce } from '../../../../shared/models/activitypub' | |||
2 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 2 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
3 | import { sequelizeTypescript } from '../../../initializers' | 3 | import { sequelizeTypescript } from '../../../initializers' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { VideoModel } from '../../../models/video/video' | ||
6 | import { VideoShareModel } from '../../../models/video/video-share' | 5 | import { VideoShareModel } from '../../../models/video/video-share' |
7 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
8 | import { forwardVideoRelatedActivity } from '../send/utils' | 6 | import { forwardVideoRelatedActivity } from '../send/utils' |
9 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
10 | 8 | ||
11 | async function processAnnounceActivity (activity: ActivityAnnounce) { | 9 | async function processAnnounceActivity (activity: ActivityAnnounce, actorAnnouncer: ActorModel) { |
12 | const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) | ||
13 | |||
14 | return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) | 10 | return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity) |
15 | } | 11 | } |
16 | 12 | ||
@@ -25,7 +21,7 @@ export { | |||
25 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { | 21 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { |
26 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id | 22 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id |
27 | 23 | ||
28 | const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) | 24 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) |
29 | 25 | ||
30 | return sequelizeTypescript.transaction(async t => { | 26 | return sequelizeTypescript.transaction(async t => { |
31 | // Add share entry | 27 | // Add share entry |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 32e555acf..5197dac73 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -7,30 +7,28 @@ import { sequelizeTypescript } from '../../../initializers' | |||
7 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 7 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
8 | import { ActorModel } from '../../../models/activitypub/actor' | 8 | import { ActorModel } from '../../../models/activitypub/actor' |
9 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | 9 | import { VideoAbuseModel } from '../../../models/video/video-abuse' |
10 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
11 | import { addVideoComment, resolveThread } from '../video-comments' | 10 | import { addVideoComment, resolveThread } from '../video-comments' |
12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 11 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
13 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' | 12 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' |
14 | import { Redis } from '../../redis' | 13 | import { Redis } from '../../redis' |
15 | import { createCacheFile } from '../cache-file' | 14 | import { createCacheFile } from '../cache-file' |
16 | 15 | ||
17 | async function processCreateActivity (activity: ActivityCreate) { | 16 | async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) { |
18 | const activityObject = activity.object | 17 | const activityObject = activity.object |
19 | const activityType = activityObject.type | 18 | const activityType = activityObject.type |
20 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | ||
21 | 19 | ||
22 | if (activityType === 'View') { | 20 | if (activityType === 'View') { |
23 | return processCreateView(actor, activity) | 21 | return processCreateView(byActor, activity) |
24 | } else if (activityType === 'Dislike') { | 22 | } else if (activityType === 'Dislike') { |
25 | return retryTransactionWrapper(processCreateDislike, actor, activity) | 23 | return retryTransactionWrapper(processCreateDislike, byActor, activity) |
26 | } else if (activityType === 'Video') { | 24 | } else if (activityType === 'Video') { |
27 | return processCreateVideo(activity) | 25 | return processCreateVideo(activity) |
28 | } else if (activityType === 'Flag') { | 26 | } else if (activityType === 'Flag') { |
29 | return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) | 27 | return retryTransactionWrapper(processCreateVideoAbuse, byActor, activityObject as VideoAbuseObject) |
30 | } else if (activityType === 'Note') { | 28 | } else if (activityType === 'Note') { |
31 | return retryTransactionWrapper(processCreateVideoComment, actor, activity) | 29 | return retryTransactionWrapper(processCreateVideoComment, byActor, activity) |
32 | } else if (activityType === 'CacheFile') { | 30 | } else if (activityType === 'CacheFile') { |
33 | return retryTransactionWrapper(processCacheFile, actor, activity) | 31 | return retryTransactionWrapper(processCacheFile, byActor, activity) |
34 | } | 32 | } |
35 | 33 | ||
36 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | 34 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) |
@@ -48,7 +46,7 @@ export { | |||
48 | async function processCreateVideo (activity: ActivityCreate) { | 46 | async function processCreateVideo (activity: ActivityCreate) { |
49 | const videoToCreateData = activity.object as VideoTorrentObject | 47 | const videoToCreateData = activity.object as VideoTorrentObject |
50 | 48 | ||
51 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) | 49 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) |
52 | 50 | ||
53 | return video | 51 | return video |
54 | } | 52 | } |
@@ -59,7 +57,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea | |||
59 | 57 | ||
60 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | 58 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
61 | 59 | ||
62 | const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) | 60 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) |
63 | 61 | ||
64 | return sequelizeTypescript.transaction(async t => { | 62 | return sequelizeTypescript.transaction(async t => { |
65 | const rate = { | 63 | const rate = { |
@@ -86,10 +84,14 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea | |||
86 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { | 84 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { |
87 | const view = activity.object as ViewObject | 85 | const view = activity.object as ViewObject |
88 | 86 | ||
89 | const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) | 87 | const options = { |
88 | videoObject: view.object, | ||
89 | fetchType: 'only-video' as 'only-video' | ||
90 | } | ||
91 | const { video } = await getOrCreateVideoAndAccountAndChannel(options) | ||
90 | 92 | ||
91 | const actor = await ActorModel.loadByUrl(view.actor) | 93 | const actorExists = await ActorModel.isActorUrlExist(view.actor) |
92 | if (!actor) throw new Error('Unknown actor ' + view.actor) | 94 | if (actorExists === false) throw new Error('Unknown actor ' + view.actor) |
93 | 95 | ||
94 | await Redis.Instance.addVideoView(video.id) | 96 | await Redis.Instance.addVideoView(video.id) |
95 | 97 | ||
@@ -103,7 +105,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) | |||
103 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { | 105 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { |
104 | const cacheFile = activity.object as CacheFileObject | 106 | const cacheFile = activity.object as CacheFileObject |
105 | 107 | ||
106 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) | 108 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) |
107 | 109 | ||
108 | await createCacheFile(cacheFile, video, byActor) | 110 | await createCacheFile(cacheFile, video, byActor) |
109 | 111 | ||
@@ -114,13 +116,13 @@ async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) | |||
114 | } | 116 | } |
115 | } | 117 | } |
116 | 118 | ||
117 | async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { | 119 | async function processCreateVideoAbuse (byActor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { |
118 | logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) | 120 | logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) |
119 | 121 | ||
120 | const account = actor.Account | 122 | const account = byActor.Account |
121 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) | 123 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
122 | 124 | ||
123 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) | 125 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) |
124 | 126 | ||
125 | return sequelizeTypescript.transaction(async t => { | 127 | return sequelizeTypescript.transaction(async t => { |
126 | const videoAbuseData = { | 128 | const videoAbuseData = { |
diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index 3c830abea..038d8c4d3 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts | |||
@@ -7,41 +7,41 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
7 | import { VideoModel } from '../../../models/video/video' | 7 | import { VideoModel } from '../../../models/video/video' |
8 | import { VideoChannelModel } from '../../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../../models/video/video-channel' |
9 | import { VideoCommentModel } from '../../../models/video/video-comment' | 9 | import { VideoCommentModel } from '../../../models/video/video-comment' |
10 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
11 | import { forwardActivity } from '../send/utils' | 10 | import { forwardActivity } from '../send/utils' |
12 | 11 | ||
13 | async function processDeleteActivity (activity: ActivityDelete) { | 12 | async function processDeleteActivity (activity: ActivityDelete, byActor: ActorModel) { |
14 | const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id | 13 | const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id |
15 | 14 | ||
16 | if (activity.actor === objectUrl) { | 15 | if (activity.actor === objectUrl) { |
17 | let actor = await ActorModel.loadByUrl(activity.actor) | 16 | // We need more attributes (all the account and channel) |
18 | if (!actor) return undefined | 17 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) |
19 | 18 | ||
20 | if (actor.type === 'Person') { | 19 | if (byActorFull.type === 'Person') { |
21 | if (!actor.Account) throw new Error('Actor ' + actor.url + ' is a person but we cannot find it in database.') | 20 | if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.') |
22 | 21 | ||
23 | actor.Account.Actor = await actor.Account.$get('Actor') as ActorModel | 22 | byActorFull.Account.Actor = await byActorFull.Account.$get('Actor') as ActorModel |
24 | return retryTransactionWrapper(processDeleteAccount, actor.Account) | 23 | return retryTransactionWrapper(processDeleteAccount, byActorFull.Account) |
25 | } else if (actor.type === 'Group') { | 24 | } else if (byActorFull.type === 'Group') { |
26 | if (!actor.VideoChannel) throw new Error('Actor ' + actor.url + ' is a group but we cannot find it in database.') | 25 | if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.') |
27 | 26 | ||
28 | actor.VideoChannel.Actor = await actor.VideoChannel.$get('Actor') as ActorModel | 27 | byActorFull.VideoChannel.Actor = await byActorFull.VideoChannel.$get('Actor') as ActorModel |
29 | return retryTransactionWrapper(processDeleteVideoChannel, actor.VideoChannel) | 28 | return retryTransactionWrapper(processDeleteVideoChannel, byActorFull.VideoChannel) |
30 | } | 29 | } |
31 | } | 30 | } |
32 | 31 | ||
33 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | ||
34 | { | 32 | { |
35 | const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl) | 33 | const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl) |
36 | if (videoCommentInstance) { | 34 | if (videoCommentInstance) { |
37 | return retryTransactionWrapper(processDeleteVideoComment, actor, videoCommentInstance, activity) | 35 | return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity) |
38 | } | 36 | } |
39 | } | 37 | } |
40 | 38 | ||
41 | { | 39 | { |
42 | const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) | 40 | const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) |
43 | if (videoInstance) { | 41 | if (videoInstance) { |
44 | return retryTransactionWrapper(processDeleteVideo, actor, videoInstance) | 42 | if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) |
43 | |||
44 | return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance) | ||
45 | } | 45 | } |
46 | } | 46 | } |
47 | 47 | ||
@@ -94,6 +94,10 @@ function processDeleteVideoComment (byActor: ActorModel, videoComment: VideoComm | |||
94 | logger.debug('Removing remote video comment "%s".', videoComment.url) | 94 | logger.debug('Removing remote video comment "%s".', videoComment.url) |
95 | 95 | ||
96 | return sequelizeTypescript.transaction(async t => { | 96 | return sequelizeTypescript.transaction(async t => { |
97 | if (videoComment.Account.id !== byActor.Account.id) { | ||
98 | throw new Error('Account ' + byActor.url + ' does not own video comment ' + videoComment.url) | ||
99 | } | ||
100 | |||
97 | await videoComment.destroy({ transaction: t }) | 101 | await videoComment.destroy({ transaction: t }) |
98 | 102 | ||
99 | if (videoComment.Video.isOwned()) { | 103 | if (videoComment.Video.isOwned()) { |
diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index f34fd66cc..24c9085f7 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts | |||
@@ -4,14 +4,12 @@ import { logger } from '../../../helpers/logger' | |||
4 | import { sequelizeTypescript } from '../../../initializers' | 4 | import { sequelizeTypescript } from '../../../initializers' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 6 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
7 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
8 | import { sendAccept } from '../send' | 7 | import { sendAccept } from '../send' |
9 | 8 | ||
10 | async function processFollowActivity (activity: ActivityFollow) { | 9 | async function processFollowActivity (activity: ActivityFollow, byActor: ActorModel) { |
11 | const activityObject = activity.object | 10 | const activityObject = activity.object |
12 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | ||
13 | 11 | ||
14 | return retryTransactionWrapper(processFollow, actor, activityObject) | 12 | return retryTransactionWrapper(processFollow, byActor, activityObject) |
15 | } | 13 | } |
16 | 14 | ||
17 | // --------------------------------------------------------------------------- | 15 | // --------------------------------------------------------------------------- |
@@ -24,7 +22,7 @@ export { | |||
24 | 22 | ||
25 | async function processFollow (actor: ActorModel, targetActorURL: string) { | 23 | async function processFollow (actor: ActorModel, targetActorURL: string) { |
26 | await sequelizeTypescript.transaction(async t => { | 24 | await sequelizeTypescript.transaction(async t => { |
27 | const targetActor = await ActorModel.loadByUrl(targetActorURL, t) | 25 | const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) |
28 | 26 | ||
29 | if (!targetActor) throw new Error('Unknown actor') | 27 | if (!targetActor) throw new Error('Unknown actor') |
30 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') | 28 | if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 9e1664fd8..f7200db61 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -3,14 +3,11 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
3 | import { sequelizeTypescript } from '../../../initializers' | 3 | import { sequelizeTypescript } from '../../../initializers' |
4 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 4 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
7 | import { forwardVideoRelatedActivity } from '../send/utils' | 6 | import { forwardVideoRelatedActivity } from '../send/utils' |
8 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 7 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
9 | 8 | ||
10 | async function processLikeActivity (activity: ActivityLike) { | 9 | async function processLikeActivity (activity: ActivityLike, byActor: ActorModel) { |
11 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | 10 | return retryTransactionWrapper(processLikeVideo, byActor, activity) |
12 | |||
13 | return retryTransactionWrapper(processLikeVideo, actor, activity) | ||
14 | } | 11 | } |
15 | 12 | ||
16 | // --------------------------------------------------------------------------- | 13 | // --------------------------------------------------------------------------- |
@@ -27,7 +24,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { | |||
27 | const byAccount = byActor.Account | 24 | const byAccount = byActor.Account |
28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 25 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
29 | 26 | ||
30 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) | 27 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) |
31 | 28 | ||
32 | return sequelizeTypescript.transaction(async t => { | 29 | return sequelizeTypescript.transaction(async t => { |
33 | const rate = { | 30 | const rate = { |
diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts index f06b03772..709a65096 100644 --- a/server/lib/activitypub/process/process-reject.ts +++ b/server/lib/activitypub/process/process-reject.ts | |||
@@ -1,15 +1,11 @@ | |||
1 | import { ActivityReject } from '../../../../shared/models/activitypub/activity' | 1 | import { ActivityReject } from '../../../../shared/models/activitypub/activity' |
2 | import { getActorUrl } from '../../../helpers/activitypub' | ||
3 | import { sequelizeTypescript } from '../../../initializers' | 2 | import { sequelizeTypescript } from '../../../initializers' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 3 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 4 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
6 | 5 | ||
7 | async function processRejectActivity (activity: ActivityReject, inboxActor?: ActorModel) { | 6 | async function processRejectActivity (activity: ActivityReject, targetActor: ActorModel, inboxActor?: ActorModel) { |
8 | if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') | 7 | if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') |
9 | 8 | ||
10 | const actorUrl = getActorUrl(activity.actor) | ||
11 | const targetActor = await ActorModel.loadByUrl(actorUrl) | ||
12 | |||
13 | return processReject(inboxActor, targetActor) | 9 | return processReject(inboxActor, targetActor) |
14 | } | 10 | } |
15 | 11 | ||
@@ -21,11 +17,11 @@ export { | |||
21 | 17 | ||
22 | // --------------------------------------------------------------------------- | 18 | // --------------------------------------------------------------------------- |
23 | 19 | ||
24 | async function processReject (actor: ActorModel, targetActor: ActorModel) { | 20 | async function processReject (follower: ActorModel, targetActor: ActorModel) { |
25 | return sequelizeTypescript.transaction(async t => { | 21 | return sequelizeTypescript.transaction(async t => { |
26 | const actorFollow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id, t) | 22 | const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t) |
27 | 23 | ||
28 | if (!actorFollow) throw new Error(`'Unknown actor follow ${actor.id} -> ${targetActor.id}.`) | 24 | if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`) |
29 | 25 | ||
30 | await actorFollow.destroy({ transaction: t }) | 26 | await actorFollow.destroy({ transaction: t }) |
31 | 27 | ||
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 0eb5fa392..73ca0a17c 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -1,10 +1,8 @@ | |||
1 | import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' | 1 | import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' |
2 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' | 2 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' |
3 | import { getActorUrl } from '../../../helpers/activitypub' | ||
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
5 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
6 | import { sequelizeTypescript } from '../../../initializers' | 5 | import { sequelizeTypescript } from '../../../initializers' |
7 | import { AccountModel } from '../../../models/account/account' | ||
8 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 6 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
9 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { ActorModel } from '../../../models/activitypub/actor' |
10 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 8 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
@@ -13,29 +11,27 @@ import { getOrCreateVideoAndAccountAndChannel } from '../videos' | |||
13 | import { VideoShareModel } from '../../../models/video/video-share' | 11 | import { VideoShareModel } from '../../../models/video/video-share' |
14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 12 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
15 | 13 | ||
16 | async function processUndoActivity (activity: ActivityUndo) { | 14 | async function processUndoActivity (activity: ActivityUndo, byActor: ActorModel) { |
17 | const activityToUndo = activity.object | 15 | const activityToUndo = activity.object |
18 | 16 | ||
19 | const actorUrl = getActorUrl(activity.actor) | ||
20 | |||
21 | if (activityToUndo.type === 'Like') { | 17 | if (activityToUndo.type === 'Like') { |
22 | return retryTransactionWrapper(processUndoLike, actorUrl, activity) | 18 | return retryTransactionWrapper(processUndoLike, byActor, activity) |
23 | } | 19 | } |
24 | 20 | ||
25 | if (activityToUndo.type === 'Create') { | 21 | if (activityToUndo.type === 'Create') { |
26 | if (activityToUndo.object.type === 'Dislike') { | 22 | if (activityToUndo.object.type === 'Dislike') { |
27 | return retryTransactionWrapper(processUndoDislike, actorUrl, activity) | 23 | return retryTransactionWrapper(processUndoDislike, byActor, activity) |
28 | } else if (activityToUndo.object.type === 'CacheFile') { | 24 | } else if (activityToUndo.object.type === 'CacheFile') { |
29 | return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity) | 25 | return retryTransactionWrapper(processUndoCacheFile, byActor, activity) |
30 | } | 26 | } |
31 | } | 27 | } |
32 | 28 | ||
33 | if (activityToUndo.type === 'Follow') { | 29 | if (activityToUndo.type === 'Follow') { |
34 | return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) | 30 | return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) |
35 | } | 31 | } |
36 | 32 | ||
37 | if (activityToUndo.type === 'Announce') { | 33 | if (activityToUndo.type === 'Announce') { |
38 | return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) | 34 | return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo) |
39 | } | 35 | } |
40 | 36 | ||
41 | logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) | 37 | logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) |
@@ -51,66 +47,63 @@ export { | |||
51 | 47 | ||
52 | // --------------------------------------------------------------------------- | 48 | // --------------------------------------------------------------------------- |
53 | 49 | ||
54 | async function processUndoLike (actorUrl: string, activity: ActivityUndo) { | 50 | async function processUndoLike (byActor: ActorModel, activity: ActivityUndo) { |
55 | const likeActivity = activity.object as ActivityLike | 51 | const likeActivity = activity.object as ActivityLike |
56 | 52 | ||
57 | const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) | 53 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) |
58 | 54 | ||
59 | return sequelizeTypescript.transaction(async t => { | 55 | return sequelizeTypescript.transaction(async t => { |
60 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) | 56 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
61 | if (!byAccount) throw new Error('Unknown account ' + actorUrl) | ||
62 | 57 | ||
63 | const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) | 58 | const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) |
64 | if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) | 59 | if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) |
65 | 60 | ||
66 | await rate.destroy({ transaction: t }) | 61 | await rate.destroy({ transaction: t }) |
67 | await video.decrement('likes', { transaction: t }) | 62 | await video.decrement('likes', { transaction: t }) |
68 | 63 | ||
69 | if (video.isOwned()) { | 64 | if (video.isOwned()) { |
70 | // Don't resend the activity to the sender | 65 | // Don't resend the activity to the sender |
71 | const exceptions = [ byAccount.Actor ] | 66 | const exceptions = [ byActor ] |
72 | 67 | ||
73 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | 68 | await forwardVideoRelatedActivity(activity, t, exceptions, video) |
74 | } | 69 | } |
75 | }) | 70 | }) |
76 | } | 71 | } |
77 | 72 | ||
78 | async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { | 73 | async function processUndoDislike (byActor: ActorModel, activity: ActivityUndo) { |
79 | const dislike = activity.object.object as DislikeObject | 74 | const dislike = activity.object.object as DislikeObject |
80 | 75 | ||
81 | const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) | 76 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) |
82 | 77 | ||
83 | return sequelizeTypescript.transaction(async t => { | 78 | return sequelizeTypescript.transaction(async t => { |
84 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) | 79 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
85 | if (!byAccount) throw new Error('Unknown account ' + actorUrl) | ||
86 | 80 | ||
87 | const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) | 81 | const rate = await AccountVideoRateModel.load(byActor.Account.id, video.id, t) |
88 | if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) | 82 | if (!rate) throw new Error(`Unknown rate by account ${byActor.Account.id} for video ${video.id}.`) |
89 | 83 | ||
90 | await rate.destroy({ transaction: t }) | 84 | await rate.destroy({ transaction: t }) |
91 | await video.decrement('dislikes', { transaction: t }) | 85 | await video.decrement('dislikes', { transaction: t }) |
92 | 86 | ||
93 | if (video.isOwned()) { | 87 | if (video.isOwned()) { |
94 | // Don't resend the activity to the sender | 88 | // Don't resend the activity to the sender |
95 | const exceptions = [ byAccount.Actor ] | 89 | const exceptions = [ byActor ] |
96 | 90 | ||
97 | await forwardVideoRelatedActivity(activity, t, exceptions, video) | 91 | await forwardVideoRelatedActivity(activity, t, exceptions, video) |
98 | } | 92 | } |
99 | }) | 93 | }) |
100 | } | 94 | } |
101 | 95 | ||
102 | async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { | 96 | async function processUndoCacheFile (byActor: ActorModel, activity: ActivityUndo) { |
103 | const cacheFileObject = activity.object.object as CacheFileObject | 97 | const cacheFileObject = activity.object.object as CacheFileObject |
104 | 98 | ||
105 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) | 99 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) |
106 | 100 | ||
107 | return sequelizeTypescript.transaction(async t => { | 101 | return sequelizeTypescript.transaction(async t => { |
108 | const byActor = await ActorModel.loadByUrl(actorUrl) | ||
109 | if (!byActor) throw new Error('Unknown actor ' + actorUrl) | ||
110 | |||
111 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 102 | const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) |
112 | if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) | 103 | if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) |
113 | 104 | ||
105 | if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') | ||
106 | |||
114 | await cacheFile.destroy() | 107 | await cacheFile.destroy() |
115 | 108 | ||
116 | if (video.isOwned()) { | 109 | if (video.isOwned()) { |
@@ -122,10 +115,9 @@ async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { | |||
122 | }) | 115 | }) |
123 | } | 116 | } |
124 | 117 | ||
125 | function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { | 118 | function processUndoFollow (follower: ActorModel, followActivity: ActivityFollow) { |
126 | return sequelizeTypescript.transaction(async t => { | 119 | return sequelizeTypescript.transaction(async t => { |
127 | const follower = await ActorModel.loadByUrl(actorUrl, t) | 120 | const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t) |
128 | const following = await ActorModel.loadByUrl(followActivity.object, t) | ||
129 | const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) | 121 | const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) |
130 | 122 | ||
131 | if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`) | 123 | if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${following.id}.`) |
@@ -136,11 +128,8 @@ function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { | |||
136 | }) | 128 | }) |
137 | } | 129 | } |
138 | 130 | ||
139 | function processUndoAnnounce (actorUrl: string, announceActivity: ActivityAnnounce) { | 131 | function processUndoAnnounce (byActor: ActorModel, announceActivity: ActivityAnnounce) { |
140 | return sequelizeTypescript.transaction(async t => { | 132 | return sequelizeTypescript.transaction(async t => { |
141 | const byActor = await ActorModel.loadByUrl(actorUrl, t) | ||
142 | if (!byActor) throw new Error('Unknown actor ' + actorUrl) | ||
143 | |||
144 | const share = await VideoShareModel.loadByUrl(announceActivity.id, t) | 133 | const share = await VideoShareModel.loadByUrl(announceActivity.id, t) |
145 | if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`) | 134 | if (!share) throw new Error(`Unknown video share ${announceActivity.id}.`) |
146 | 135 | ||
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d3af1a181..ed3489ebf 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -6,27 +6,30 @@ import { sequelizeTypescript } from '../../../initializers' | |||
6 | import { AccountModel } from '../../../models/account/account' | 6 | import { AccountModel } from '../../../models/account/account' |
7 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { ActorModel } from '../../../models/activitypub/actor' |
8 | import { VideoChannelModel } from '../../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../../models/video/video-channel' |
9 | import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' | 9 | import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' |
10 | import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos' | 10 | import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' |
11 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' | 11 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' |
12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | 12 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' |
13 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 13 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
14 | import { createCacheFile, updateCacheFile } from '../cache-file' | 14 | import { createCacheFile, updateCacheFile } from '../cache-file' |
15 | 15 | ||
16 | async function processUpdateActivity (activity: ActivityUpdate) { | 16 | async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) { |
17 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | ||
18 | const objectType = activity.object.type | 17 | const objectType = activity.object.type |
19 | 18 | ||
20 | if (objectType === 'Video') { | 19 | if (objectType === 'Video') { |
21 | return retryTransactionWrapper(processUpdateVideo, actor, activity) | 20 | return retryTransactionWrapper(processUpdateVideo, byActor, activity) |
22 | } | 21 | } |
23 | 22 | ||
24 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { | 23 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { |
25 | return retryTransactionWrapper(processUpdateActor, actor, activity) | 24 | // We need more attributes |
25 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | ||
26 | return retryTransactionWrapper(processUpdateActor, byActorFull, activity) | ||
26 | } | 27 | } |
27 | 28 | ||
28 | if (objectType === 'CacheFile') { | 29 | if (objectType === 'CacheFile') { |
29 | return retryTransactionWrapper(processUpdateCacheFile, actor, activity) | 30 | // We need more attributes |
31 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | ||
32 | return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity) | ||
30 | } | 33 | } |
31 | 34 | ||
32 | return undefined | 35 | return undefined |
@@ -48,10 +51,18 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
48 | return undefined | 51 | return undefined |
49 | } | 52 | } |
50 | 53 | ||
51 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) | 54 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) |
52 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 55 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) |
53 | 56 | ||
54 | return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) | 57 | const updateOptions = { |
58 | video, | ||
59 | videoObject, | ||
60 | account: actor.Account, | ||
61 | channel: channelActor.VideoChannel, | ||
62 | updateViews: true, | ||
63 | overrideTo: activity.to | ||
64 | } | ||
65 | return updateVideoFromAP(updateOptions) | ||
55 | } | 66 | } |
56 | 67 | ||
57 | async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { | 68 | async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { |
@@ -64,7 +75,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp | |||
64 | 75 | ||
65 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 76 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) |
66 | if (!redundancyModel) { | 77 | if (!redundancyModel) { |
67 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) | 78 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) |
68 | return createCacheFile(cacheFileObject, video, byActor) | 79 | return createCacheFile(cacheFileObject, video, byActor) |
69 | } | 80 | } |
70 | 81 | ||
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index da91675ce..b263f1ea2 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts | |||
@@ -11,8 +11,9 @@ import { processLikeActivity } from './process-like' | |||
11 | import { processRejectActivity } from './process-reject' | 11 | import { processRejectActivity } from './process-reject' |
12 | import { processUndoActivity } from './process-undo' | 12 | import { processUndoActivity } from './process-undo' |
13 | import { processUpdateActivity } from './process-update' | 13 | import { processUpdateActivity } from './process-update' |
14 | import { getOrCreateActorAndServerAndModel } from '../actor' | ||
14 | 15 | ||
15 | const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor?: ActorModel) => Promise<any> } = { | 16 | const processActivity: { [ P in ActivityType ]: (activity: Activity, byActor: ActorModel, inboxActor?: ActorModel) => Promise<any> } = { |
16 | Create: processCreateActivity, | 17 | Create: processCreateActivity, |
17 | Update: processUpdateActivity, | 18 | Update: processUpdateActivity, |
18 | Delete: processDeleteActivity, | 19 | Delete: processDeleteActivity, |
@@ -25,7 +26,14 @@ const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxActor? | |||
25 | } | 26 | } |
26 | 27 | ||
27 | async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { | 28 | async function processActivities (activities: Activity[], signatureActor?: ActorModel, inboxActor?: ActorModel) { |
29 | const actorsCache: { [ url: string ]: ActorModel } = {} | ||
30 | |||
28 | for (const activity of activities) { | 31 | for (const activity of activities) { |
32 | if (!signatureActor && [ 'Create', 'Announce', 'Like' ].indexOf(activity.type) === -1) { | ||
33 | logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) | ||
34 | continue | ||
35 | } | ||
36 | |||
29 | const actorUrl = getActorUrl(activity.actor) | 37 | const actorUrl = getActorUrl(activity.actor) |
30 | 38 | ||
31 | // When we fetch remote data, we don't have signature | 39 | // When we fetch remote data, we don't have signature |
@@ -34,6 +42,9 @@ async function processActivities (activities: Activity[], signatureActor?: Actor | |||
34 | continue | 42 | continue |
35 | } | 43 | } |
36 | 44 | ||
45 | const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) | ||
46 | actorsCache[actorUrl] = byActor | ||
47 | |||
37 | const activityProcessor = processActivity[activity.type] | 48 | const activityProcessor = processActivity[activity.type] |
38 | if (activityProcessor === undefined) { | 49 | if (activityProcessor === undefined) { |
39 | logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) | 50 | logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) |
@@ -41,7 +52,7 @@ async function processActivities (activities: Activity[], signatureActor?: Actor | |||
41 | } | 52 | } |
42 | 53 | ||
43 | try { | 54 | try { |
44 | await activityProcessor(activity, inboxActor) | 55 | await activityProcessor(activity, byActor, inboxActor) |
45 | } catch (err) { | 56 | } catch (err) { |
46 | logger.warn('Cannot process activity %s.', activity.type, { err }) | 57 | logger.warn('Cannot process activity %s.', activity.type, { err }) |
47 | } | 58 | } |
diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index f137217f8..cd0cab7ee 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts | |||
@@ -4,14 +4,14 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
4 | import { VideoModel } from '../../../models/video/video' | 4 | import { VideoModel } from '../../../models/video/video' |
5 | import { VideoShareModel } from '../../../models/video/video-share' | 5 | import { VideoShareModel } from '../../../models/video/video-share' |
6 | import { broadcastToFollowers } from './utils' | 6 | import { broadcastToFollowers } from './utils' |
7 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' | 7 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | 9 | ||
10 | async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { | 10 | async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { |
11 | const announcedObject = video.url | 11 | const announcedObject = video.url |
12 | 12 | ||
13 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 13 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) |
14 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | 14 | const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) |
15 | 15 | ||
16 | const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) | 16 | const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) |
17 | 17 | ||
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 6f89b1a22..285edba3b 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -1,21 +1,13 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' | 2 | import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' |
3 | import { VideoPrivacy } from '../../../../shared/models/videos' | 3 | import { VideoPrivacy } from '../../../../shared/models/videos' |
4 | import { getServerActor } from '../../../helpers/utils' | ||
5 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { VideoModel } from '../../../models/video/video' | 5 | import { VideoModel } from '../../../models/video/video' |
7 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | 6 | import { VideoAbuseModel } from '../../../models/video/video-abuse' |
8 | import { VideoCommentModel } from '../../../models/video/video-comment' | 7 | import { VideoCommentModel } from '../../../models/video/video-comment' |
9 | import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' | 8 | import { getVideoAbuseActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoViewActivityPubUrl } from '../url' |
10 | import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' | 9 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
11 | import { | 10 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience' |
12 | audiencify, | ||
13 | getActorsInvolvedInVideo, | ||
14 | getAudience, | ||
15 | getObjectFollowersAudience, | ||
16 | getVideoAudience, | ||
17 | getVideoCommentAudience | ||
18 | } from '../audience' | ||
19 | import { logger } from '../../../helpers/logger' | 11 | import { logger } from '../../../helpers/logger' |
20 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 12 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
21 | 13 | ||
@@ -40,6 +32,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, | |||
40 | 32 | ||
41 | logger.info('Creating job to send video abuse %s.', url) | 33 | logger.info('Creating job to send video abuse %s.', url) |
42 | 34 | ||
35 | // Custom audience, we only send the abuse to the origin instance | ||
43 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } | 36 | const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } |
44 | const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) | 37 | const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) |
45 | 38 | ||
@@ -49,15 +42,15 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, | |||
49 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { | 42 | async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { |
50 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) | 43 | logger.info('Creating job to send file cache of %s.', fileRedundancy.url) |
51 | 44 | ||
52 | const redundancyObject = fileRedundancy.toActivityPubObject() | ||
53 | |||
54 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) | 45 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) |
55 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) | 46 | const redundancyObject = fileRedundancy.toActivityPubObject() |
56 | |||
57 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | ||
58 | const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience) | ||
59 | 47 | ||
60 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 48 | return sendVideoRelatedCreateActivity({ |
49 | byActor, | ||
50 | video, | ||
51 | url: fileRedundancy.url, | ||
52 | object: redundancyObject | ||
53 | }) | ||
61 | } | 54 | } |
62 | 55 | ||
63 | async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { | 56 | async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { |
@@ -70,6 +63,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio | |||
70 | const commentObject = comment.toActivityPubObject(threadParentComments) | 63 | const commentObject = comment.toActivityPubObject(threadParentComments) |
71 | 64 | ||
72 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t) | 65 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, t) |
66 | // Add the actor that commented too | ||
73 | actorsInvolvedInComment.push(byActor) | 67 | actorsInvolvedInComment.push(byActor) |
74 | 68 | ||
75 | const parentsCommentActors = threadParentComments.map(c => c.Account.Actor) | 69 | const parentsCommentActors = threadParentComments.map(c => c.Account.Actor) |
@@ -78,7 +72,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio | |||
78 | if (isOrigin) { | 72 | if (isOrigin) { |
79 | audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) | 73 | audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) |
80 | } else { | 74 | } else { |
81 | audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) | 75 | audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors)) |
82 | } | 76 | } |
83 | 77 | ||
84 | const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) | 78 | const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) |
@@ -103,24 +97,14 @@ async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transa | |||
103 | const url = getVideoViewActivityPubUrl(byActor, video) | 97 | const url = getVideoViewActivityPubUrl(byActor, video) |
104 | const viewActivity = buildViewActivity(byActor, video) | 98 | const viewActivity = buildViewActivity(byActor, video) |
105 | 99 | ||
106 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 100 | return sendVideoRelatedCreateActivity({ |
107 | 101 | // Use the server actor to send the view | |
108 | // Send to origin | 102 | byActor, |
109 | if (video.isOwned() === false) { | 103 | video, |
110 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 104 | url, |
111 | const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) | 105 | object: viewActivity, |
112 | 106 | transaction: t | |
113 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 107 | }) |
114 | } | ||
115 | |||
116 | // Send to followers | ||
117 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | ||
118 | const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) | ||
119 | |||
120 | // Use the server actor to send the view | ||
121 | const serverActor = await getServerActor() | ||
122 | const actorsException = [ byActor ] | ||
123 | return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException) | ||
124 | } | 108 | } |
125 | 109 | ||
126 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 110 | async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
@@ -129,22 +113,13 @@ async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Tra | |||
129 | const url = getVideoDislikeActivityPubUrl(byActor, video) | 113 | const url = getVideoDislikeActivityPubUrl(byActor, video) |
130 | const dislikeActivity = buildDislikeActivity(byActor, video) | 114 | const dislikeActivity = buildDislikeActivity(byActor, video) |
131 | 115 | ||
132 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 116 | return sendVideoRelatedCreateActivity({ |
133 | 117 | byActor, | |
134 | // Send to origin | 118 | video, |
135 | if (video.isOwned() === false) { | 119 | url, |
136 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 120 | object: dislikeActivity, |
137 | const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) | 121 | transaction: t |
138 | 122 | }) | |
139 | return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
140 | } | ||
141 | |||
142 | // Send to followers | ||
143 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | ||
144 | const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) | ||
145 | |||
146 | const actorsException = [ byActor ] | ||
147 | return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException) | ||
148 | } | 123 | } |
149 | 124 | ||
150 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { | 125 | function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { |
@@ -189,3 +164,19 @@ export { | |||
189 | sendCreateVideoComment, | 164 | sendCreateVideoComment, |
190 | sendCreateCacheFile | 165 | sendCreateCacheFile |
191 | } | 166 | } |
167 | |||
168 | // --------------------------------------------------------------------------- | ||
169 | |||
170 | async function sendVideoRelatedCreateActivity (options: { | ||
171 | byActor: ActorModel, | ||
172 | video: VideoModel, | ||
173 | url: string, | ||
174 | object: any, | ||
175 | transaction?: Transaction | ||
176 | }) { | ||
177 | const activityBuilder = (audience: ActivityAudience) => { | ||
178 | return buildCreateActivity(options.url, options.byActor, options.object, audience) | ||
179 | } | ||
180 | |||
181 | return sendVideoRelatedActivity(activityBuilder, options) | ||
182 | } | ||
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index 479182543..18969433a 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts | |||
@@ -5,21 +5,22 @@ import { VideoModel } from '../../../models/video/video' | |||
5 | import { VideoCommentModel } from '../../../models/video/video-comment' | 5 | import { VideoCommentModel } from '../../../models/video/video-comment' |
6 | import { VideoShareModel } from '../../../models/video/video-share' | 6 | import { VideoShareModel } from '../../../models/video/video-share' |
7 | import { getDeleteActivityPubUrl } from '../url' | 7 | import { getDeleteActivityPubUrl } from '../url' |
8 | import { broadcastToActors, broadcastToFollowers, unicastTo } from './utils' | 8 | import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
9 | import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' | 9 | import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience' |
10 | import { logger } from '../../../helpers/logger' | 10 | import { logger } from '../../../helpers/logger' |
11 | 11 | ||
12 | async function sendDeleteVideo (video: VideoModel, t: Transaction) { | 12 | async function sendDeleteVideo (video: VideoModel, transaction: Transaction) { |
13 | logger.info('Creating job to broadcast delete of video %s.', video.url) | 13 | logger.info('Creating job to broadcast delete of video %s.', video.url) |
14 | 14 | ||
15 | const url = getDeleteActivityPubUrl(video.url) | ||
16 | const byActor = video.VideoChannel.Account.Actor | 15 | const byActor = video.VideoChannel.Account.Actor |
17 | 16 | ||
18 | const activity = buildDeleteActivity(url, video.url, byActor) | 17 | const activityBuilder = (audience: ActivityAudience) => { |
18 | const url = getDeleteActivityPubUrl(video.url) | ||
19 | 19 | ||
20 | const actorsInvolved = await getActorsInvolvedInVideo(video, t) | 20 | return buildDeleteActivity(url, video.url, byActor, audience) |
21 | } | ||
21 | 22 | ||
22 | return broadcastToFollowers(activity, byActor, actorsInvolved, t) | 23 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction }) |
23 | } | 24 | } |
24 | 25 | ||
25 | async function sendDeleteActor (byActor: ActorModel, t: Transaction) { | 26 | async function sendDeleteActor (byActor: ActorModel, t: Transaction) { |
diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index a5408ac6a..89307acc6 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts | |||
@@ -3,31 +3,20 @@ import { ActivityAudience, ActivityLike } from '../../../../shared/models/activi | |||
3 | import { ActorModel } from '../../../models/activitypub/actor' | 3 | import { ActorModel } from '../../../models/activitypub/actor' |
4 | import { VideoModel } from '../../../models/video/video' | 4 | import { VideoModel } from '../../../models/video/video' |
5 | import { getVideoLikeActivityPubUrl } from '../url' | 5 | import { getVideoLikeActivityPubUrl } from '../url' |
6 | import { broadcastToFollowers, unicastTo } from './utils' | 6 | import { sendVideoRelatedActivity } from './utils' |
7 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' | 7 | import { audiencify, getAudience } from '../audience' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | 9 | ||
10 | async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 10 | async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
11 | logger.info('Creating job to like %s.', video.url) | 11 | logger.info('Creating job to like %s.', video.url) |
12 | 12 | ||
13 | const url = getVideoLikeActivityPubUrl(byActor, video) | 13 | const activityBuilder = (audience: ActivityAudience) => { |
14 | const url = getVideoLikeActivityPubUrl(byActor, video) | ||
14 | 15 | ||
15 | const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 16 | return buildLikeActivity(url, byActor, video, audience) |
16 | |||
17 | // Send to origin | ||
18 | if (video.isOwned() === false) { | ||
19 | const audience = getVideoAudience(video, accountsInvolvedInVideo) | ||
20 | const data = buildLikeActivity(url, byActor, video, audience) | ||
21 | |||
22 | return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
23 | } | 17 | } |
24 | 18 | ||
25 | // Send to followers | 19 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t }) |
26 | const audience = getObjectFollowersAudience(accountsInvolvedInVideo) | ||
27 | const activity = buildLikeActivity(url, byActor, video, audience) | ||
28 | |||
29 | const followersException = [ byActor ] | ||
30 | return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException) | ||
31 | } | 20 | } |
32 | 21 | ||
33 | function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { | 22 | function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { |
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index a50673c79..5236d2cb3 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -11,8 +11,8 @@ import { ActorModel } from '../../../models/activitypub/actor' | |||
11 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 11 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
12 | import { VideoModel } from '../../../models/video/video' | 12 | import { VideoModel } from '../../../models/video/video' |
13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' | 13 | import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' |
14 | import { broadcastToFollowers, unicastTo } from './utils' | 14 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
15 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' | 15 | import { audiencify, getAudience } from '../audience' |
16 | import { buildCreateActivity, buildDislikeActivity } from './send-create' | 16 | import { buildCreateActivity, buildDislikeActivity } from './send-create' |
17 | import { buildFollowActivity } from './send-follow' | 17 | import { buildFollowActivity } from './send-follow' |
18 | import { buildLikeActivity } from './send-like' | 18 | import { buildLikeActivity } from './send-like' |
@@ -39,79 +39,44 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { | |||
39 | return unicastTo(undoActivity, me, following.inboxUrl) | 39 | return unicastTo(undoActivity, me, following.inboxUrl) |
40 | } | 40 | } |
41 | 41 | ||
42 | async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 42 | async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { |
43 | logger.info('Creating job to undo a like of video %s.', video.url) | 43 | logger.info('Creating job to undo announce %s.', videoShare.url) |
44 | 44 | ||
45 | const likeUrl = getVideoLikeActivityPubUrl(byActor, video) | 45 | const undoUrl = getUndoActivityPubUrl(videoShare.url) |
46 | const undoUrl = getUndoActivityPubUrl(likeUrl) | ||
47 | 46 | ||
48 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | 47 | const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t) |
49 | const likeActivity = buildLikeActivity(likeUrl, byActor, video) | 48 | const undoActivity = undoActivityData(undoUrl, byActor, announceActivity) |
50 | 49 | ||
51 | // Send to origin | 50 | const followersException = [ byActor ] |
52 | if (video.isOwned() === false) { | 51 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) |
53 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | 52 | } |
54 | const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) | ||
55 | 53 | ||
56 | return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | 54 | async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
57 | } | 55 | logger.info('Creating job to undo a like of video %s.', video.url) |
58 | 56 | ||
59 | const audience = getObjectFollowersAudience(actorsInvolvedInVideo) | 57 | const likeUrl = getVideoLikeActivityPubUrl(byActor, video) |
60 | const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) | 58 | const likeActivity = buildLikeActivity(likeUrl, byActor, video) |
61 | 59 | ||
62 | const followersException = [ byActor ] | 60 | return sendUndoVideoRelatedActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t }) |
63 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) | ||
64 | } | 61 | } |
65 | 62 | ||
66 | async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { | 63 | async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { |
67 | logger.info('Creating job to undo a dislike of video %s.', video.url) | 64 | logger.info('Creating job to undo a dislike of video %s.', video.url) |
68 | 65 | ||
69 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) | 66 | const dislikeUrl = getVideoDislikeActivityPubUrl(byActor, video) |
70 | const undoUrl = getUndoActivityPubUrl(dislikeUrl) | ||
71 | |||
72 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | ||
73 | const dislikeActivity = buildDislikeActivity(byActor, video) | 67 | const dislikeActivity = buildDislikeActivity(byActor, video) |
74 | const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) | 68 | const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) |
75 | 69 | ||
76 | if (video.isOwned() === false) { | 70 | return sendUndoVideoRelatedActivity({ byActor, video, url: dislikeUrl, activity: createDislikeActivity, transaction: t }) |
77 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | ||
78 | const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience) | ||
79 | |||
80 | return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
81 | } | ||
82 | |||
83 | const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity) | ||
84 | |||
85 | const followersException = [ byActor ] | ||
86 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) | ||
87 | } | ||
88 | |||
89 | async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { | ||
90 | logger.info('Creating job to undo announce %s.', videoShare.url) | ||
91 | |||
92 | const undoUrl = getUndoActivityPubUrl(videoShare.url) | ||
93 | |||
94 | const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t) | ||
95 | const undoActivity = undoActivityData(undoUrl, byActor, announceActivity) | ||
96 | |||
97 | const followersException = [ byActor ] | ||
98 | return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) | ||
99 | } | 71 | } |
100 | 72 | ||
101 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { | 73 | async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { |
102 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) | 74 | logger.info('Creating job to undo cache file %s.', redundancyModel.url) |
103 | 75 | ||
104 | const undoUrl = getUndoActivityPubUrl(redundancyModel.url) | ||
105 | |||
106 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 76 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) |
107 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) | ||
108 | |||
109 | const audience = getVideoAudience(video, actorsInvolvedInVideo) | ||
110 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) | 77 | const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) |
111 | 78 | ||
112 | const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience) | 79 | return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) |
113 | |||
114 | return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
115 | } | 80 | } |
116 | 81 | ||
117 | // --------------------------------------------------------------------------- | 82 | // --------------------------------------------------------------------------- |
@@ -144,3 +109,19 @@ function undoActivityData ( | |||
144 | audience | 109 | audience |
145 | ) | 110 | ) |
146 | } | 111 | } |
112 | |||
113 | async function sendUndoVideoRelatedActivity (options: { | ||
114 | byActor: ActorModel, | ||
115 | video: VideoModel, | ||
116 | url: string, | ||
117 | activity: ActivityFollow | ActivityLike | ActivityCreate | ActivityAnnounce, | ||
118 | transaction: Transaction | ||
119 | }) { | ||
120 | const activityBuilder = (audience: ActivityAudience) => { | ||
121 | const undoUrl = getUndoActivityPubUrl(options.url) | ||
122 | |||
123 | return undoActivityData(undoUrl, options.byActor, options.activity, audience) | ||
124 | } | ||
125 | |||
126 | return sendVideoRelatedActivity(activityBuilder, options) | ||
127 | } | ||
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 605473338..ec46789b7 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -7,8 +7,8 @@ import { VideoModel } from '../../../models/video/video' | |||
7 | import { VideoChannelModel } from '../../../models/video/video-channel' | 7 | import { VideoChannelModel } from '../../../models/video/video-channel' |
8 | import { VideoShareModel } from '../../../models/video/video-share' | 8 | import { VideoShareModel } from '../../../models/video/video-share' |
9 | import { getUpdateActivityPubUrl } from '../url' | 9 | import { getUpdateActivityPubUrl } from '../url' |
10 | import { broadcastToFollowers, unicastTo } from './utils' | 10 | import { broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils' |
11 | import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' | 11 | import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf } from '../audience' |
12 | import { logger } from '../../../helpers/logger' | 12 | import { logger } from '../../../helpers/logger' |
13 | import { VideoCaptionModel } from '../../../models/video/video-caption' | 13 | import { VideoCaptionModel } from '../../../models/video/video-caption' |
14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' | 14 | import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' |
@@ -61,16 +61,16 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod | |||
61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { | 61 | async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { |
62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) | 62 | logger.info('Creating job to update cache file %s.', redundancyModel.url) |
63 | 63 | ||
64 | const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) | ||
65 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) | 64 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) |
66 | 65 | ||
67 | const redundancyObject = redundancyModel.toActivityPubObject() | 66 | const activityBuilder = (audience: ActivityAudience) => { |
67 | const redundancyObject = redundancyModel.toActivityPubObject() | ||
68 | const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) | ||
68 | 69 | ||
69 | const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) | 70 | return buildUpdateActivity(url, byActor, redundancyObject, audience) |
70 | const audience = getObjectFollowersAudience(accountsInvolvedInVideo) | 71 | } |
71 | 72 | ||
72 | const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience) | 73 | return sendVideoRelatedActivity(activityBuilder, { byActor, video }) |
73 | return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
74 | } | 74 | } |
75 | 75 | ||
76 | // --------------------------------------------------------------------------- | 76 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index c20c15633..69706e620 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts | |||
@@ -1,13 +1,36 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { Activity } from '../../../../shared/models/activitypub' | 2 | import { Activity, ActivityAudience } from '../../../../shared/models/activitypub' |
3 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
4 | import { ActorModel } from '../../../models/activitypub/actor' | 4 | import { ActorModel } from '../../../models/activitypub/actor' |
5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 5 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
6 | import { JobQueue } from '../../job-queue' | 6 | import { JobQueue } from '../../job-queue' |
7 | import { VideoModel } from '../../../models/video/video' | 7 | import { VideoModel } from '../../../models/video/video' |
8 | import { getActorsInvolvedInVideo } from '../audience' | 8 | import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience' |
9 | import { getServerActor } from '../../../helpers/utils' | 9 | import { getServerActor } from '../../../helpers/utils' |
10 | 10 | ||
11 | async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { | ||
12 | byActor: ActorModel, | ||
13 | video: VideoModel, | ||
14 | transaction?: Transaction | ||
15 | }) { | ||
16 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(options.video, options.transaction) | ||
17 | |||
18 | // Send to origin | ||
19 | if (options.video.isOwned() === false) { | ||
20 | const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo) | ||
21 | const activity = activityBuilder(audience) | ||
22 | |||
23 | return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl) | ||
24 | } | ||
25 | |||
26 | // Send to followers | ||
27 | const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) | ||
28 | const activity = activityBuilder(audience) | ||
29 | |||
30 | const actorsException = [ options.byActor ] | ||
31 | return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException) | ||
32 | } | ||
33 | |||
11 | async function forwardVideoRelatedActivity ( | 34 | async function forwardVideoRelatedActivity ( |
12 | activity: Activity, | 35 | activity: Activity, |
13 | t: Transaction, | 36 | t: Transaction, |
@@ -110,7 +133,8 @@ export { | |||
110 | unicastTo, | 133 | unicastTo, |
111 | forwardActivity, | 134 | forwardActivity, |
112 | broadcastToActors, | 135 | broadcastToActors, |
113 | forwardVideoRelatedActivity | 136 | forwardVideoRelatedActivity, |
137 | sendVideoRelatedActivity | ||
114 | } | 138 | } |
115 | 139 | ||
116 | // --------------------------------------------------------------------------- | 140 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index ffbd3a64e..4ca8bf659 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { | |||
94 | try { | 94 | try { |
95 | // Maybe it's a reply to a video? | 95 | // Maybe it's a reply to a video? |
96 | // If yes, it's done: we resolved all the thread | 96 | // If yes, it's done: we resolved all the thread |
97 | const { video } = await getOrCreateVideoAndAccountAndChannel(url) | 97 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url }) |
98 | 98 | ||
99 | if (comments.length !== 0) { | 99 | if (comments.length !== 0) { |
100 | const firstReply = comments[ comments.length - 1 ] | 100 | const firstReply = comments[ comments.length - 1 ] |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 783f78d3e..48c0e0a5c 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize' | |||
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import { join } from 'path' | 4 | import { join } from 'path' |
5 | import * as request from 'request' | 5 | import * as request from 'request' |
6 | import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' | 6 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' |
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy } from '../../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub | |||
28 | import { createRates } from './video-rates' | 28 | import { createRates } from './video-rates' |
29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | 29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' |
30 | import { AccountModel } from '../../models/account/account' | 30 | import { AccountModel } from '../../models/account/account' |
31 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | ||
31 | 32 | ||
32 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 33 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
33 | // If the video is not private and published, we federate it | 34 | // If the video is not private and published, we federate it |
@@ -50,18 +51,29 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr | |||
50 | } | 51 | } |
51 | } | 52 | } |
52 | 53 | ||
53 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | 54 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { |
54 | const host = video.VideoChannel.Account.Actor.Server.host | 55 | const options = { |
56 | uri: videoUrl, | ||
57 | method: 'GET', | ||
58 | json: true, | ||
59 | activityPub: true | ||
60 | } | ||
55 | 61 | ||
56 | // We need to provide a callback, if no we could have an uncaught exception | 62 | logger.info('Fetching remote video %s.', videoUrl) |
57 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | 63 | |
58 | if (err) reject(err) | 64 | const { response, body } = await doRequest(options) |
59 | }) | 65 | |
66 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | ||
67 | logger.debug('Remote video JSON is not valid.', { body }) | ||
68 | return { response, videoObject: undefined } | ||
69 | } | ||
70 | |||
71 | return { response, videoObject: body } | ||
60 | } | 72 | } |
61 | 73 | ||
62 | async function fetchRemoteVideoDescription (video: VideoModel) { | 74 | async function fetchRemoteVideoDescription (video: VideoModel) { |
63 | const host = video.VideoChannel.Account.Actor.Server.host | 75 | const host = video.VideoChannel.Account.Actor.Server.host |
64 | const path = video.getDescriptionPath() | 76 | const path = video.getDescriptionAPIPath() |
65 | const options = { | 77 | const options = { |
66 | uri: REMOTE_SCHEME.HTTP + '://' + host + path, | 78 | uri: REMOTE_SCHEME.HTTP + '://' + host + path, |
67 | json: true | 79 | json: true |
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) { | |||
71 | return body.description ? body.description : '' | 83 | return body.description ? body.description : '' |
72 | } | 84 | } |
73 | 85 | ||
86 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | ||
87 | const host = video.VideoChannel.Account.Actor.Server.host | ||
88 | |||
89 | // We need to provide a callback, if no we could have an uncaught exception | ||
90 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | ||
91 | if (err) reject(err) | ||
92 | }) | ||
93 | } | ||
94 | |||
74 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { | 95 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { |
75 | const thumbnailName = video.getThumbnailName() | 96 | const thumbnailName = video.getThumbnailName() |
76 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) | 97 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) |
@@ -82,144 +103,11 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) | |||
82 | return doRequestAndSaveToFile(options, thumbnailPath) | 103 | return doRequestAndSaveToFile(options, thumbnailPath) |
83 | } | 104 | } |
84 | 105 | ||
85 | async function videoActivityObjectToDBAttributes ( | ||
86 | videoChannel: VideoChannelModel, | ||
87 | videoObject: VideoTorrentObject, | ||
88 | to: string[] = [] | ||
89 | ) { | ||
90 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED | ||
91 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
92 | |||
93 | let language: string | undefined | ||
94 | if (videoObject.language) { | ||
95 | language = videoObject.language.identifier | ||
96 | } | ||
97 | |||
98 | let category: number | undefined | ||
99 | if (videoObject.category) { | ||
100 | category = parseInt(videoObject.category.identifier, 10) | ||
101 | } | ||
102 | |||
103 | let licence: number | undefined | ||
104 | if (videoObject.licence) { | ||
105 | licence = parseInt(videoObject.licence.identifier, 10) | ||
106 | } | ||
107 | |||
108 | const description = videoObject.content || null | ||
109 | const support = videoObject.support || null | ||
110 | |||
111 | return { | ||
112 | name: videoObject.name, | ||
113 | uuid: videoObject.uuid, | ||
114 | url: videoObject.id, | ||
115 | category, | ||
116 | licence, | ||
117 | language, | ||
118 | description, | ||
119 | support, | ||
120 | nsfw: videoObject.sensitive, | ||
121 | commentsEnabled: videoObject.commentsEnabled, | ||
122 | waitTranscoding: videoObject.waitTranscoding, | ||
123 | state: videoObject.state, | ||
124 | channelId: videoChannel.id, | ||
125 | duration: parseInt(duration, 10), | ||
126 | createdAt: new Date(videoObject.published), | ||
127 | publishedAt: new Date(videoObject.published), | ||
128 | // FIXME: updatedAt does not seems to be considered by Sequelize | ||
129 | updatedAt: new Date(videoObject.updated), | ||
130 | views: videoObject.views, | ||
131 | likes: 0, | ||
132 | dislikes: 0, | ||
133 | remote: true, | ||
134 | privacy | ||
135 | } | ||
136 | } | ||
137 | |||
138 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | ||
139 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
140 | |||
141 | if (fileUrls.length === 0) { | ||
142 | throw new Error('Cannot find video files for ' + videoCreated.url) | ||
143 | } | ||
144 | |||
145 | const attributes: VideoFileModel[] = [] | ||
146 | for (const fileUrl of fileUrls) { | ||
147 | // Fetch associated magnet uri | ||
148 | const magnet = videoObject.url.find(u => { | ||
149 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | ||
150 | }) | ||
151 | |||
152 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
153 | |||
154 | const parsed = magnetUtil.decode(magnet.href) | ||
155 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
156 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
157 | } | ||
158 | |||
159 | const attribute = { | ||
160 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | ||
161 | infoHash: parsed.infoHash, | ||
162 | resolution: fileUrl.height, | ||
163 | size: fileUrl.size, | ||
164 | videoId: videoCreated.id, | ||
165 | fps: fileUrl.fps | ||
166 | } as VideoFileModel | ||
167 | attributes.push(attribute) | ||
168 | } | ||
169 | |||
170 | return attributes | ||
171 | } | ||
172 | |||
173 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { | 106 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { |
174 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | 107 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') |
175 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | 108 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) |
176 | 109 | ||
177 | return getOrCreateActorAndServerAndModel(channel.id) | 110 | return getOrCreateActorAndServerAndModel(channel.id, 'all') |
178 | } | ||
179 | |||
180 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | ||
181 | logger.debug('Adding remote video %s.', videoObject.id) | ||
182 | |||
183 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
184 | const sequelizeOptions = { transaction: t } | ||
185 | |||
186 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
187 | const video = VideoModel.build(videoData) | ||
188 | |||
189 | const videoCreated = await video.save(sequelizeOptions) | ||
190 | |||
191 | // Process files | ||
192 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | ||
193 | if (videoFileAttributes.length === 0) { | ||
194 | throw new Error('Cannot find valid files for video %s ' + videoObject.url) | ||
195 | } | ||
196 | |||
197 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
198 | await Promise.all(videoFilePromises) | ||
199 | |||
200 | // Process tags | ||
201 | const tags = videoObject.tag.map(t => t.name) | ||
202 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
203 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
204 | |||
205 | // Process captions | ||
206 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
207 | return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) | ||
208 | }) | ||
209 | await Promise.all(videoCaptionsPromises) | ||
210 | |||
211 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
212 | |||
213 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
214 | return videoCreated | ||
215 | }) | ||
216 | |||
217 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | ||
218 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
219 | |||
220 | if (waitThumbnail === true) await p | ||
221 | |||
222 | return videoCreated | ||
223 | } | 111 | } |
224 | 112 | ||
225 | type SyncParam = { | 113 | type SyncParam = { |
@@ -230,28 +118,7 @@ type SyncParam = { | |||
230 | thumbnail: boolean | 118 | thumbnail: boolean |
231 | refreshVideo: boolean | 119 | refreshVideo: boolean |
232 | } | 120 | } |
233 | async function getOrCreateVideoAndAccountAndChannel ( | 121 | async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { |
234 | videoObject: VideoTorrentObject | string, | ||
235 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
236 | ) { | ||
237 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id | ||
238 | |||
239 | let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) | ||
240 | if (videoFromDatabase) { | ||
241 | const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) | ||
242 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
243 | |||
244 | return { video: videoFromDatabase } | ||
245 | } | ||
246 | |||
247 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | ||
248 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
249 | |||
250 | const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) | ||
251 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) | ||
252 | |||
253 | // Process outside the transaction because we could fetch remote data | ||
254 | |||
255 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | 122 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) |
256 | 123 | ||
257 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] | 124 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] |
@@ -285,64 +152,56 @@ async function getOrCreateVideoAndAccountAndChannel ( | |||
285 | } | 152 | } |
286 | 153 | ||
287 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) | 154 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) |
288 | |||
289 | return { video } | ||
290 | } | 155 | } |
291 | 156 | ||
292 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { | 157 | async function getOrCreateVideoAndAccountAndChannel (options: { |
293 | const options = { | 158 | videoObject: VideoTorrentObject | string, |
294 | uri: videoUrl, | 159 | syncParam?: SyncParam, |
295 | method: 'GET', | 160 | fetchType?: VideoFetchByUrlType, |
296 | json: true, | 161 | refreshViews?: boolean |
297 | activityPub: true | 162 | }) { |
298 | } | 163 | // Default params |
299 | 164 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | |
300 | logger.info('Fetching remote video %s.', videoUrl) | 165 | const fetchType = options.fetchType || 'all' |
301 | 166 | const refreshViews = options.refreshViews || false | |
302 | const { response, body } = await doRequest(options) | 167 | |
168 | // Get video url | ||
169 | const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id | ||
170 | |||
171 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
172 | if (videoFromDatabase) { | ||
173 | const refreshOptions = { | ||
174 | video: videoFromDatabase, | ||
175 | fetchedType: fetchType, | ||
176 | syncParam, | ||
177 | refreshViews | ||
178 | } | ||
179 | const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) | ||
180 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
303 | 181 | ||
304 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | 182 | return { video: videoFromDatabase } |
305 | logger.debug('Remote video JSON is not valid.', { body }) | ||
306 | return { response, videoObject: undefined } | ||
307 | } | 183 | } |
308 | 184 | ||
309 | return { response, videoObject: body } | 185 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) |
310 | } | 186 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) |
311 | |||
312 | async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { | ||
313 | if (!video.isOutdated()) return video | ||
314 | |||
315 | try { | ||
316 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
317 | if (response.statusCode === 404) { | ||
318 | // Video does not exist anymore | ||
319 | await video.destroy() | ||
320 | return undefined | ||
321 | } | ||
322 | 187 | ||
323 | if (videoObject === undefined) { | 188 | const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) |
324 | logger.warn('Cannot refresh remote video: invalid body.') | 189 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) |
325 | return video | ||
326 | } | ||
327 | 190 | ||
328 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 191 | await syncVideoExternalAttributes(video, fetchedVideo, syncParam) |
329 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
330 | 192 | ||
331 | return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) | 193 | return { video } |
332 | } catch (err) { | ||
333 | logger.warn('Cannot refresh video.', { err }) | ||
334 | return video | ||
335 | } | ||
336 | } | 194 | } |
337 | 195 | ||
338 | async function updateVideoFromAP ( | 196 | async function updateVideoFromAP (options: { |
339 | video: VideoModel, | 197 | video: VideoModel, |
340 | videoObject: VideoTorrentObject, | 198 | videoObject: VideoTorrentObject, |
341 | account: AccountModel, | 199 | account: AccountModel, |
342 | channel: VideoChannelModel, | 200 | channel: VideoChannelModel, |
201 | updateViews: boolean, | ||
343 | overrideTo?: string[] | 202 | overrideTo?: string[] |
344 | ) { | 203 | }) { |
345 | logger.debug('Updating remote video "%s".', videoObject.uuid) | 204 | logger.debug('Updating remote video "%s".', options.videoObject.uuid) |
346 | let videoFieldsSave: any | 205 | let videoFieldsSave: any |
347 | 206 | ||
348 | try { | 207 | try { |
@@ -351,72 +210,72 @@ async function updateVideoFromAP ( | |||
351 | transaction: t | 210 | transaction: t |
352 | } | 211 | } |
353 | 212 | ||
354 | videoFieldsSave = video.toJSON() | 213 | videoFieldsSave = options.video.toJSON() |
355 | 214 | ||
356 | // Check actor has the right to update the video | 215 | // Check actor has the right to update the video |
357 | const videoChannel = video.VideoChannel | 216 | const videoChannel = options.video.VideoChannel |
358 | if (videoChannel.Account.id !== account.id) { | 217 | if (videoChannel.Account.id !== options.account.id) { |
359 | throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) | 218 | throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) |
360 | } | 219 | } |
361 | 220 | ||
362 | const to = overrideTo ? overrideTo : videoObject.to | 221 | const to = options.overrideTo ? options.overrideTo : options.videoObject.to |
363 | const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) | 222 | const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to) |
364 | video.set('name', videoData.name) | 223 | options.video.set('name', videoData.name) |
365 | video.set('uuid', videoData.uuid) | 224 | options.video.set('uuid', videoData.uuid) |
366 | video.set('url', videoData.url) | 225 | options.video.set('url', videoData.url) |
367 | video.set('category', videoData.category) | 226 | options.video.set('category', videoData.category) |
368 | video.set('licence', videoData.licence) | 227 | options.video.set('licence', videoData.licence) |
369 | video.set('language', videoData.language) | 228 | options.video.set('language', videoData.language) |
370 | video.set('description', videoData.description) | 229 | options.video.set('description', videoData.description) |
371 | video.set('support', videoData.support) | 230 | options.video.set('support', videoData.support) |
372 | video.set('nsfw', videoData.nsfw) | 231 | options.video.set('nsfw', videoData.nsfw) |
373 | video.set('commentsEnabled', videoData.commentsEnabled) | 232 | options.video.set('commentsEnabled', videoData.commentsEnabled) |
374 | video.set('waitTranscoding', videoData.waitTranscoding) | 233 | options.video.set('waitTranscoding', videoData.waitTranscoding) |
375 | video.set('state', videoData.state) | 234 | options.video.set('state', videoData.state) |
376 | video.set('duration', videoData.duration) | 235 | options.video.set('duration', videoData.duration) |
377 | video.set('createdAt', videoData.createdAt) | 236 | options.video.set('createdAt', videoData.createdAt) |
378 | video.set('publishedAt', videoData.publishedAt) | 237 | options.video.set('publishedAt', videoData.publishedAt) |
379 | video.set('views', videoData.views) | 238 | options.video.set('privacy', videoData.privacy) |
380 | video.set('privacy', videoData.privacy) | 239 | options.video.set('channelId', videoData.channelId) |
381 | video.set('channelId', videoData.channelId) | 240 | |
382 | 241 | if (options.updateViews === true) options.video.set('views', videoData.views) | |
383 | await video.save(sequelizeOptions) | 242 | await options.video.save(sequelizeOptions) |
384 | 243 | ||
385 | // Don't block on request | 244 | // Don't block on request |
386 | generateThumbnailFromUrl(video, videoObject.icon) | 245 | generateThumbnailFromUrl(options.video, options.videoObject.icon) |
387 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | 246 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) |
388 | 247 | ||
389 | // Remove old video files | 248 | // Remove old video files |
390 | const videoFileDestroyTasks: Bluebird<void>[] = [] | 249 | const videoFileDestroyTasks: Bluebird<void>[] = [] |
391 | for (const videoFile of video.VideoFiles) { | 250 | for (const videoFile of options.video.VideoFiles) { |
392 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) | 251 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) |
393 | } | 252 | } |
394 | await Promise.all(videoFileDestroyTasks) | 253 | await Promise.all(videoFileDestroyTasks) |
395 | 254 | ||
396 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) | 255 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) |
397 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) | 256 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) |
398 | await Promise.all(tasks) | 257 | await Promise.all(tasks) |
399 | 258 | ||
400 | // Update Tags | 259 | // Update Tags |
401 | const tags = videoObject.tag.map(tag => tag.name) | 260 | const tags = options.videoObject.tag.map(tag => tag.name) |
402 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 261 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
403 | await video.$set('Tags', tagInstances, sequelizeOptions) | 262 | await options.video.$set('Tags', tagInstances, sequelizeOptions) |
404 | 263 | ||
405 | // Update captions | 264 | // Update captions |
406 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) | 265 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) |
407 | 266 | ||
408 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | 267 | const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { |
409 | return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) | 268 | return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) |
410 | }) | 269 | }) |
411 | await Promise.all(videoCaptionsPromises) | 270 | await Promise.all(videoCaptionsPromises) |
412 | }) | 271 | }) |
413 | 272 | ||
414 | logger.info('Remote video with uuid %s updated', videoObject.uuid) | 273 | logger.info('Remote video with uuid %s updated', options.videoObject.uuid) |
415 | 274 | ||
416 | return updatedVideo | 275 | return updatedVideo |
417 | } catch (err) { | 276 | } catch (err) { |
418 | if (video !== undefined && videoFieldsSave !== undefined) { | 277 | if (options.video !== undefined && videoFieldsSave !== undefined) { |
419 | resetSequelizeInstance(video, videoFieldsSave) | 278 | resetSequelizeInstance(options.video, videoFieldsSave) |
420 | } | 279 | } |
421 | 280 | ||
422 | // This is just a debug because we will retry the insert | 281 | // This is just a debug because we will retry the insert |
@@ -433,12 +292,7 @@ export { | |||
433 | fetchRemoteVideoStaticFile, | 292 | fetchRemoteVideoStaticFile, |
434 | fetchRemoteVideoDescription, | 293 | fetchRemoteVideoDescription, |
435 | generateThumbnailFromUrl, | 294 | generateThumbnailFromUrl, |
436 | videoActivityObjectToDBAttributes, | 295 | getOrCreateVideoChannelFromVideoObject |
437 | videoFileActivityUrlToDBAttributes, | ||
438 | createVideo, | ||
439 | getOrCreateVideoChannelFromVideoObject, | ||
440 | addVideoShares, | ||
441 | createRates | ||
442 | } | 296 | } |
443 | 297 | ||
444 | // --------------------------------------------------------------------------- | 298 | // --------------------------------------------------------------------------- |
@@ -448,3 +302,178 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo | |||
448 | 302 | ||
449 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') | 303 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') |
450 | } | 304 | } |
305 | |||
306 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | ||
307 | logger.debug('Adding remote video %s.', videoObject.id) | ||
308 | |||
309 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
310 | const sequelizeOptions = { transaction: t } | ||
311 | |||
312 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
313 | const video = VideoModel.build(videoData) | ||
314 | |||
315 | const videoCreated = await video.save(sequelizeOptions) | ||
316 | |||
317 | // Process files | ||
318 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | ||
319 | if (videoFileAttributes.length === 0) { | ||
320 | throw new Error('Cannot find valid files for video %s ' + videoObject.url) | ||
321 | } | ||
322 | |||
323 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
324 | await Promise.all(videoFilePromises) | ||
325 | |||
326 | // Process tags | ||
327 | const tags = videoObject.tag.map(t => t.name) | ||
328 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
329 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
330 | |||
331 | // Process captions | ||
332 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
333 | return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) | ||
334 | }) | ||
335 | await Promise.all(videoCaptionsPromises) | ||
336 | |||
337 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
338 | |||
339 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
340 | return videoCreated | ||
341 | }) | ||
342 | |||
343 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | ||
344 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
345 | |||
346 | if (waitThumbnail === true) await p | ||
347 | |||
348 | return videoCreated | ||
349 | } | ||
350 | |||
351 | async function refreshVideoIfNeeded (options: { | ||
352 | video: VideoModel, | ||
353 | fetchedType: VideoFetchByUrlType, | ||
354 | syncParam: SyncParam, | ||
355 | refreshViews: boolean | ||
356 | }): Promise<VideoModel> { | ||
357 | if (!options.video.isOutdated()) return options.video | ||
358 | |||
359 | // We need more attributes if the argument video was fetched with not enough joints | ||
360 | const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
361 | |||
362 | try { | ||
363 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
364 | if (response.statusCode === 404) { | ||
365 | // Video does not exist anymore | ||
366 | await video.destroy() | ||
367 | return undefined | ||
368 | } | ||
369 | |||
370 | if (videoObject === undefined) { | ||
371 | logger.warn('Cannot refresh remote video: invalid body.') | ||
372 | return video | ||
373 | } | ||
374 | |||
375 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
376 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
377 | |||
378 | const updateOptions = { | ||
379 | video, | ||
380 | videoObject, | ||
381 | account, | ||
382 | channel: channelActor.VideoChannel, | ||
383 | updateViews: options.refreshViews | ||
384 | } | ||
385 | await updateVideoFromAP(updateOptions) | ||
386 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
387 | } catch (err) { | ||
388 | logger.warn('Cannot refresh video.', { err }) | ||
389 | return video | ||
390 | } | ||
391 | } | ||
392 | |||
393 | async function videoActivityObjectToDBAttributes ( | ||
394 | videoChannel: VideoChannelModel, | ||
395 | videoObject: VideoTorrentObject, | ||
396 | to: string[] = [] | ||
397 | ) { | ||
398 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED | ||
399 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
400 | |||
401 | let language: string | undefined | ||
402 | if (videoObject.language) { | ||
403 | language = videoObject.language.identifier | ||
404 | } | ||
405 | |||
406 | let category: number | undefined | ||
407 | if (videoObject.category) { | ||
408 | category = parseInt(videoObject.category.identifier, 10) | ||
409 | } | ||
410 | |||
411 | let licence: number | undefined | ||
412 | if (videoObject.licence) { | ||
413 | licence = parseInt(videoObject.licence.identifier, 10) | ||
414 | } | ||
415 | |||
416 | const description = videoObject.content || null | ||
417 | const support = videoObject.support || null | ||
418 | |||
419 | return { | ||
420 | name: videoObject.name, | ||
421 | uuid: videoObject.uuid, | ||
422 | url: videoObject.id, | ||
423 | category, | ||
424 | licence, | ||
425 | language, | ||
426 | description, | ||
427 | support, | ||
428 | nsfw: videoObject.sensitive, | ||
429 | commentsEnabled: videoObject.commentsEnabled, | ||
430 | waitTranscoding: videoObject.waitTranscoding, | ||
431 | state: videoObject.state, | ||
432 | channelId: videoChannel.id, | ||
433 | duration: parseInt(duration, 10), | ||
434 | createdAt: new Date(videoObject.published), | ||
435 | publishedAt: new Date(videoObject.published), | ||
436 | // FIXME: updatedAt does not seems to be considered by Sequelize | ||
437 | updatedAt: new Date(videoObject.updated), | ||
438 | views: videoObject.views, | ||
439 | likes: 0, | ||
440 | dislikes: 0, | ||
441 | remote: true, | ||
442 | privacy | ||
443 | } | ||
444 | } | ||
445 | |||
446 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | ||
447 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
448 | |||
449 | if (fileUrls.length === 0) { | ||
450 | throw new Error('Cannot find video files for ' + videoCreated.url) | ||
451 | } | ||
452 | |||
453 | const attributes: VideoFileModel[] = [] | ||
454 | for (const fileUrl of fileUrls) { | ||
455 | // Fetch associated magnet uri | ||
456 | const magnet = videoObject.url.find(u => { | ||
457 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | ||
458 | }) | ||
459 | |||
460 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
461 | |||
462 | const parsed = magnetUtil.decode(magnet.href) | ||
463 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
464 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
465 | } | ||
466 | |||
467 | const attribute = { | ||
468 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | ||
469 | infoHash: parsed.infoHash, | ||
470 | resolution: fileUrl.height, | ||
471 | size: fileUrl.size, | ||
472 | videoId: videoCreated.id, | ||
473 | fps: fileUrl.fps | ||
474 | } as VideoFileModel | ||
475 | attributes.push(attribute) | ||
476 | } | ||
477 | |||
478 | return attributes | ||
479 | } | ||
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 5cfb81fc7..14f0a05f5 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts | |||
@@ -3,23 +3,18 @@ import { sendUpdateActor } from './activitypub/send' | |||
3 | import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' | 3 | import { AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../initializers' |
4 | import { updateActorAvatarInstance } from './activitypub' | 4 | import { updateActorAvatarInstance } from './activitypub' |
5 | import { processImage } from '../helpers/image-utils' | 5 | import { processImage } from '../helpers/image-utils' |
6 | import { ActorModel } from '../models/activitypub/actor' | ||
7 | import { AccountModel } from '../models/account/account' | 6 | import { AccountModel } from '../models/account/account' |
8 | import { VideoChannelModel } from '../models/video/video-channel' | 7 | import { VideoChannelModel } from '../models/video/video-channel' |
9 | import { extname, join } from 'path' | 8 | import { extname, join } from 'path' |
10 | 9 | ||
11 | async function updateActorAvatarFile ( | 10 | async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { |
12 | avatarPhysicalFile: Express.Multer.File, | ||
13 | actor: ActorModel, | ||
14 | accountOrChannel: AccountModel | VideoChannelModel | ||
15 | ) { | ||
16 | const extension = extname(avatarPhysicalFile.filename) | 11 | const extension = extname(avatarPhysicalFile.filename) |
17 | const avatarName = actor.uuid + extension | 12 | const avatarName = accountOrChannel.Actor.uuid + extension |
18 | const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) | 13 | const destination = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) |
19 | await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) | 14 | await processImage(avatarPhysicalFile, destination, AVATARS_SIZE) |
20 | 15 | ||
21 | return sequelizeTypescript.transaction(async t => { | 16 | return sequelizeTypescript.transaction(async t => { |
22 | const updatedActor = await updateActorAvatarInstance(actor, avatarName, t) | 17 | const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t) |
23 | await updatedActor.save({ transaction: t }) | 18 | await updatedActor.save({ transaction: t }) |
24 | 19 | ||
25 | await sendUpdateActor(accountOrChannel, t) | 20 | await sendUpdateActor(accountOrChannel, t) |
diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/cache/videos-caption-cache.ts index 380d42b2c..f240affbc 100644 --- a/server/lib/cache/videos-caption-cache.ts +++ b/server/lib/cache/videos-caption-cache.ts | |||
@@ -38,7 +38,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> { | |||
38 | if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') | 38 | if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') |
39 | 39 | ||
40 | // Used to fetch the path | 40 | // Used to fetch the path |
41 | const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) | 41 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) |
42 | if (!video) return undefined | 42 | if (!video) return undefined |
43 | 43 | ||
44 | const remoteStaticPath = videoCaption.getCaptionStaticPath() | 44 | const remoteStaticPath = videoCaption.getCaptionStaticPath() |
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts index 22b6d9cb0..a5d6f5b62 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/cache/videos-preview-cache.ts | |||
@@ -16,7 +16,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | |||
16 | } | 16 | } |
17 | 17 | ||
18 | async getFilePath (videoUUID: string) { | 18 | async getFilePath (videoUUID: string) { |
19 | const video = await VideoModel.loadByUUID(videoUUID) | 19 | const video = await VideoModel.loadByUUIDWithFile(videoUUID) |
20 | if (!video) return undefined | 20 | if (!video) return undefined |
21 | 21 | ||
22 | if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) | 22 | if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) |
@@ -25,7 +25,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | |||
25 | } | 25 | } |
26 | 26 | ||
27 | protected async loadRemoteFile (key: string) { | 27 | protected async loadRemoteFile (key: string) { |
28 | const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(key) | 28 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key) |
29 | if (!video) return undefined | 29 | if (!video) return undefined |
30 | 30 | ||
31 | if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') | 31 | if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') |
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index a69e09c32..fc013e0c3 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -8,6 +8,7 @@ import { VideoModel } from '../models/video/video' | |||
8 | import * as validator from 'validator' | 8 | import * as validator from 'validator' |
9 | import { VideoPrivacy } from '../../shared/models/videos' | 9 | import { VideoPrivacy } from '../../shared/models/videos' |
10 | import { readFile } from 'fs-extra' | 10 | import { readFile } from 'fs-extra' |
11 | import { getActivityStreamDuration } from '../models/video/video-format-utils' | ||
11 | 12 | ||
12 | export class ClientHtml { | 13 | export class ClientHtml { |
13 | 14 | ||
@@ -38,10 +39,8 @@ export class ClientHtml { | |||
38 | let videoPromise: Bluebird<VideoModel> | 39 | let videoPromise: Bluebird<VideoModel> |
39 | 40 | ||
40 | // Let Angular application handle errors | 41 | // Let Angular application handle errors |
41 | if (validator.isUUID(videoId, 4)) { | 42 | if (validator.isInt(videoId) || validator.isUUID(videoId, 4)) { |
42 | videoPromise = VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(videoId) | 43 | videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) |
43 | } else if (validator.isInt(videoId)) { | ||
44 | videoPromise = VideoModel.loadAndPopulateAccountAndServerAndTags(+videoId) | ||
45 | } else { | 44 | } else { |
46 | return ClientHtml.getIndexHTML(req, res) | 45 | return ClientHtml.getIndexHTML(req, res) |
47 | } | 46 | } |
@@ -150,7 +149,7 @@ export class ClientHtml { | |||
150 | description: videoDescriptionEscaped, | 149 | description: videoDescriptionEscaped, |
151 | thumbnailUrl: previewUrl, | 150 | thumbnailUrl: previewUrl, |
152 | uploadDate: video.createdAt.toISOString(), | 151 | uploadDate: video.createdAt.toISOString(), |
153 | duration: video.getActivityStreamDuration(), | 152 | duration: getActivityStreamDuration(video.duration), |
154 | contentUrl: videoUrl, | 153 | contentUrl: videoUrl, |
155 | embedUrl: embedUrl, | 154 | embedUrl: embedUrl, |
156 | interactionCount: video.views | 155 | interactionCount: video.views |
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 72d670277..42217c27c 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts | |||
@@ -1,10 +1,10 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { processActivities } from '../../activitypub/process' | 3 | import { processActivities } from '../../activitypub/process' |
4 | import { VideoModel } from '../../../models/video/video' | ||
5 | import { addVideoShares, createRates } from '../../activitypub/videos' | ||
6 | import { addVideoComments } from '../../activitypub/video-comments' | 4 | import { addVideoComments } from '../../activitypub/video-comments' |
7 | import { crawlCollectionPage } from '../../activitypub/crawl' | 5 | import { crawlCollectionPage } from '../../activitypub/crawl' |
6 | import { VideoModel } from '../../../models/video/video' | ||
7 | import { addVideoShares, createRates } from '../../activitypub' | ||
8 | 8 | ||
9 | type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 9 | type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' |
10 | 10 | ||
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts index c6308f7a6..1463c93fc 100644 --- a/server/lib/job-queue/handlers/video-file.ts +++ b/server/lib/job-queue/handlers/video-file.ts | |||
@@ -8,6 +8,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' | |||
8 | import { sequelizeTypescript } from '../../../initializers' | 8 | import { sequelizeTypescript } from '../../../initializers' |
9 | import * as Bluebird from 'bluebird' | 9 | import * as Bluebird from 'bluebird' |
10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' | 10 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' |
11 | import { importVideoFile, transcodeOriginalVideofile, optimizeOriginalVideofile } from '../../video-transcoding' | ||
11 | 12 | ||
12 | export type VideoFilePayload = { | 13 | export type VideoFilePayload = { |
13 | videoUUID: string | 14 | videoUUID: string |
@@ -25,14 +26,14 @@ async function processVideoFileImport (job: Bull.Job) { | |||
25 | const payload = job.data as VideoFileImportPayload | 26 | const payload = job.data as VideoFileImportPayload |
26 | logger.info('Processing video file import in job %d.', job.id) | 27 | logger.info('Processing video file import in job %d.', job.id) |
27 | 28 | ||
28 | const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) | 29 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) |
29 | // No video, maybe deleted? | 30 | // No video, maybe deleted? |
30 | if (!video) { | 31 | if (!video) { |
31 | logger.info('Do not process job %d, video does not exist.', job.id) | 32 | logger.info('Do not process job %d, video does not exist.', job.id) |
32 | return undefined | 33 | return undefined |
33 | } | 34 | } |
34 | 35 | ||
35 | await video.importVideoFile(payload.filePath) | 36 | await importVideoFile(video, payload.filePath) |
36 | 37 | ||
37 | await onVideoFileTranscoderOrImportSuccess(video) | 38 | await onVideoFileTranscoderOrImportSuccess(video) |
38 | return video | 39 | return video |
@@ -42,7 +43,7 @@ async function processVideoFile (job: Bull.Job) { | |||
42 | const payload = job.data as VideoFilePayload | 43 | const payload = job.data as VideoFilePayload |
43 | logger.info('Processing video file in job %d.', job.id) | 44 | logger.info('Processing video file in job %d.', job.id) |
44 | 45 | ||
45 | const video = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(payload.videoUUID) | 46 | const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoUUID) |
46 | // No video, maybe deleted? | 47 | // No video, maybe deleted? |
47 | if (!video) { | 48 | if (!video) { |
48 | logger.info('Do not process job %d, video does not exist.', job.id) | 49 | logger.info('Do not process job %d, video does not exist.', job.id) |
@@ -51,11 +52,11 @@ async function processVideoFile (job: Bull.Job) { | |||
51 | 52 | ||
52 | // Transcoding in other resolution | 53 | // Transcoding in other resolution |
53 | if (payload.resolution) { | 54 | if (payload.resolution) { |
54 | await video.transcodeOriginalVideofile(payload.resolution, payload.isPortraitMode || false) | 55 | await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) |
55 | 56 | ||
56 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) | 57 | await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) |
57 | } else { | 58 | } else { |
58 | await video.optimizeOriginalVideofile() | 59 | await optimizeOriginalVideofile(video) |
59 | 60 | ||
60 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) | 61 | await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) |
61 | } | 62 | } |
@@ -68,7 +69,7 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { | |||
68 | 69 | ||
69 | return sequelizeTypescript.transaction(async t => { | 70 | return sequelizeTypescript.transaction(async t => { |
70 | // Maybe the video changed in database, refresh it | 71 | // Maybe the video changed in database, refresh it |
71 | let videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) | 72 | let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
72 | // Video does not exist anymore | 73 | // Video does not exist anymore |
73 | if (!videoDatabase) return undefined | 74 | if (!videoDatabase) return undefined |
74 | 75 | ||
@@ -98,7 +99,7 @@ async function onVideoFileOptimizerSuccess (video: VideoModel, isNewVideo: boole | |||
98 | 99 | ||
99 | return sequelizeTypescript.transaction(async t => { | 100 | return sequelizeTypescript.transaction(async t => { |
100 | // Maybe the video changed in database, refresh it | 101 | // Maybe the video changed in database, refresh it |
101 | const videoDatabase = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) | 102 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
102 | // Video does not exist anymore | 103 | // Video does not exist anymore |
103 | if (!videoDatabase) return undefined | 104 | if (!videoDatabase) return undefined |
104 | 105 | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index ebcb2090c..9e14e57e6 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -183,7 +183,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide | |||
183 | const videoUpdated = await video.save({ transaction: t }) | 183 | const videoUpdated = await video.save({ transaction: t }) |
184 | 184 | ||
185 | // Now we can federate the video (reload from database, we need more attributes) | 185 | // Now we can federate the video (reload from database, we need more attributes) |
186 | const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) | 186 | const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) |
187 | await federateVideoIfNeeded(videoForFederation, true, t) | 187 | await federateVideoIfNeeded(videoForFederation, true, t) |
188 | 188 | ||
189 | // Update video import object | 189 | // Update video import object |
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 2f8667e19..5cbe60b82 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts | |||
@@ -4,15 +4,50 @@ import { UserModel } from '../models/account/user' | |||
4 | import { OAuthClientModel } from '../models/oauth/oauth-client' | 4 | import { OAuthClientModel } from '../models/oauth/oauth-client' |
5 | import { OAuthTokenModel } from '../models/oauth/oauth-token' | 5 | import { OAuthTokenModel } from '../models/oauth/oauth-token' |
6 | import { CONFIG } from '../initializers/constants' | 6 | import { CONFIG } from '../initializers/constants' |
7 | import { Transaction } from 'sequelize' | ||
7 | 8 | ||
8 | type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } | 9 | type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } |
10 | const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} | ||
11 | const userHavingToken: { [ userId: number ]: string } = {} | ||
9 | 12 | ||
10 | // --------------------------------------------------------------------------- | 13 | // --------------------------------------------------------------------------- |
11 | 14 | ||
15 | function deleteUserToken (userId: number, t?: Transaction) { | ||
16 | clearCacheByUserId(userId) | ||
17 | |||
18 | return OAuthTokenModel.deleteUserToken(userId, t) | ||
19 | } | ||
20 | |||
21 | function clearCacheByUserId (userId: number) { | ||
22 | const token = userHavingToken[userId] | ||
23 | if (token !== undefined) { | ||
24 | accessTokenCache[ token ] = undefined | ||
25 | userHavingToken[ userId ] = undefined | ||
26 | } | ||
27 | } | ||
28 | |||
29 | function clearCacheByToken (token: string) { | ||
30 | const tokenModel = accessTokenCache[ token ] | ||
31 | if (tokenModel !== undefined) { | ||
32 | userHavingToken[tokenModel.userId] = undefined | ||
33 | accessTokenCache[ token ] = undefined | ||
34 | } | ||
35 | } | ||
36 | |||
12 | function getAccessToken (bearerToken: string) { | 37 | function getAccessToken (bearerToken: string) { |
13 | logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') | 38 | logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') |
14 | 39 | ||
40 | if (accessTokenCache[bearerToken] !== undefined) return accessTokenCache[bearerToken] | ||
41 | |||
15 | return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) | 42 | return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) |
43 | .then(tokenModel => { | ||
44 | if (tokenModel) { | ||
45 | accessTokenCache[ bearerToken ] = tokenModel | ||
46 | userHavingToken[ tokenModel.userId ] = tokenModel.accessToken | ||
47 | } | ||
48 | |||
49 | return tokenModel | ||
50 | }) | ||
16 | } | 51 | } |
17 | 52 | ||
18 | function getClient (clientId: string, clientSecret: string) { | 53 | function getClient (clientId: string, clientSecret: string) { |
@@ -48,6 +83,8 @@ async function getUser (usernameOrEmail: string, password: string) { | |||
48 | async function revokeToken (tokenInfo: TokenInfo) { | 83 | async function revokeToken (tokenInfo: TokenInfo) { |
49 | const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) | 84 | const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) |
50 | if (token) { | 85 | if (token) { |
86 | clearCacheByToken(token.accessToken) | ||
87 | |||
51 | token.destroy() | 88 | token.destroy() |
52 | .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) | 89 | .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) |
53 | } | 90 | } |
@@ -85,6 +122,9 @@ async function saveToken (token: TokenInfo, client: OAuthClientModel, user: User | |||
85 | 122 | ||
86 | // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications | 123 | // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications |
87 | export { | 124 | export { |
125 | deleteUserToken, | ||
126 | clearCacheByUserId, | ||
127 | clearCacheByToken, | ||
88 | getAccessToken, | 128 | getAccessToken, |
89 | getClient, | 129 | getClient, |
90 | getRefreshToken, | 130 | getRefreshToken, |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index ee9ba1766..960651712 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,10 +1,9 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' | 2 | import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' | 4 | import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | import { VideoFileModel } from '../../models/video/video-file' | 6 | import { VideoFileModel } from '../../models/video/video-file' |
7 | import { sortBy } from 'lodash' | ||
8 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' | 7 | import { downloadWebTorrentVideo } from '../../helpers/webtorrent' |
9 | import { join } from 'path' | 8 | import { join } from 'path' |
10 | import { rename } from 'fs-extra' | 9 | import { rename } from 'fs-extra' |
@@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils' | |||
12 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' | 11 | import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' |
13 | import { VideoModel } from '../../models/video/video' | 12 | import { VideoModel } from '../../models/video/video' |
14 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' | 13 | import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' |
15 | import { removeVideoRedundancy } from '../redundancy' | ||
16 | import { isTestInstance } from '../../helpers/core-utils' | 14 | import { isTestInstance } from '../../helpers/core-utils' |
17 | 15 | ||
18 | export class VideosRedundancyScheduler extends AbstractScheduler { | 16 | export class VideosRedundancyScheduler extends AbstractScheduler { |
@@ -20,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
20 | private static instance: AbstractScheduler | 18 | private static instance: AbstractScheduler |
21 | private executing = false | 19 | private executing = false |
22 | 20 | ||
23 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy | 21 | protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL |
24 | 22 | ||
25 | private constructor () { | 23 | private constructor () { |
26 | super() | 24 | super() |
@@ -31,17 +29,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
31 | 29 | ||
32 | this.executing = true | 30 | this.executing = true |
33 | 31 | ||
34 | for (const obj of CONFIG.REDUNDANCY.VIDEOS) { | 32 | for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
35 | |||
36 | try { | 33 | try { |
37 | const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) | 34 | const videoToDuplicate = await this.findVideoToDuplicate(obj) |
38 | if (!videoToDuplicate) continue | 35 | if (!videoToDuplicate) continue |
39 | 36 | ||
40 | const videoFiles = videoToDuplicate.VideoFiles | 37 | const videoFiles = videoToDuplicate.VideoFiles |
41 | videoFiles.forEach(f => f.Video = videoToDuplicate) | 38 | videoFiles.forEach(f => f.Video = videoToDuplicate) |
42 | 39 | ||
43 | const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) | 40 | if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) { |
44 | if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) { | ||
45 | if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) | 41 | if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) |
46 | continue | 42 | continue |
47 | } | 43 | } |
@@ -54,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
54 | } | 50 | } |
55 | } | 51 | } |
56 | 52 | ||
53 | await this.removeExpired() | ||
54 | |||
55 | this.executing = false | ||
56 | } | ||
57 | |||
58 | static get Instance () { | ||
59 | return this.instance || (this.instance = new this()) | ||
60 | } | ||
61 | |||
62 | private async removeExpired () { | ||
57 | const expired = await VideoRedundancyModel.listAllExpired() | 63 | const expired = await VideoRedundancyModel.listAllExpired() |
58 | 64 | ||
59 | for (const m of expired) { | 65 | for (const m of expired) { |
@@ -65,16 +71,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
65 | logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) | 71 | logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) |
66 | } | 72 | } |
67 | } | 73 | } |
68 | |||
69 | this.executing = false | ||
70 | } | 74 | } |
71 | 75 | ||
72 | static get Instance () { | 76 | private findVideoToDuplicate (cache: VideosRedundancy) { |
73 | return this.instance || (this.instance = new this()) | 77 | if (cache.strategy === 'most-views') { |
74 | } | 78 | return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) |
79 | } | ||
80 | |||
81 | if (cache.strategy === 'trending') { | ||
82 | return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | ||
83 | } | ||
75 | 84 | ||
76 | private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { | 85 | if (cache.strategy === 'recently-added') { |
77 | if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | 86 | const minViews = cache.minViews |
87 | return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) | ||
88 | } | ||
78 | } | 89 | } |
79 | 90 | ||
80 | private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { | 91 | private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { |
@@ -120,27 +131,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
120 | } | 131 | } |
121 | } | 132 | } |
122 | 133 | ||
123 | // Unused, but could be useful in the future, with a custom strategy | 134 | private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) { |
124 | private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) { | ||
125 | const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt') | ||
126 | |||
127 | while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) { | ||
128 | const toDelete = sortedVideosRedundancy.shift() | ||
129 | |||
130 | const videoFile = toDelete.VideoFile | ||
131 | logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution) | ||
132 | |||
133 | await removeVideoRedundancy(toDelete, undefined) | ||
134 | } | ||
135 | |||
136 | return sortedVideosRedundancy | ||
137 | } | ||
138 | |||
139 | private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) { | ||
140 | const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) | 135 | const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) |
141 | 136 | ||
142 | const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size | 137 | const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy) |
143 | const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0) | ||
144 | 138 | ||
145 | return totalDuplicated > maxSize | 139 | return totalDuplicated > maxSize |
146 | } | 140 | } |
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index faadb4334..461cd045e 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts | |||
@@ -1,13 +1,6 @@ | |||
1 | // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js | ||
2 | // We rewrote it to avoid sync calls | ||
3 | |||
4 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' | 2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' |
6 | import { logger } from '../../helpers/logger' | 3 | import { updateYoutubeDLBinary } from '../../helpers/youtube-dl' |
7 | import * as request from 'request' | ||
8 | import { createWriteStream, ensureDir, writeFile } from 'fs-extra' | ||
9 | import { join } from 'path' | ||
10 | import { root } from '../../helpers/core-utils' | ||
11 | 4 | ||
12 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { | 5 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { |
13 | 6 | ||
@@ -19,60 +12,8 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler { | |||
19 | super() | 12 | super() |
20 | } | 13 | } |
21 | 14 | ||
22 | async execute () { | 15 | execute () { |
23 | logger.info('Updating youtubeDL binary.') | 16 | return updateYoutubeDLBinary() |
24 | |||
25 | const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') | ||
26 | const bin = join(binDirectory, 'youtube-dl') | ||
27 | const detailsPath = join(binDirectory, 'details') | ||
28 | const url = 'https://yt-dl.org/downloads/latest/youtube-dl' | ||
29 | |||
30 | await ensureDir(binDirectory) | ||
31 | |||
32 | return new Promise(res => { | ||
33 | request.get(url, { followRedirect: false }, (err, result) => { | ||
34 | if (err) { | ||
35 | logger.error('Cannot update youtube-dl.', { err }) | ||
36 | return res() | ||
37 | } | ||
38 | |||
39 | if (result.statusCode !== 302) { | ||
40 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) | ||
41 | return res() | ||
42 | } | ||
43 | |||
44 | const url = result.headers.location | ||
45 | const downloadFile = request.get(url) | ||
46 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] | ||
47 | |||
48 | downloadFile.on('response', result => { | ||
49 | if (result.statusCode !== 200) { | ||
50 | logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) | ||
51 | return res() | ||
52 | } | ||
53 | |||
54 | downloadFile.pipe(createWriteStream(bin, { mode: 493 })) | ||
55 | }) | ||
56 | |||
57 | downloadFile.on('error', err => { | ||
58 | logger.error('youtube-dl update error.', { err }) | ||
59 | return res() | ||
60 | }) | ||
61 | |||
62 | downloadFile.on('end', () => { | ||
63 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) | ||
64 | writeFile(detailsPath, details, { encoding: 'utf8' }, err => { | ||
65 | if (err) { | ||
66 | logger.error('youtube-dl update error: cannot write details.', { err }) | ||
67 | return res() | ||
68 | } | ||
69 | |||
70 | logger.info('youtube-dl updated to version %s.', newVersion) | ||
71 | return res() | ||
72 | }) | ||
73 | }) | ||
74 | }) | ||
75 | }) | ||
76 | } | 17 | } |
77 | 18 | ||
78 | static get Instance () { | 19 | static get Instance () { |
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts new file mode 100644 index 000000000..bf3ff78c2 --- /dev/null +++ b/server/lib/video-transcoding.ts | |||
@@ -0,0 +1,130 @@ | |||
1 | import { CONFIG } from '../initializers' | ||
2 | import { join, extname } from 'path' | ||
3 | import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' | ||
4 | import { copy, remove, rename, stat } from 'fs-extra' | ||
5 | import { logger } from '../helpers/logger' | ||
6 | import { VideoResolution } from '../../shared/models/videos' | ||
7 | import { VideoFileModel } from '../models/video/video-file' | ||
8 | import { VideoModel } from '../models/video/video' | ||
9 | |||
10 | async function optimizeOriginalVideofile (video: VideoModel) { | ||
11 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | ||
12 | const newExtname = '.mp4' | ||
13 | const inputVideoFile = video.getOriginalFile() | ||
14 | const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) | ||
15 | const videoTranscodedPath = join(videosDirectory, video.id + '-transcoded' + newExtname) | ||
16 | |||
17 | const transcodeOptions = { | ||
18 | inputPath: videoInputPath, | ||
19 | outputPath: videoTranscodedPath | ||
20 | } | ||
21 | |||
22 | // Could be very long! | ||
23 | await transcode(transcodeOptions) | ||
24 | |||
25 | try { | ||
26 | await remove(videoInputPath) | ||
27 | |||
28 | // Important to do this before getVideoFilename() to take in account the new file extension | ||
29 | inputVideoFile.set('extname', newExtname) | ||
30 | |||
31 | const videoOutputPath = video.getVideoFilePath(inputVideoFile) | ||
32 | await rename(videoTranscodedPath, videoOutputPath) | ||
33 | const stats = await stat(videoOutputPath) | ||
34 | const fps = await getVideoFileFPS(videoOutputPath) | ||
35 | |||
36 | inputVideoFile.set('size', stats.size) | ||
37 | inputVideoFile.set('fps', fps) | ||
38 | |||
39 | await video.createTorrentAndSetInfoHash(inputVideoFile) | ||
40 | await inputVideoFile.save() | ||
41 | } catch (err) { | ||
42 | // Auto destruction... | ||
43 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) | ||
44 | |||
45 | throw err | ||
46 | } | ||
47 | } | ||
48 | |||
49 | async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { | ||
50 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | ||
51 | const extname = '.mp4' | ||
52 | |||
53 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed | ||
54 | const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) | ||
55 | |||
56 | const newVideoFile = new VideoFileModel({ | ||
57 | resolution, | ||
58 | extname, | ||
59 | size: 0, | ||
60 | videoId: video.id | ||
61 | }) | ||
62 | const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) | ||
63 | |||
64 | const transcodeOptions = { | ||
65 | inputPath: videoInputPath, | ||
66 | outputPath: videoOutputPath, | ||
67 | resolution, | ||
68 | isPortraitMode | ||
69 | } | ||
70 | |||
71 | await transcode(transcodeOptions) | ||
72 | |||
73 | const stats = await stat(videoOutputPath) | ||
74 | const fps = await getVideoFileFPS(videoOutputPath) | ||
75 | |||
76 | newVideoFile.set('size', stats.size) | ||
77 | newVideoFile.set('fps', fps) | ||
78 | |||
79 | await video.createTorrentAndSetInfoHash(newVideoFile) | ||
80 | |||
81 | await newVideoFile.save() | ||
82 | |||
83 | video.VideoFiles.push(newVideoFile) | ||
84 | } | ||
85 | |||
86 | async function importVideoFile (video: VideoModel, inputFilePath: string) { | ||
87 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | ||
88 | const { size } = await stat(inputFilePath) | ||
89 | const fps = await getVideoFileFPS(inputFilePath) | ||
90 | |||
91 | let updatedVideoFile = new VideoFileModel({ | ||
92 | resolution: videoFileResolution, | ||
93 | extname: extname(inputFilePath), | ||
94 | size, | ||
95 | fps, | ||
96 | videoId: video.id | ||
97 | }) | ||
98 | |||
99 | const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) | ||
100 | |||
101 | if (currentVideoFile) { | ||
102 | // Remove old file and old torrent | ||
103 | await video.removeFile(currentVideoFile) | ||
104 | await video.removeTorrent(currentVideoFile) | ||
105 | // Remove the old video file from the array | ||
106 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) | ||
107 | |||
108 | // Update the database | ||
109 | currentVideoFile.set('extname', updatedVideoFile.extname) | ||
110 | currentVideoFile.set('size', updatedVideoFile.size) | ||
111 | currentVideoFile.set('fps', updatedVideoFile.fps) | ||
112 | |||
113 | updatedVideoFile = currentVideoFile | ||
114 | } | ||
115 | |||
116 | const outputPath = video.getVideoFilePath(updatedVideoFile) | ||
117 | await copy(inputFilePath, outputPath) | ||
118 | |||
119 | await video.createTorrentAndSetInfoHash(updatedVideoFile) | ||
120 | |||
121 | await updatedVideoFile.save() | ||
122 | |||
123 | video.VideoFiles.push(updatedVideoFile) | ||
124 | } | ||
125 | |||
126 | export { | ||
127 | optimizeOriginalVideofile, | ||
128 | transcodeOriginalVideofile, | ||
129 | importVideoFile | ||
130 | } | ||
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index d13c50c84..d3ba1ae23 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts | |||
@@ -172,7 +172,7 @@ const usersVideoRatingValidator = [ | |||
172 | logger.debug('Checking usersVideoRating parameters', { parameters: req.params }) | 172 | logger.debug('Checking usersVideoRating parameters', { parameters: req.params }) |
173 | 173 | ||
174 | if (areValidationErrors(req, res)) return | 174 | if (areValidationErrors(req, res)) return |
175 | if (!await isVideoExist(req.params.videoId, res)) return | 175 | if (!await isVideoExist(req.params.videoId, res, 'id')) return |
176 | 176 | ||
177 | return next() | 177 | return next() |
178 | } | 178 | } |
diff --git a/server/middlewares/validators/video-captions.ts b/server/middlewares/validators/video-captions.ts index 4f393ea84..51ffd7f3c 100644 --- a/server/middlewares/validators/video-captions.ts +++ b/server/middlewares/validators/video-captions.ts | |||
@@ -58,7 +58,7 @@ const listVideoCaptionsValidator = [ | |||
58 | logger.debug('Checking listVideoCaptions parameters', { parameters: req.params }) | 58 | logger.debug('Checking listVideoCaptions parameters', { parameters: req.params }) |
59 | 59 | ||
60 | if (areValidationErrors(req, res)) return | 60 | if (areValidationErrors(req, res)) return |
61 | if (!await isVideoExist(req.params.videoId, res)) return | 61 | if (!await isVideoExist(req.params.videoId, res, 'id')) return |
62 | 62 | ||
63 | return next() | 63 | return next() |
64 | } | 64 | } |
diff --git a/server/middlewares/validators/video-comments.ts b/server/middlewares/validators/video-comments.ts index 227bc1fca..693852499 100644 --- a/server/middlewares/validators/video-comments.ts +++ b/server/middlewares/validators/video-comments.ts | |||
@@ -17,7 +17,7 @@ const listVideoCommentThreadsValidator = [ | |||
17 | logger.debug('Checking listVideoCommentThreads parameters.', { parameters: req.params }) | 17 | logger.debug('Checking listVideoCommentThreads parameters.', { parameters: req.params }) |
18 | 18 | ||
19 | if (areValidationErrors(req, res)) return | 19 | if (areValidationErrors(req, res)) return |
20 | if (!await isVideoExist(req.params.videoId, res)) return | 20 | if (!await isVideoExist(req.params.videoId, res, 'only-video')) return |
21 | 21 | ||
22 | return next() | 22 | return next() |
23 | } | 23 | } |
@@ -31,7 +31,7 @@ const listVideoThreadCommentsValidator = [ | |||
31 | logger.debug('Checking listVideoThreadComments parameters.', { parameters: req.params }) | 31 | logger.debug('Checking listVideoThreadComments parameters.', { parameters: req.params }) |
32 | 32 | ||
33 | if (areValidationErrors(req, res)) return | 33 | if (areValidationErrors(req, res)) return |
34 | if (!await isVideoExist(req.params.videoId, res)) return | 34 | if (!await isVideoExist(req.params.videoId, res, 'only-video')) return |
35 | if (!await isVideoCommentThreadExist(req.params.threadId, res.locals.video, res)) return | 35 | if (!await isVideoCommentThreadExist(req.params.threadId, res.locals.video, res)) return |
36 | 36 | ||
37 | return next() | 37 | return next() |
@@ -78,7 +78,7 @@ const videoCommentGetValidator = [ | |||
78 | logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) | 78 | logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) |
79 | 79 | ||
80 | if (areValidationErrors(req, res)) return | 80 | if (areValidationErrors(req, res)) return |
81 | if (!await isVideoExist(req.params.videoId, res)) return | 81 | if (!await isVideoExist(req.params.videoId, res, 'id')) return |
82 | if (!await isVideoCommentExist(req.params.commentId, res.locals.video, res)) return | 82 | if (!await isVideoCommentExist(req.params.commentId, res.locals.video, res)) return |
83 | 83 | ||
84 | return next() | 84 | return next() |
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 9befbc9ee..67eabe468 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts | |||
@@ -41,6 +41,7 @@ import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } f | |||
41 | import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model' | 41 | import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model' |
42 | import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership' | 42 | import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership' |
43 | import { AccountModel } from '../../models/account/account' | 43 | import { AccountModel } from '../../models/account/account' |
44 | import { VideoFetchType } from '../../helpers/video' | ||
44 | 45 | ||
45 | const videosAddValidator = getCommonVideoAttributes().concat([ | 46 | const videosAddValidator = getCommonVideoAttributes().concat([ |
46 | body('videofile') | 47 | body('videofile') |
@@ -128,47 +129,49 @@ const videosUpdateValidator = getCommonVideoAttributes().concat([ | |||
128 | } | 129 | } |
129 | ]) | 130 | ]) |
130 | 131 | ||
131 | const videosGetValidator = [ | 132 | const videosCustomGetValidator = (fetchType: VideoFetchType) => { |
132 | param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), | 133 | return [ |
133 | 134 | param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), | |
134 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
135 | logger.debug('Checking videosGet parameters', { parameters: req.params }) | ||
136 | 135 | ||
137 | if (areValidationErrors(req, res)) return | 136 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
138 | if (!await isVideoExist(req.params.id, res)) return | 137 | logger.debug('Checking videosGet parameters', { parameters: req.params }) |
139 | 138 | ||
140 | const video: VideoModel = res.locals.video | 139 | if (areValidationErrors(req, res)) return |
140 | if (!await isVideoExist(req.params.id, res, fetchType)) return | ||
141 | 141 | ||
142 | // Video private or blacklisted | 142 | const video: VideoModel = res.locals.video |
143 | if (video.privacy === VideoPrivacy.PRIVATE || video.VideoBlacklist) { | ||
144 | return authenticate(req, res, () => { | ||
145 | const user: UserModel = res.locals.oauth.token.User | ||
146 | 143 | ||
147 | // Only the owner or a user that have blacklist rights can see the video | 144 | // Video private or blacklisted |
148 | if (video.VideoChannel.Account.userId !== user.id && !user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) { | 145 | if (video.privacy === VideoPrivacy.PRIVATE || video.VideoBlacklist) { |
149 | return res.status(403) | 146 | return authenticate(req, res, () => { |
150 | .json({ error: 'Cannot get this private or blacklisted video.' }) | 147 | const user: UserModel = res.locals.oauth.token.User |
151 | .end() | ||
152 | } | ||
153 | 148 | ||
154 | return next() | 149 | // Only the owner or a user that have blacklist rights can see the video |
155 | }) | 150 | if (video.VideoChannel.Account.userId !== user.id && !user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) { |
151 | return res.status(403) | ||
152 | .json({ error: 'Cannot get this private or blacklisted video.' }) | ||
153 | .end() | ||
154 | } | ||
156 | 155 | ||
157 | return | 156 | return next() |
158 | } | 157 | }) |
158 | } | ||
159 | 159 | ||
160 | // Video is public, anyone can access it | 160 | // Video is public, anyone can access it |
161 | if (video.privacy === VideoPrivacy.PUBLIC) return next() | 161 | if (video.privacy === VideoPrivacy.PUBLIC) return next() |
162 | 162 | ||
163 | // Video is unlisted, check we used the uuid to fetch it | 163 | // Video is unlisted, check we used the uuid to fetch it |
164 | if (video.privacy === VideoPrivacy.UNLISTED) { | 164 | if (video.privacy === VideoPrivacy.UNLISTED) { |
165 | if (isUUIDValid(req.params.id)) return next() | 165 | if (isUUIDValid(req.params.id)) return next() |
166 | 166 | ||
167 | // Don't leak this unlisted video | 167 | // Don't leak this unlisted video |
168 | return res.status(404).end() | 168 | return res.status(404).end() |
169 | } | ||
169 | } | 170 | } |
170 | } | 171 | ] |
171 | ] | 172 | } |
173 | |||
174 | const videosGetValidator = videosCustomGetValidator('all') | ||
172 | 175 | ||
173 | const videosRemoveValidator = [ | 176 | const videosRemoveValidator = [ |
174 | param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), | 177 | param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), |
@@ -366,6 +369,7 @@ export { | |||
366 | videosAddValidator, | 369 | videosAddValidator, |
367 | videosUpdateValidator, | 370 | videosUpdateValidator, |
368 | videosGetValidator, | 371 | videosGetValidator, |
372 | videosCustomGetValidator, | ||
369 | videosRemoveValidator, | 373 | videosRemoveValidator, |
370 | videosShareValidator, | 374 | videosShareValidator, |
371 | 375 | ||
diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 6bbfc6f4e..580d920ce 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts | |||
@@ -134,8 +134,8 @@ export class AccountModel extends Model<AccountModel> { | |||
134 | return undefined | 134 | return undefined |
135 | } | 135 | } |
136 | 136 | ||
137 | static load (id: number) { | 137 | static load (id: number, transaction?: Sequelize.Transaction) { |
138 | return AccountModel.findById(id) | 138 | return AccountModel.findById(id, { transaction }) |
139 | } | 139 | } |
140 | 140 | ||
141 | static loadByUUID (uuid: string) { | 141 | static loadByUUID (uuid: string) { |
diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 680b1d52d..e56b0bf40 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts | |||
@@ -1,5 +1,7 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import { | 2 | import { |
3 | AfterDelete, | ||
4 | AfterUpdate, | ||
3 | AllowNull, | 5 | AllowNull, |
4 | BeforeCreate, | 6 | BeforeCreate, |
5 | BeforeUpdate, | 7 | BeforeUpdate, |
@@ -39,6 +41,7 @@ import { AccountModel } from './account' | |||
39 | import { NSFWPolicyType } from '../../../shared/models/videos/nsfw-policy.type' | 41 | import { NSFWPolicyType } from '../../../shared/models/videos/nsfw-policy.type' |
40 | import { values } from 'lodash' | 42 | import { values } from 'lodash' |
41 | import { NSFW_POLICY_TYPES } from '../../initializers' | 43 | import { NSFW_POLICY_TYPES } from '../../initializers' |
44 | import { clearCacheByUserId } from '../../lib/oauth-model' | ||
42 | 45 | ||
43 | enum ScopeNames { | 46 | enum ScopeNames { |
44 | WITH_VIDEO_CHANNEL = 'WITH_VIDEO_CHANNEL' | 47 | WITH_VIDEO_CHANNEL = 'WITH_VIDEO_CHANNEL' |
@@ -168,6 +171,12 @@ export class UserModel extends Model<UserModel> { | |||
168 | } | 171 | } |
169 | } | 172 | } |
170 | 173 | ||
174 | @AfterUpdate | ||
175 | @AfterDelete | ||
176 | static removeTokenCache (instance: UserModel) { | ||
177 | return clearCacheByUserId(instance.id) | ||
178 | } | ||
179 | |||
171 | static countTotal () { | 180 | static countTotal () { |
172 | return this.count() | 181 | return this.count() |
173 | } | 182 | } |
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index ef8dd9f7c..f8bb59323 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts | |||
@@ -266,6 +266,18 @@ export class ActorModel extends Model<ActorModel> { | |||
266 | return ActorModel.unscoped().findById(id) | 266 | return ActorModel.unscoped().findById(id) |
267 | } | 267 | } |
268 | 268 | ||
269 | static isActorUrlExist (url: string) { | ||
270 | const query = { | ||
271 | raw: true, | ||
272 | where: { | ||
273 | url | ||
274 | } | ||
275 | } | ||
276 | |||
277 | return ActorModel.unscoped().findOne(query) | ||
278 | .then(a => !!a) | ||
279 | } | ||
280 | |||
269 | static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { | 281 | static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { |
270 | const query = { | 282 | const query = { |
271 | where: { | 283 | where: { |
@@ -315,6 +327,29 @@ export class ActorModel extends Model<ActorModel> { | |||
315 | where: { | 327 | where: { |
316 | url | 328 | url |
317 | }, | 329 | }, |
330 | transaction, | ||
331 | include: [ | ||
332 | { | ||
333 | attributes: [ 'id' ], | ||
334 | model: AccountModel.unscoped(), | ||
335 | required: false | ||
336 | }, | ||
337 | { | ||
338 | attributes: [ 'id' ], | ||
339 | model: VideoChannelModel.unscoped(), | ||
340 | required: false | ||
341 | } | ||
342 | ] | ||
343 | } | ||
344 | |||
345 | return ActorModel.unscoped().findOne(query) | ||
346 | } | ||
347 | |||
348 | static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Sequelize.Transaction) { | ||
349 | const query = { | ||
350 | where: { | ||
351 | url | ||
352 | }, | ||
318 | transaction | 353 | transaction |
319 | } | 354 | } |
320 | 355 | ||
diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts index 4c53848dc..ef9592c04 100644 --- a/server/models/oauth/oauth-token.ts +++ b/server/models/oauth/oauth-token.ts | |||
@@ -1,9 +1,23 @@ | |||
1 | import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' | 1 | import { |
2 | AfterDelete, | ||
3 | AfterUpdate, | ||
4 | AllowNull, | ||
5 | BelongsTo, | ||
6 | Column, | ||
7 | CreatedAt, | ||
8 | ForeignKey, | ||
9 | Model, | ||
10 | Scopes, | ||
11 | Table, | ||
12 | UpdatedAt | ||
13 | } from 'sequelize-typescript' | ||
2 | import { logger } from '../../helpers/logger' | 14 | import { logger } from '../../helpers/logger' |
3 | import { AccountModel } from '../account/account' | ||
4 | import { UserModel } from '../account/user' | 15 | import { UserModel } from '../account/user' |
5 | import { OAuthClientModel } from './oauth-client' | 16 | import { OAuthClientModel } from './oauth-client' |
6 | import { Transaction } from 'sequelize' | 17 | import { Transaction } from 'sequelize' |
18 | import { AccountModel } from '../account/account' | ||
19 | import { ActorModel } from '../activitypub/actor' | ||
20 | import { clearCacheByToken } from '../../lib/oauth-model' | ||
7 | 21 | ||
8 | export type OAuthTokenInfo = { | 22 | export type OAuthTokenInfo = { |
9 | refreshToken: string | 23 | refreshToken: string |
@@ -17,18 +31,27 @@ export type OAuthTokenInfo = { | |||
17 | } | 31 | } |
18 | 32 | ||
19 | enum ScopeNames { | 33 | enum ScopeNames { |
20 | WITH_ACCOUNT = 'WITH_ACCOUNT' | 34 | WITH_USER = 'WITH_USER' |
21 | } | 35 | } |
22 | 36 | ||
23 | @Scopes({ | 37 | @Scopes({ |
24 | [ScopeNames.WITH_ACCOUNT]: { | 38 | [ScopeNames.WITH_USER]: { |
25 | include: [ | 39 | include: [ |
26 | { | 40 | { |
27 | model: () => UserModel, | 41 | model: () => UserModel.unscoped(), |
42 | required: true, | ||
28 | include: [ | 43 | include: [ |
29 | { | 44 | { |
30 | model: () => AccountModel, | 45 | attributes: [ 'id' ], |
31 | required: true | 46 | model: () => AccountModel.unscoped(), |
47 | required: true, | ||
48 | include: [ | ||
49 | { | ||
50 | attributes: [ 'id' ], | ||
51 | model: () => ActorModel.unscoped(), | ||
52 | required: true | ||
53 | } | ||
54 | ] | ||
32 | } | 55 | } |
33 | ] | 56 | ] |
34 | } | 57 | } |
@@ -102,6 +125,12 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> { | |||
102 | }) | 125 | }) |
103 | OAuthClients: OAuthClientModel[] | 126 | OAuthClients: OAuthClientModel[] |
104 | 127 | ||
128 | @AfterUpdate | ||
129 | @AfterDelete | ||
130 | static removeTokenCache (token: OAuthTokenModel) { | ||
131 | return clearCacheByToken(token.accessToken) | ||
132 | } | ||
133 | |||
105 | static getByRefreshTokenAndPopulateClient (refreshToken: string) { | 134 | static getByRefreshTokenAndPopulateClient (refreshToken: string) { |
106 | const query = { | 135 | const query = { |
107 | where: { | 136 | where: { |
@@ -138,7 +167,7 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> { | |||
138 | } | 167 | } |
139 | } | 168 | } |
140 | 169 | ||
141 | return OAuthTokenModel.scope(ScopeNames.WITH_ACCOUNT).findOne(query).then(token => { | 170 | return OAuthTokenModel.scope(ScopeNames.WITH_USER).findOne(query).then(token => { |
142 | if (token) token['user'] = token.User | 171 | if (token) token['user'] = token.User |
143 | 172 | ||
144 | return token | 173 | return token |
@@ -152,7 +181,7 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> { | |||
152 | } | 181 | } |
153 | } | 182 | } |
154 | 183 | ||
155 | return OAuthTokenModel.scope(ScopeNames.WITH_ACCOUNT) | 184 | return OAuthTokenModel.scope(ScopeNames.WITH_USER) |
156 | .findOne(query) | 185 | .findOne(query) |
157 | .then(token => { | 186 | .then(token => { |
158 | if (token) { | 187 | if (token) { |
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 48ec77206..fb07287a8 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts | |||
@@ -14,11 +14,10 @@ import { | |||
14 | UpdatedAt | 14 | UpdatedAt |
15 | } from 'sequelize-typescript' | 15 | } from 'sequelize-typescript' |
16 | import { ActorModel } from '../activitypub/actor' | 16 | import { ActorModel } from '../activitypub/actor' |
17 | import { throwIfNotValid } from '../utils' | 17 | import { getVideoSort, throwIfNotValid } from '../utils' |
18 | import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 18 | import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
19 | import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' | 19 | import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' |
20 | import { VideoFileModel } from '../video/video-file' | 20 | import { VideoFileModel } from '../video/video-file' |
21 | import { isDateValid } from '../../helpers/custom-validators/misc' | ||
22 | import { getServerActor } from '../../helpers/utils' | 21 | import { getServerActor } from '../../helpers/utils' |
23 | import { VideoModel } from '../video/video' | 22 | import { VideoModel } from '../video/video' |
24 | import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' | 23 | import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' |
@@ -28,6 +27,7 @@ import { VideoChannelModel } from '../video/video-channel' | |||
28 | import { ServerModel } from '../server/server' | 27 | import { ServerModel } from '../server/server' |
29 | import { sample } from 'lodash' | 28 | import { sample } from 'lodash' |
30 | import { isTestInstance } from '../../helpers/core-utils' | 29 | import { isTestInstance } from '../../helpers/core-utils' |
30 | import * as Bluebird from 'bluebird' | ||
31 | 31 | ||
32 | export enum ScopeNames { | 32 | export enum ScopeNames { |
33 | WITH_VIDEO = 'WITH_VIDEO' | 33 | WITH_VIDEO = 'WITH_VIDEO' |
@@ -145,65 +145,90 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
145 | return VideoRedundancyModel.findOne(query) | 145 | return VideoRedundancyModel.findOne(query) |
146 | } | 146 | } |
147 | 147 | ||
148 | static async getVideoSample (p: Bluebird<VideoModel[]>) { | ||
149 | const rows = await p | ||
150 | const ids = rows.map(r => r.id) | ||
151 | const id = sample(ids) | ||
152 | |||
153 | return VideoModel.loadWithFile(id, undefined, !isTestInstance()) | ||
154 | } | ||
155 | |||
148 | static async findMostViewToDuplicate (randomizedFactor: number) { | 156 | static async findMostViewToDuplicate (randomizedFactor: number) { |
149 | // On VideoModel! | 157 | // On VideoModel! |
150 | const query = { | 158 | const query = { |
159 | attributes: [ 'id', 'views' ], | ||
151 | logging: !isTestInstance(), | 160 | logging: !isTestInstance(), |
152 | limit: randomizedFactor, | 161 | limit: randomizedFactor, |
153 | order: [ [ 'views', 'DESC' ] ], | 162 | order: getVideoSort('-views'), |
154 | include: [ | 163 | include: [ |
155 | { | 164 | await VideoRedundancyModel.buildVideoFileForDuplication(), |
156 | model: VideoFileModel.unscoped(), | 165 | VideoRedundancyModel.buildServerRedundancyInclude() |
157 | required: true, | ||
158 | where: { | ||
159 | id: { | ||
160 | [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn() | ||
161 | } | ||
162 | } | ||
163 | }, | ||
164 | { | ||
165 | attributes: [], | ||
166 | model: VideoChannelModel.unscoped(), | ||
167 | required: true, | ||
168 | include: [ | ||
169 | { | ||
170 | attributes: [], | ||
171 | model: ActorModel.unscoped(), | ||
172 | required: true, | ||
173 | include: [ | ||
174 | { | ||
175 | attributes: [], | ||
176 | model: ServerModel.unscoped(), | ||
177 | required: true, | ||
178 | where: { | ||
179 | redundancyAllowed: true | ||
180 | } | ||
181 | } | ||
182 | ] | ||
183 | } | ||
184 | ] | ||
185 | } | ||
186 | ] | 166 | ] |
187 | } | 167 | } |
188 | 168 | ||
189 | const rows = await VideoModel.unscoped().findAll(query) | 169 | return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) |
170 | } | ||
190 | 171 | ||
191 | return sample(rows) | 172 | static async findTrendingToDuplicate (randomizedFactor: number) { |
173 | // On VideoModel! | ||
174 | const query = { | ||
175 | attributes: [ 'id', 'views' ], | ||
176 | subQuery: false, | ||
177 | logging: !isTestInstance(), | ||
178 | group: 'VideoModel.id', | ||
179 | limit: randomizedFactor, | ||
180 | order: getVideoSort('-trending'), | ||
181 | include: [ | ||
182 | await VideoRedundancyModel.buildVideoFileForDuplication(), | ||
183 | VideoRedundancyModel.buildServerRedundancyInclude(), | ||
184 | |||
185 | VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) | ||
186 | ] | ||
187 | } | ||
188 | |||
189 | return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) | ||
192 | } | 190 | } |
193 | 191 | ||
194 | static async getVideoFiles (strategy: VideoRedundancyStrategy) { | 192 | static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { |
193 | // On VideoModel! | ||
194 | const query = { | ||
195 | attributes: [ 'id', 'publishedAt' ], | ||
196 | logging: !isTestInstance(), | ||
197 | limit: randomizedFactor, | ||
198 | order: getVideoSort('-publishedAt'), | ||
199 | where: { | ||
200 | views: { | ||
201 | [ Sequelize.Op.gte ]: minViews | ||
202 | } | ||
203 | }, | ||
204 | include: [ | ||
205 | await VideoRedundancyModel.buildVideoFileForDuplication(), | ||
206 | VideoRedundancyModel.buildServerRedundancyInclude() | ||
207 | ] | ||
208 | } | ||
209 | |||
210 | return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) | ||
211 | } | ||
212 | |||
213 | static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { | ||
195 | const actor = await getServerActor() | 214 | const actor = await getServerActor() |
196 | 215 | ||
197 | const queryVideoFiles = { | 216 | const options = { |
198 | logging: !isTestInstance(), | 217 | logging: !isTestInstance(), |
199 | where: { | 218 | include: [ |
200 | actorId: actor.id, | 219 | { |
201 | strategy | 220 | attributes: [], |
202 | } | 221 | model: VideoRedundancyModel, |
222 | required: true, | ||
223 | where: { | ||
224 | actorId: actor.id, | ||
225 | strategy | ||
226 | } | ||
227 | } | ||
228 | ] | ||
203 | } | 229 | } |
204 | 230 | ||
205 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) | 231 | return VideoFileModel.sum('size', options) |
206 | .findAll(queryVideoFiles) | ||
207 | } | 232 | } |
208 | 233 | ||
209 | static listAllExpired () { | 234 | static listAllExpired () { |
@@ -211,7 +236,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
211 | logging: !isTestInstance(), | 236 | logging: !isTestInstance(), |
212 | where: { | 237 | where: { |
213 | expiresOn: { | 238 | expiresOn: { |
214 | [Sequelize.Op.lt]: new Date() | 239 | [ Sequelize.Op.lt ]: new Date() |
215 | } | 240 | } |
216 | } | 241 | } |
217 | } | 242 | } |
@@ -220,6 +245,37 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
220 | .findAll(query) | 245 | .findAll(query) |
221 | } | 246 | } |
222 | 247 | ||
248 | static async getStats (strategy: VideoRedundancyStrategy) { | ||
249 | const actor = await getServerActor() | ||
250 | |||
251 | const query = { | ||
252 | raw: true, | ||
253 | attributes: [ | ||
254 | [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ], | ||
255 | [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ], | ||
256 | [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ] | ||
257 | ], | ||
258 | where: { | ||
259 | strategy, | ||
260 | actorId: actor.id | ||
261 | }, | ||
262 | include: [ | ||
263 | { | ||
264 | attributes: [], | ||
265 | model: VideoFileModel, | ||
266 | required: true | ||
267 | } | ||
268 | ] | ||
269 | } | ||
270 | |||
271 | return VideoRedundancyModel.find(query as any) // FIXME: typings | ||
272 | .then((r: any) => ({ | ||
273 | totalUsed: parseInt(r.totalUsed.toString(), 10), | ||
274 | totalVideos: r.totalVideos, | ||
275 | totalVideoFiles: r.totalVideoFiles | ||
276 | })) | ||
277 | } | ||
278 | |||
223 | toActivityPubObject (): CacheFileObject { | 279 | toActivityPubObject (): CacheFileObject { |
224 | return { | 280 | return { |
225 | id: this.url, | 281 | id: this.url, |
@@ -237,13 +293,50 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
237 | } | 293 | } |
238 | } | 294 | } |
239 | 295 | ||
240 | private static async buildExcludeIn () { | 296 | // Don't include video files we already duplicated |
297 | private static async buildVideoFileForDuplication () { | ||
241 | const actor = await getServerActor() | 298 | const actor = await getServerActor() |
242 | 299 | ||
243 | return Sequelize.literal( | 300 | const notIn = Sequelize.literal( |
244 | '(' + | 301 | '(' + |
245 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + | 302 | `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + |
246 | ')' | 303 | ')' |
247 | ) | 304 | ) |
305 | |||
306 | return { | ||
307 | attributes: [], | ||
308 | model: VideoFileModel.unscoped(), | ||
309 | required: true, | ||
310 | where: { | ||
311 | id: { | ||
312 | [ Sequelize.Op.notIn ]: notIn | ||
313 | } | ||
314 | } | ||
315 | } | ||
316 | } | ||
317 | |||
318 | private static buildServerRedundancyInclude () { | ||
319 | return { | ||
320 | attributes: [], | ||
321 | model: VideoChannelModel.unscoped(), | ||
322 | required: true, | ||
323 | include: [ | ||
324 | { | ||
325 | attributes: [], | ||
326 | model: ActorModel.unscoped(), | ||
327 | required: true, | ||
328 | include: [ | ||
329 | { | ||
330 | attributes: [], | ||
331 | model: ServerModel.unscoped(), | ||
332 | required: true, | ||
333 | where: { | ||
334 | redundancyAllowed: true | ||
335 | } | ||
336 | } | ||
337 | ] | ||
338 | } | ||
339 | ] | ||
340 | } | ||
248 | } | 341 | } |
249 | } | 342 | } |
diff --git a/server/models/video/tag.ts b/server/models/video/tag.ts index e39a418cd..b39621eaf 100644 --- a/server/models/video/tag.ts +++ b/server/models/video/tag.ts | |||
@@ -48,11 +48,10 @@ export class TagModel extends Model<TagModel> { | |||
48 | }, | 48 | }, |
49 | defaults: { | 49 | defaults: { |
50 | name: tag | 50 | name: tag |
51 | } | 51 | }, |
52 | transaction | ||
52 | } | 53 | } |
53 | 54 | ||
54 | if (transaction) query['transaction'] = transaction | ||
55 | |||
56 | const promise = TagModel.findOrCreate(query) | 55 | const promise = TagModel.findOrCreate(query) |
57 | .then(([ tagInstance ]) => tagInstance) | 56 | .then(([ tagInstance ]) => tagInstance) |
58 | tasks.push(promise) | 57 | tasks.push(promise) |
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts new file mode 100644 index 000000000..a9a58624d --- /dev/null +++ b/server/models/video/video-format-utils.ts | |||
@@ -0,0 +1,296 @@ | |||
1 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | ||
2 | import { VideoModel } from './video' | ||
3 | import { VideoFileModel } from './video-file' | ||
4 | import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' | ||
5 | import { CONFIG, THUMBNAILS_SIZE, VIDEO_EXT_MIMETYPE } from '../../initializers' | ||
6 | import { VideoCaptionModel } from './video-caption' | ||
7 | import { | ||
8 | getVideoCommentsActivityPubUrl, | ||
9 | getVideoDislikesActivityPubUrl, | ||
10 | getVideoLikesActivityPubUrl, | ||
11 | getVideoSharesActivityPubUrl | ||
12 | } from '../../lib/activitypub' | ||
13 | |||
14 | export type VideoFormattingJSONOptions = { | ||
15 | additionalAttributes: { | ||
16 | state?: boolean, | ||
17 | waitTranscoding?: boolean, | ||
18 | scheduledUpdate?: boolean, | ||
19 | blacklistInfo?: boolean | ||
20 | } | ||
21 | } | ||
22 | function videoModelToFormattedJSON (video: VideoModel, options?: VideoFormattingJSONOptions): Video { | ||
23 | const formattedAccount = video.VideoChannel.Account.toFormattedJSON() | ||
24 | const formattedVideoChannel = video.VideoChannel.toFormattedJSON() | ||
25 | |||
26 | const videoObject: Video = { | ||
27 | id: video.id, | ||
28 | uuid: video.uuid, | ||
29 | name: video.name, | ||
30 | category: { | ||
31 | id: video.category, | ||
32 | label: VideoModel.getCategoryLabel(video.category) | ||
33 | }, | ||
34 | licence: { | ||
35 | id: video.licence, | ||
36 | label: VideoModel.getLicenceLabel(video.licence) | ||
37 | }, | ||
38 | language: { | ||
39 | id: video.language, | ||
40 | label: VideoModel.getLanguageLabel(video.language) | ||
41 | }, | ||
42 | privacy: { | ||
43 | id: video.privacy, | ||
44 | label: VideoModel.getPrivacyLabel(video.privacy) | ||
45 | }, | ||
46 | nsfw: video.nsfw, | ||
47 | description: video.getTruncatedDescription(), | ||
48 | isLocal: video.isOwned(), | ||
49 | duration: video.duration, | ||
50 | views: video.views, | ||
51 | likes: video.likes, | ||
52 | dislikes: video.dislikes, | ||
53 | thumbnailPath: video.getThumbnailStaticPath(), | ||
54 | previewPath: video.getPreviewStaticPath(), | ||
55 | embedPath: video.getEmbedStaticPath(), | ||
56 | createdAt: video.createdAt, | ||
57 | updatedAt: video.updatedAt, | ||
58 | publishedAt: video.publishedAt, | ||
59 | account: { | ||
60 | id: formattedAccount.id, | ||
61 | uuid: formattedAccount.uuid, | ||
62 | name: formattedAccount.name, | ||
63 | displayName: formattedAccount.displayName, | ||
64 | url: formattedAccount.url, | ||
65 | host: formattedAccount.host, | ||
66 | avatar: formattedAccount.avatar | ||
67 | }, | ||
68 | channel: { | ||
69 | id: formattedVideoChannel.id, | ||
70 | uuid: formattedVideoChannel.uuid, | ||
71 | name: formattedVideoChannel.name, | ||
72 | displayName: formattedVideoChannel.displayName, | ||
73 | url: formattedVideoChannel.url, | ||
74 | host: formattedVideoChannel.host, | ||
75 | avatar: formattedVideoChannel.avatar | ||
76 | } | ||
77 | } | ||
78 | |||
79 | if (options) { | ||
80 | if (options.additionalAttributes.state === true) { | ||
81 | videoObject.state = { | ||
82 | id: video.state, | ||
83 | label: VideoModel.getStateLabel(video.state) | ||
84 | } | ||
85 | } | ||
86 | |||
87 | if (options.additionalAttributes.waitTranscoding === true) { | ||
88 | videoObject.waitTranscoding = video.waitTranscoding | ||
89 | } | ||
90 | |||
91 | if (options.additionalAttributes.scheduledUpdate === true && video.ScheduleVideoUpdate) { | ||
92 | videoObject.scheduledUpdate = { | ||
93 | updateAt: video.ScheduleVideoUpdate.updateAt, | ||
94 | privacy: video.ScheduleVideoUpdate.privacy || undefined | ||
95 | } | ||
96 | } | ||
97 | |||
98 | if (options.additionalAttributes.blacklistInfo === true) { | ||
99 | videoObject.blacklisted = !!video.VideoBlacklist | ||
100 | videoObject.blacklistedReason = video.VideoBlacklist ? video.VideoBlacklist.reason : null | ||
101 | } | ||
102 | } | ||
103 | |||
104 | return videoObject | ||
105 | } | ||
106 | |||
107 | function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails { | ||
108 | const formattedJson = video.toFormattedJSON({ | ||
109 | additionalAttributes: { | ||
110 | scheduledUpdate: true, | ||
111 | blacklistInfo: true | ||
112 | } | ||
113 | }) | ||
114 | |||
115 | const tags = video.Tags ? video.Tags.map(t => t.name) : [] | ||
116 | const detailsJson = { | ||
117 | support: video.support, | ||
118 | descriptionPath: video.getDescriptionAPIPath(), | ||
119 | channel: video.VideoChannel.toFormattedJSON(), | ||
120 | account: video.VideoChannel.Account.toFormattedJSON(), | ||
121 | tags, | ||
122 | commentsEnabled: video.commentsEnabled, | ||
123 | waitTranscoding: video.waitTranscoding, | ||
124 | state: { | ||
125 | id: video.state, | ||
126 | label: VideoModel.getStateLabel(video.state) | ||
127 | }, | ||
128 | files: [] | ||
129 | } | ||
130 | |||
131 | // Format and sort video files | ||
132 | detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) | ||
133 | |||
134 | return Object.assign(formattedJson, detailsJson) | ||
135 | } | ||
136 | |||
137 | function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { | ||
138 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | ||
139 | |||
140 | return videoFiles | ||
141 | .map(videoFile => { | ||
142 | let resolutionLabel = videoFile.resolution + 'p' | ||
143 | |||
144 | return { | ||
145 | resolution: { | ||
146 | id: videoFile.resolution, | ||
147 | label: resolutionLabel | ||
148 | }, | ||
149 | magnetUri: video.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), | ||
150 | size: videoFile.size, | ||
151 | fps: videoFile.fps, | ||
152 | torrentUrl: video.getTorrentUrl(videoFile, baseUrlHttp), | ||
153 | torrentDownloadUrl: video.getTorrentDownloadUrl(videoFile, baseUrlHttp), | ||
154 | fileUrl: video.getVideoFileUrl(videoFile, baseUrlHttp), | ||
155 | fileDownloadUrl: video.getVideoFileDownloadUrl(videoFile, baseUrlHttp) | ||
156 | } as VideoFile | ||
157 | }) | ||
158 | .sort((a, b) => { | ||
159 | if (a.resolution.id < b.resolution.id) return 1 | ||
160 | if (a.resolution.id === b.resolution.id) return 0 | ||
161 | return -1 | ||
162 | }) | ||
163 | } | ||
164 | |||
165 | function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { | ||
166 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | ||
167 | if (!video.Tags) video.Tags = [] | ||
168 | |||
169 | const tag = video.Tags.map(t => ({ | ||
170 | type: 'Hashtag' as 'Hashtag', | ||
171 | name: t.name | ||
172 | })) | ||
173 | |||
174 | let language | ||
175 | if (video.language) { | ||
176 | language = { | ||
177 | identifier: video.language, | ||
178 | name: VideoModel.getLanguageLabel(video.language) | ||
179 | } | ||
180 | } | ||
181 | |||
182 | let category | ||
183 | if (video.category) { | ||
184 | category = { | ||
185 | identifier: video.category + '', | ||
186 | name: VideoModel.getCategoryLabel(video.category) | ||
187 | } | ||
188 | } | ||
189 | |||
190 | let licence | ||
191 | if (video.licence) { | ||
192 | licence = { | ||
193 | identifier: video.licence + '', | ||
194 | name: VideoModel.getLicenceLabel(video.licence) | ||
195 | } | ||
196 | } | ||
197 | |||
198 | const url: ActivityUrlObject[] = [] | ||
199 | for (const file of video.VideoFiles) { | ||
200 | url.push({ | ||
201 | type: 'Link', | ||
202 | mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, | ||
203 | href: video.getVideoFileUrl(file, baseUrlHttp), | ||
204 | height: file.resolution, | ||
205 | size: file.size, | ||
206 | fps: file.fps | ||
207 | }) | ||
208 | |||
209 | url.push({ | ||
210 | type: 'Link', | ||
211 | mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', | ||
212 | href: video.getTorrentUrl(file, baseUrlHttp), | ||
213 | height: file.resolution | ||
214 | }) | ||
215 | |||
216 | url.push({ | ||
217 | type: 'Link', | ||
218 | mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', | ||
219 | href: video.generateMagnetUri(file, baseUrlHttp, baseUrlWs), | ||
220 | height: file.resolution | ||
221 | }) | ||
222 | } | ||
223 | |||
224 | // Add video url too | ||
225 | url.push({ | ||
226 | type: 'Link', | ||
227 | mimeType: 'text/html', | ||
228 | href: CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid | ||
229 | }) | ||
230 | |||
231 | const subtitleLanguage = [] | ||
232 | for (const caption of video.VideoCaptions) { | ||
233 | subtitleLanguage.push({ | ||
234 | identifier: caption.language, | ||
235 | name: VideoCaptionModel.getLanguageLabel(caption.language) | ||
236 | }) | ||
237 | } | ||
238 | |||
239 | return { | ||
240 | type: 'Video' as 'Video', | ||
241 | id: video.url, | ||
242 | name: video.name, | ||
243 | duration: getActivityStreamDuration(video.duration), | ||
244 | uuid: video.uuid, | ||
245 | tag, | ||
246 | category, | ||
247 | licence, | ||
248 | language, | ||
249 | views: video.views, | ||
250 | sensitive: video.nsfw, | ||
251 | waitTranscoding: video.waitTranscoding, | ||
252 | state: video.state, | ||
253 | commentsEnabled: video.commentsEnabled, | ||
254 | published: video.publishedAt.toISOString(), | ||
255 | updated: video.updatedAt.toISOString(), | ||
256 | mediaType: 'text/markdown', | ||
257 | content: video.getTruncatedDescription(), | ||
258 | support: video.support, | ||
259 | subtitleLanguage, | ||
260 | icon: { | ||
261 | type: 'Image', | ||
262 | url: video.getThumbnailUrl(baseUrlHttp), | ||
263 | mediaType: 'image/jpeg', | ||
264 | width: THUMBNAILS_SIZE.width, | ||
265 | height: THUMBNAILS_SIZE.height | ||
266 | }, | ||
267 | url, | ||
268 | likes: getVideoLikesActivityPubUrl(video), | ||
269 | dislikes: getVideoDislikesActivityPubUrl(video), | ||
270 | shares: getVideoSharesActivityPubUrl(video), | ||
271 | comments: getVideoCommentsActivityPubUrl(video), | ||
272 | attributedTo: [ | ||
273 | { | ||
274 | type: 'Person', | ||
275 | id: video.VideoChannel.Account.Actor.url | ||
276 | }, | ||
277 | { | ||
278 | type: 'Group', | ||
279 | id: video.VideoChannel.Actor.url | ||
280 | } | ||
281 | ] | ||
282 | } | ||
283 | } | ||
284 | |||
285 | function getActivityStreamDuration (duration: number) { | ||
286 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
287 | return 'PT' + duration + 'S' | ||
288 | } | ||
289 | |||
290 | export { | ||
291 | videoModelToFormattedJSON, | ||
292 | videoModelToFormattedDetailsJSON, | ||
293 | videoFilesModelToFormattedJSON, | ||
294 | videoModelToActivityPubObject, | ||
295 | getActivityStreamDuration | ||
296 | } | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 27c631dcd..6c89c16bf 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { map, maxBy } from 'lodash' | 2 | import { maxBy } from 'lodash' |
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import * as parseTorrent from 'parse-torrent' | 4 | import * as parseTorrent from 'parse-torrent' |
5 | import { extname, join } from 'path' | 5 | import { join } from 'path' |
6 | import * as Sequelize from 'sequelize' | 6 | import * as Sequelize from 'sequelize' |
7 | import { | 7 | import { |
8 | AllowNull, | 8 | AllowNull, |
@@ -27,7 +27,7 @@ import { | |||
27 | Table, | 27 | Table, |
28 | UpdatedAt | 28 | UpdatedAt |
29 | } from 'sequelize-typescript' | 29 | } from 'sequelize-typescript' |
30 | import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' | 30 | import { VideoPrivacy, VideoState } from '../../../shared' |
31 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 31 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
32 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | 32 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' |
33 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 33 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
@@ -45,7 +45,7 @@ import { | |||
45 | isVideoStateValid, | 45 | isVideoStateValid, |
46 | isVideoSupportValid | 46 | isVideoSupportValid |
47 | } from '../../helpers/custom-validators/videos' | 47 | } from '../../helpers/custom-validators/videos' |
48 | import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, transcode } from '../../helpers/ffmpeg-utils' | 48 | import { generateImageFromVideoFile, getVideoFileResolution } from '../../helpers/ffmpeg-utils' |
49 | import { logger } from '../../helpers/logger' | 49 | import { logger } from '../../helpers/logger' |
50 | import { getServerActor } from '../../helpers/utils' | 50 | import { getServerActor } from '../../helpers/utils' |
51 | import { | 51 | import { |
@@ -59,18 +59,11 @@ import { | |||
59 | STATIC_PATHS, | 59 | STATIC_PATHS, |
60 | THUMBNAILS_SIZE, | 60 | THUMBNAILS_SIZE, |
61 | VIDEO_CATEGORIES, | 61 | VIDEO_CATEGORIES, |
62 | VIDEO_EXT_MIMETYPE, | ||
63 | VIDEO_LANGUAGES, | 62 | VIDEO_LANGUAGES, |
64 | VIDEO_LICENCES, | 63 | VIDEO_LICENCES, |
65 | VIDEO_PRIVACIES, | 64 | VIDEO_PRIVACIES, |
66 | VIDEO_STATES | 65 | VIDEO_STATES |
67 | } from '../../initializers' | 66 | } from '../../initializers' |
68 | import { | ||
69 | getVideoCommentsActivityPubUrl, | ||
70 | getVideoDislikesActivityPubUrl, | ||
71 | getVideoLikesActivityPubUrl, | ||
72 | getVideoSharesActivityPubUrl | ||
73 | } from '../../lib/activitypub' | ||
74 | import { sendDeleteVideo } from '../../lib/activitypub/send' | 67 | import { sendDeleteVideo } from '../../lib/activitypub/send' |
75 | import { AccountModel } from '../account/account' | 68 | import { AccountModel } from '../account/account' |
76 | import { AccountVideoRateModel } from '../account/account-video-rate' | 69 | import { AccountVideoRateModel } from '../account/account-video-rate' |
@@ -88,9 +81,17 @@ import { VideoTagModel } from './video-tag' | |||
88 | import { ScheduleVideoUpdateModel } from './schedule-video-update' | 81 | import { ScheduleVideoUpdateModel } from './schedule-video-update' |
89 | import { VideoCaptionModel } from './video-caption' | 82 | import { VideoCaptionModel } from './video-caption' |
90 | import { VideoBlacklistModel } from './video-blacklist' | 83 | import { VideoBlacklistModel } from './video-blacklist' |
91 | import { copy, remove, rename, stat, writeFile } from 'fs-extra' | 84 | import { remove, writeFile } from 'fs-extra' |
92 | import { VideoViewModel } from './video-views' | 85 | import { VideoViewModel } from './video-views' |
93 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' | 86 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' |
87 | import { | ||
88 | videoFilesModelToFormattedJSON, | ||
89 | VideoFormattingJSONOptions, | ||
90 | videoModelToActivityPubObject, | ||
91 | videoModelToFormattedDetailsJSON, | ||
92 | videoModelToFormattedJSON | ||
93 | } from './video-format-utils' | ||
94 | import * as validator from 'validator' | ||
94 | 95 | ||
95 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation | 96 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation |
96 | const indexes: Sequelize.DefineIndexesOptions[] = [ | 97 | const indexes: Sequelize.DefineIndexesOptions[] = [ |
@@ -221,6 +222,7 @@ type AvailableForListIDsOptions = { | |||
221 | }, | 222 | }, |
222 | [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { | 223 | [ ScopeNames.AVAILABLE_FOR_LIST_IDS ]: (options: AvailableForListIDsOptions) => { |
223 | const query: IFindOptions<VideoModel> = { | 224 | const query: IFindOptions<VideoModel> = { |
225 | raw: true, | ||
224 | attributes: [ 'id' ], | 226 | attributes: [ 'id' ], |
225 | where: { | 227 | where: { |
226 | id: { | 228 | id: { |
@@ -387,16 +389,7 @@ type AvailableForListIDsOptions = { | |||
387 | } | 389 | } |
388 | 390 | ||
389 | if (options.trendingDays) { | 391 | if (options.trendingDays) { |
390 | query.include.push({ | 392 | query.include.push(VideoModel.buildTrendingQuery(options.trendingDays)) |
391 | attributes: [], | ||
392 | model: VideoViewModel, | ||
393 | required: false, | ||
394 | where: { | ||
395 | startDate: { | ||
396 | [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays) | ||
397 | } | ||
398 | } | ||
399 | }) | ||
400 | 393 | ||
401 | query.subQuery = false | 394 | query.subQuery = false |
402 | } | 395 | } |
@@ -474,6 +467,7 @@ type AvailableForListIDsOptions = { | |||
474 | required: false, | 467 | required: false, |
475 | include: [ | 468 | include: [ |
476 | { | 469 | { |
470 | attributes: [ 'fileUrl' ], | ||
477 | model: () => VideoRedundancyModel.unscoped(), | 471 | model: () => VideoRedundancyModel.unscoped(), |
478 | required: false | 472 | required: false |
479 | } | 473 | } |
@@ -937,7 +931,7 @@ export class VideoModel extends Model<VideoModel> { | |||
937 | videoChannelId?: number, | 931 | videoChannelId?: number, |
938 | actorId?: number | 932 | actorId?: number |
939 | trendingDays?: number | 933 | trendingDays?: number |
940 | }) { | 934 | }, countVideos = true) { |
941 | const query: IFindOptions<VideoModel> = { | 935 | const query: IFindOptions<VideoModel> = { |
942 | offset: options.start, | 936 | offset: options.start, |
943 | limit: options.count, | 937 | limit: options.count, |
@@ -970,7 +964,7 @@ export class VideoModel extends Model<VideoModel> { | |||
970 | trendingDays | 964 | trendingDays |
971 | } | 965 | } |
972 | 966 | ||
973 | return VideoModel.getAvailableForApi(query, queryOptions) | 967 | return VideoModel.getAvailableForApi(query, queryOptions, countVideos) |
974 | } | 968 | } |
975 | 969 | ||
976 | static async searchAndPopulateAccountAndServer (options: { | 970 | static async searchAndPopulateAccountAndServer (options: { |
@@ -1070,41 +1064,34 @@ export class VideoModel extends Model<VideoModel> { | |||
1070 | return VideoModel.getAvailableForApi(query, queryOptions) | 1064 | return VideoModel.getAvailableForApi(query, queryOptions) |
1071 | } | 1065 | } |
1072 | 1066 | ||
1073 | static load (id: number, t?: Sequelize.Transaction) { | 1067 | static load (id: number | string, t?: Sequelize.Transaction) { |
1074 | const options = t ? { transaction: t } : undefined | 1068 | const where = VideoModel.buildWhereIdOrUUID(id) |
1075 | 1069 | const options = { | |
1076 | return VideoModel.findById(id, options) | 1070 | where, |
1077 | } | 1071 | transaction: t |
1078 | |||
1079 | static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { | ||
1080 | const query: IFindOptions<VideoModel> = { | ||
1081 | where: { | ||
1082 | url | ||
1083 | } | ||
1084 | } | 1072 | } |
1085 | 1073 | ||
1086 | if (t !== undefined) query.transaction = t | 1074 | return VideoModel.findOne(options) |
1087 | |||
1088 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) | ||
1089 | } | 1075 | } |
1090 | 1076 | ||
1091 | static loadAndPopulateAccountAndServerAndTags (id: number) { | 1077 | static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { |
1078 | const where = VideoModel.buildWhereIdOrUUID(id) | ||
1079 | |||
1092 | const options = { | 1080 | const options = { |
1093 | order: [ [ 'Tags', 'name', 'ASC' ] ] | 1081 | attributes: [ 'id' ], |
1082 | where, | ||
1083 | transaction: t | ||
1094 | } | 1084 | } |
1095 | 1085 | ||
1096 | return VideoModel | 1086 | return VideoModel.findOne(options) |
1097 | .scope([ | ||
1098 | ScopeNames.WITH_TAGS, | ||
1099 | ScopeNames.WITH_BLACKLISTED, | ||
1100 | ScopeNames.WITH_FILES, | ||
1101 | ScopeNames.WITH_ACCOUNT_DETAILS, | ||
1102 | ScopeNames.WITH_SCHEDULED_UPDATE | ||
1103 | ]) | ||
1104 | .findById(id, options) | ||
1105 | } | 1087 | } |
1106 | 1088 | ||
1107 | static loadByUUID (uuid: string) { | 1089 | static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { |
1090 | return VideoModel.scope(ScopeNames.WITH_FILES) | ||
1091 | .findById(id, { transaction: t, logging }) | ||
1092 | } | ||
1093 | |||
1094 | static loadByUUIDWithFile (uuid: string) { | ||
1108 | const options = { | 1095 | const options = { |
1109 | where: { | 1096 | where: { |
1110 | uuid | 1097 | uuid |
@@ -1116,12 +1103,34 @@ export class VideoModel extends Model<VideoModel> { | |||
1116 | .findOne(options) | 1103 | .findOne(options) |
1117 | } | 1104 | } |
1118 | 1105 | ||
1119 | static loadByUUIDAndPopulateAccountAndServerAndTags (uuid: string, t?: Sequelize.Transaction) { | 1106 | static loadByUrl (url: string, transaction?: Sequelize.Transaction) { |
1120 | const options = { | 1107 | const query: IFindOptions<VideoModel> = { |
1121 | order: [ [ 'Tags', 'name', 'ASC' ] ], | ||
1122 | where: { | 1108 | where: { |
1123 | uuid | 1109 | url |
1110 | }, | ||
1111 | transaction | ||
1112 | } | ||
1113 | |||
1114 | return VideoModel.findOne(query) | ||
1115 | } | ||
1116 | |||
1117 | static loadByUrlAndPopulateAccount (url: string, transaction?: Sequelize.Transaction) { | ||
1118 | const query: IFindOptions<VideoModel> = { | ||
1119 | where: { | ||
1120 | url | ||
1124 | }, | 1121 | }, |
1122 | transaction | ||
1123 | } | ||
1124 | |||
1125 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) | ||
1126 | } | ||
1127 | |||
1128 | static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction) { | ||
1129 | const where = VideoModel.buildWhereIdOrUUID(id) | ||
1130 | |||
1131 | const options = { | ||
1132 | order: [ [ 'Tags', 'name', 'ASC' ] ], | ||
1133 | where, | ||
1125 | transaction: t | 1134 | transaction: t |
1126 | } | 1135 | } |
1127 | 1136 | ||
@@ -1169,7 +1178,14 @@ export class VideoModel extends Model<VideoModel> { | |||
1169 | } | 1178 | } |
1170 | 1179 | ||
1171 | // threshold corresponds to how many video the field should have to be returned | 1180 | // threshold corresponds to how many video the field should have to be returned |
1172 | static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { | 1181 | static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { |
1182 | const actorId = (await getServerActor()).id | ||
1183 | |||
1184 | const scopeOptions = { | ||
1185 | actorId, | ||
1186 | includeLocalVideos: true | ||
1187 | } | ||
1188 | |||
1173 | const query: IFindOptions<VideoModel> = { | 1189 | const query: IFindOptions<VideoModel> = { |
1174 | attributes: [ field ], | 1190 | attributes: [ field ], |
1175 | limit: count, | 1191 | limit: count, |
@@ -1177,20 +1193,28 @@ export class VideoModel extends Model<VideoModel> { | |||
1177 | having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { | 1193 | having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { |
1178 | [ Sequelize.Op.gte ]: threshold | 1194 | [ Sequelize.Op.gte ]: threshold |
1179 | }) as any, // FIXME: typings | 1195 | }) as any, // FIXME: typings |
1180 | where: { | ||
1181 | [ field ]: { | ||
1182 | [ Sequelize.Op.not ]: null | ||
1183 | }, | ||
1184 | privacy: VideoPrivacy.PUBLIC, | ||
1185 | state: VideoState.PUBLISHED | ||
1186 | }, | ||
1187 | order: [ this.sequelize.random() ] | 1196 | order: [ this.sequelize.random() ] |
1188 | } | 1197 | } |
1189 | 1198 | ||
1190 | return VideoModel.findAll(query) | 1199 | return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] }) |
1200 | .findAll(query) | ||
1191 | .then(rows => rows.map(r => r[ field ])) | 1201 | .then(rows => rows.map(r => r[ field ])) |
1192 | } | 1202 | } |
1193 | 1203 | ||
1204 | static buildTrendingQuery (trendingDays: number) { | ||
1205 | return { | ||
1206 | attributes: [], | ||
1207 | subQuery: false, | ||
1208 | model: VideoViewModel, | ||
1209 | required: false, | ||
1210 | where: { | ||
1211 | startDate: { | ||
1212 | [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) | ||
1213 | } | ||
1214 | } | ||
1215 | } | ||
1216 | } | ||
1217 | |||
1194 | private static buildActorWhereWithFilter (filter?: VideoFilter) { | 1218 | private static buildActorWhereWithFilter (filter?: VideoFilter) { |
1195 | if (filter && filter === 'local') { | 1219 | if (filter && filter === 'local') { |
1196 | return { | 1220 | return { |
@@ -1201,7 +1225,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1201 | return {} | 1225 | return {} |
1202 | } | 1226 | } |
1203 | 1227 | ||
1204 | private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions) { | 1228 | private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions, countVideos = true) { |
1205 | const idsScope = { | 1229 | const idsScope = { |
1206 | method: [ | 1230 | method: [ |
1207 | ScopeNames.AVAILABLE_FOR_LIST_IDS, options | 1231 | ScopeNames.AVAILABLE_FOR_LIST_IDS, options |
@@ -1218,7 +1242,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1218 | } | 1242 | } |
1219 | 1243 | ||
1220 | const [ count, rowsId ] = await Promise.all([ | 1244 | const [ count, rowsId ] = await Promise.all([ |
1221 | VideoModel.scope(countScope).count(countQuery), | 1245 | countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), |
1222 | VideoModel.scope(idsScope).findAll(query) | 1246 | VideoModel.scope(idsScope).findAll(query) |
1223 | ]) | 1247 | ]) |
1224 | const ids = rowsId.map(r => r.id) | 1248 | const ids = rowsId.map(r => r.id) |
@@ -1247,26 +1271,30 @@ export class VideoModel extends Model<VideoModel> { | |||
1247 | } | 1271 | } |
1248 | } | 1272 | } |
1249 | 1273 | ||
1250 | private static getCategoryLabel (id: number) { | 1274 | static getCategoryLabel (id: number) { |
1251 | return VIDEO_CATEGORIES[ id ] || 'Misc' | 1275 | return VIDEO_CATEGORIES[ id ] || 'Misc' |
1252 | } | 1276 | } |
1253 | 1277 | ||
1254 | private static getLicenceLabel (id: number) { | 1278 | static getLicenceLabel (id: number) { |
1255 | return VIDEO_LICENCES[ id ] || 'Unknown' | 1279 | return VIDEO_LICENCES[ id ] || 'Unknown' |
1256 | } | 1280 | } |
1257 | 1281 | ||
1258 | private static getLanguageLabel (id: string) { | 1282 | static getLanguageLabel (id: string) { |
1259 | return VIDEO_LANGUAGES[ id ] || 'Unknown' | 1283 | return VIDEO_LANGUAGES[ id ] || 'Unknown' |
1260 | } | 1284 | } |
1261 | 1285 | ||
1262 | private static getPrivacyLabel (id: number) { | 1286 | static getPrivacyLabel (id: number) { |
1263 | return VIDEO_PRIVACIES[ id ] || 'Unknown' | 1287 | return VIDEO_PRIVACIES[ id ] || 'Unknown' |
1264 | } | 1288 | } |
1265 | 1289 | ||
1266 | private static getStateLabel (id: number) { | 1290 | static getStateLabel (id: number) { |
1267 | return VIDEO_STATES[ id ] || 'Unknown' | 1291 | return VIDEO_STATES[ id ] || 'Unknown' |
1268 | } | 1292 | } |
1269 | 1293 | ||
1294 | static buildWhereIdOrUUID (id: number | string) { | ||
1295 | return validator.isInt('' + id) ? { id } : { uuid: id } | ||
1296 | } | ||
1297 | |||
1270 | getOriginalFile () { | 1298 | getOriginalFile () { |
1271 | if (Array.isArray(this.VideoFiles) === false) return undefined | 1299 | if (Array.isArray(this.VideoFiles) === false) return undefined |
1272 | 1300 | ||
@@ -1359,273 +1387,20 @@ export class VideoModel extends Model<VideoModel> { | |||
1359 | return join(STATIC_PATHS.PREVIEWS, this.getPreviewName()) | 1387 | return join(STATIC_PATHS.PREVIEWS, this.getPreviewName()) |
1360 | } | 1388 | } |
1361 | 1389 | ||
1362 | toFormattedJSON (options?: { | 1390 | toFormattedJSON (options?: VideoFormattingJSONOptions): Video { |
1363 | additionalAttributes: { | 1391 | return videoModelToFormattedJSON(this, options) |
1364 | state?: boolean, | ||
1365 | waitTranscoding?: boolean, | ||
1366 | scheduledUpdate?: boolean, | ||
1367 | blacklistInfo?: boolean | ||
1368 | } | ||
1369 | }): Video { | ||
1370 | const formattedAccount = this.VideoChannel.Account.toFormattedJSON() | ||
1371 | const formattedVideoChannel = this.VideoChannel.toFormattedJSON() | ||
1372 | |||
1373 | const videoObject: Video = { | ||
1374 | id: this.id, | ||
1375 | uuid: this.uuid, | ||
1376 | name: this.name, | ||
1377 | category: { | ||
1378 | id: this.category, | ||
1379 | label: VideoModel.getCategoryLabel(this.category) | ||
1380 | }, | ||
1381 | licence: { | ||
1382 | id: this.licence, | ||
1383 | label: VideoModel.getLicenceLabel(this.licence) | ||
1384 | }, | ||
1385 | language: { | ||
1386 | id: this.language, | ||
1387 | label: VideoModel.getLanguageLabel(this.language) | ||
1388 | }, | ||
1389 | privacy: { | ||
1390 | id: this.privacy, | ||
1391 | label: VideoModel.getPrivacyLabel(this.privacy) | ||
1392 | }, | ||
1393 | nsfw: this.nsfw, | ||
1394 | description: this.getTruncatedDescription(), | ||
1395 | isLocal: this.isOwned(), | ||
1396 | duration: this.duration, | ||
1397 | views: this.views, | ||
1398 | likes: this.likes, | ||
1399 | dislikes: this.dislikes, | ||
1400 | thumbnailPath: this.getThumbnailStaticPath(), | ||
1401 | previewPath: this.getPreviewStaticPath(), | ||
1402 | embedPath: this.getEmbedStaticPath(), | ||
1403 | createdAt: this.createdAt, | ||
1404 | updatedAt: this.updatedAt, | ||
1405 | publishedAt: this.publishedAt, | ||
1406 | account: { | ||
1407 | id: formattedAccount.id, | ||
1408 | uuid: formattedAccount.uuid, | ||
1409 | name: formattedAccount.name, | ||
1410 | displayName: formattedAccount.displayName, | ||
1411 | url: formattedAccount.url, | ||
1412 | host: formattedAccount.host, | ||
1413 | avatar: formattedAccount.avatar | ||
1414 | }, | ||
1415 | channel: { | ||
1416 | id: formattedVideoChannel.id, | ||
1417 | uuid: formattedVideoChannel.uuid, | ||
1418 | name: formattedVideoChannel.name, | ||
1419 | displayName: formattedVideoChannel.displayName, | ||
1420 | url: formattedVideoChannel.url, | ||
1421 | host: formattedVideoChannel.host, | ||
1422 | avatar: formattedVideoChannel.avatar | ||
1423 | } | ||
1424 | } | ||
1425 | |||
1426 | if (options) { | ||
1427 | if (options.additionalAttributes.state === true) { | ||
1428 | videoObject.state = { | ||
1429 | id: this.state, | ||
1430 | label: VideoModel.getStateLabel(this.state) | ||
1431 | } | ||
1432 | } | ||
1433 | |||
1434 | if (options.additionalAttributes.waitTranscoding === true) { | ||
1435 | videoObject.waitTranscoding = this.waitTranscoding | ||
1436 | } | ||
1437 | |||
1438 | if (options.additionalAttributes.scheduledUpdate === true && this.ScheduleVideoUpdate) { | ||
1439 | videoObject.scheduledUpdate = { | ||
1440 | updateAt: this.ScheduleVideoUpdate.updateAt, | ||
1441 | privacy: this.ScheduleVideoUpdate.privacy || undefined | ||
1442 | } | ||
1443 | } | ||
1444 | |||
1445 | if (options.additionalAttributes.blacklistInfo === true) { | ||
1446 | videoObject.blacklisted = !!this.VideoBlacklist | ||
1447 | videoObject.blacklistedReason = this.VideoBlacklist ? this.VideoBlacklist.reason : null | ||
1448 | } | ||
1449 | } | ||
1450 | |||
1451 | return videoObject | ||
1452 | } | 1392 | } |
1453 | 1393 | ||
1454 | toFormattedDetailsJSON (): VideoDetails { | 1394 | toFormattedDetailsJSON (): VideoDetails { |
1455 | const formattedJson = this.toFormattedJSON({ | 1395 | return videoModelToFormattedDetailsJSON(this) |
1456 | additionalAttributes: { | ||
1457 | scheduledUpdate: true, | ||
1458 | blacklistInfo: true | ||
1459 | } | ||
1460 | }) | ||
1461 | |||
1462 | const detailsJson = { | ||
1463 | support: this.support, | ||
1464 | descriptionPath: this.getDescriptionPath(), | ||
1465 | channel: this.VideoChannel.toFormattedJSON(), | ||
1466 | account: this.VideoChannel.Account.toFormattedJSON(), | ||
1467 | tags: map(this.Tags, 'name'), | ||
1468 | commentsEnabled: this.commentsEnabled, | ||
1469 | waitTranscoding: this.waitTranscoding, | ||
1470 | state: { | ||
1471 | id: this.state, | ||
1472 | label: VideoModel.getStateLabel(this.state) | ||
1473 | }, | ||
1474 | files: [] | ||
1475 | } | ||
1476 | |||
1477 | // Format and sort video files | ||
1478 | detailsJson.files = this.getFormattedVideoFilesJSON() | ||
1479 | |||
1480 | return Object.assign(formattedJson, detailsJson) | ||
1481 | } | 1396 | } |
1482 | 1397 | ||
1483 | getFormattedVideoFilesJSON (): VideoFile[] { | 1398 | getFormattedVideoFilesJSON (): VideoFile[] { |
1484 | const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() | 1399 | return videoFilesModelToFormattedJSON(this, this.VideoFiles) |
1485 | |||
1486 | return this.VideoFiles | ||
1487 | .map(videoFile => { | ||
1488 | let resolutionLabel = videoFile.resolution + 'p' | ||
1489 | |||
1490 | return { | ||
1491 | resolution: { | ||
1492 | id: videoFile.resolution, | ||
1493 | label: resolutionLabel | ||
1494 | }, | ||
1495 | magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), | ||
1496 | size: videoFile.size, | ||
1497 | fps: videoFile.fps, | ||
1498 | torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), | ||
1499 | torrentDownloadUrl: this.getTorrentDownloadUrl(videoFile, baseUrlHttp), | ||
1500 | fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp), | ||
1501 | fileDownloadUrl: this.getVideoFileDownloadUrl(videoFile, baseUrlHttp) | ||
1502 | } as VideoFile | ||
1503 | }) | ||
1504 | .sort((a, b) => { | ||
1505 | if (a.resolution.id < b.resolution.id) return 1 | ||
1506 | if (a.resolution.id === b.resolution.id) return 0 | ||
1507 | return -1 | ||
1508 | }) | ||
1509 | } | 1400 | } |
1510 | 1401 | ||
1511 | toActivityPubObject (): VideoTorrentObject { | 1402 | toActivityPubObject (): VideoTorrentObject { |
1512 | const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() | 1403 | return videoModelToActivityPubObject(this) |
1513 | if (!this.Tags) this.Tags = [] | ||
1514 | |||
1515 | const tag = this.Tags.map(t => ({ | ||
1516 | type: 'Hashtag' as 'Hashtag', | ||
1517 | name: t.name | ||
1518 | })) | ||
1519 | |||
1520 | let language | ||
1521 | if (this.language) { | ||
1522 | language = { | ||
1523 | identifier: this.language, | ||
1524 | name: VideoModel.getLanguageLabel(this.language) | ||
1525 | } | ||
1526 | } | ||
1527 | |||
1528 | let category | ||
1529 | if (this.category) { | ||
1530 | category = { | ||
1531 | identifier: this.category + '', | ||
1532 | name: VideoModel.getCategoryLabel(this.category) | ||
1533 | } | ||
1534 | } | ||
1535 | |||
1536 | let licence | ||
1537 | if (this.licence) { | ||
1538 | licence = { | ||
1539 | identifier: this.licence + '', | ||
1540 | name: VideoModel.getLicenceLabel(this.licence) | ||
1541 | } | ||
1542 | } | ||
1543 | |||
1544 | const url: ActivityUrlObject[] = [] | ||
1545 | for (const file of this.VideoFiles) { | ||
1546 | url.push({ | ||
1547 | type: 'Link', | ||
1548 | mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, | ||
1549 | href: this.getVideoFileUrl(file, baseUrlHttp), | ||
1550 | height: file.resolution, | ||
1551 | size: file.size, | ||
1552 | fps: file.fps | ||
1553 | }) | ||
1554 | |||
1555 | url.push({ | ||
1556 | type: 'Link', | ||
1557 | mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', | ||
1558 | href: this.getTorrentUrl(file, baseUrlHttp), | ||
1559 | height: file.resolution | ||
1560 | }) | ||
1561 | |||
1562 | url.push({ | ||
1563 | type: 'Link', | ||
1564 | mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', | ||
1565 | href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), | ||
1566 | height: file.resolution | ||
1567 | }) | ||
1568 | } | ||
1569 | |||
1570 | // Add video url too | ||
1571 | url.push({ | ||
1572 | type: 'Link', | ||
1573 | mimeType: 'text/html', | ||
1574 | href: CONFIG.WEBSERVER.URL + '/videos/watch/' + this.uuid | ||
1575 | }) | ||
1576 | |||
1577 | const subtitleLanguage = [] | ||
1578 | for (const caption of this.VideoCaptions) { | ||
1579 | subtitleLanguage.push({ | ||
1580 | identifier: caption.language, | ||
1581 | name: VideoCaptionModel.getLanguageLabel(caption.language) | ||
1582 | }) | ||
1583 | } | ||
1584 | |||
1585 | return { | ||
1586 | type: 'Video' as 'Video', | ||
1587 | id: this.url, | ||
1588 | name: this.name, | ||
1589 | duration: this.getActivityStreamDuration(), | ||
1590 | uuid: this.uuid, | ||
1591 | tag, | ||
1592 | category, | ||
1593 | licence, | ||
1594 | language, | ||
1595 | views: this.views, | ||
1596 | sensitive: this.nsfw, | ||
1597 | waitTranscoding: this.waitTranscoding, | ||
1598 | state: this.state, | ||
1599 | commentsEnabled: this.commentsEnabled, | ||
1600 | published: this.publishedAt.toISOString(), | ||
1601 | updated: this.updatedAt.toISOString(), | ||
1602 | mediaType: 'text/markdown', | ||
1603 | content: this.getTruncatedDescription(), | ||
1604 | support: this.support, | ||
1605 | subtitleLanguage, | ||
1606 | icon: { | ||
1607 | type: 'Image', | ||
1608 | url: this.getThumbnailUrl(baseUrlHttp), | ||
1609 | mediaType: 'image/jpeg', | ||
1610 | width: THUMBNAILS_SIZE.width, | ||
1611 | height: THUMBNAILS_SIZE.height | ||
1612 | }, | ||
1613 | url, | ||
1614 | likes: getVideoLikesActivityPubUrl(this), | ||
1615 | dislikes: getVideoDislikesActivityPubUrl(this), | ||
1616 | shares: getVideoSharesActivityPubUrl(this), | ||
1617 | comments: getVideoCommentsActivityPubUrl(this), | ||
1618 | attributedTo: [ | ||
1619 | { | ||
1620 | type: 'Person', | ||
1621 | id: this.VideoChannel.Account.Actor.url | ||
1622 | }, | ||
1623 | { | ||
1624 | type: 'Group', | ||
1625 | id: this.VideoChannel.Actor.url | ||
1626 | } | ||
1627 | ] | ||
1628 | } | ||
1629 | } | 1404 | } |
1630 | 1405 | ||
1631 | getTruncatedDescription () { | 1406 | getTruncatedDescription () { |
@@ -1635,130 +1410,13 @@ export class VideoModel extends Model<VideoModel> { | |||
1635 | return peertubeTruncate(this.description, maxLength) | 1410 | return peertubeTruncate(this.description, maxLength) |
1636 | } | 1411 | } |
1637 | 1412 | ||
1638 | async optimizeOriginalVideofile () { | ||
1639 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | ||
1640 | const newExtname = '.mp4' | ||
1641 | const inputVideoFile = this.getOriginalFile() | ||
1642 | const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile)) | ||
1643 | const videoTranscodedPath = join(videosDirectory, this.id + '-transcoded' + newExtname) | ||
1644 | |||
1645 | const transcodeOptions = { | ||
1646 | inputPath: videoInputPath, | ||
1647 | outputPath: videoTranscodedPath | ||
1648 | } | ||
1649 | |||
1650 | // Could be very long! | ||
1651 | await transcode(transcodeOptions) | ||
1652 | |||
1653 | try { | ||
1654 | await remove(videoInputPath) | ||
1655 | |||
1656 | // Important to do this before getVideoFilename() to take in account the new file extension | ||
1657 | inputVideoFile.set('extname', newExtname) | ||
1658 | |||
1659 | const videoOutputPath = this.getVideoFilePath(inputVideoFile) | ||
1660 | await rename(videoTranscodedPath, videoOutputPath) | ||
1661 | const stats = await stat(videoOutputPath) | ||
1662 | const fps = await getVideoFileFPS(videoOutputPath) | ||
1663 | |||
1664 | inputVideoFile.set('size', stats.size) | ||
1665 | inputVideoFile.set('fps', fps) | ||
1666 | |||
1667 | await this.createTorrentAndSetInfoHash(inputVideoFile) | ||
1668 | await inputVideoFile.save() | ||
1669 | |||
1670 | } catch (err) { | ||
1671 | // Auto destruction... | ||
1672 | this.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) | ||
1673 | |||
1674 | throw err | ||
1675 | } | ||
1676 | } | ||
1677 | |||
1678 | async transcodeOriginalVideofile (resolution: VideoResolution, isPortraitMode: boolean) { | ||
1679 | const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR | ||
1680 | const extname = '.mp4' | ||
1681 | |||
1682 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed | ||
1683 | const videoInputPath = join(videosDirectory, this.getVideoFilename(this.getOriginalFile())) | ||
1684 | |||
1685 | const newVideoFile = new VideoFileModel({ | ||
1686 | resolution, | ||
1687 | extname, | ||
1688 | size: 0, | ||
1689 | videoId: this.id | ||
1690 | }) | ||
1691 | const videoOutputPath = join(videosDirectory, this.getVideoFilename(newVideoFile)) | ||
1692 | |||
1693 | const transcodeOptions = { | ||
1694 | inputPath: videoInputPath, | ||
1695 | outputPath: videoOutputPath, | ||
1696 | resolution, | ||
1697 | isPortraitMode | ||
1698 | } | ||
1699 | |||
1700 | await transcode(transcodeOptions) | ||
1701 | |||
1702 | const stats = await stat(videoOutputPath) | ||
1703 | const fps = await getVideoFileFPS(videoOutputPath) | ||
1704 | |||
1705 | newVideoFile.set('size', stats.size) | ||
1706 | newVideoFile.set('fps', fps) | ||
1707 | |||
1708 | await this.createTorrentAndSetInfoHash(newVideoFile) | ||
1709 | |||
1710 | await newVideoFile.save() | ||
1711 | |||
1712 | this.VideoFiles.push(newVideoFile) | ||
1713 | } | ||
1714 | |||
1715 | async importVideoFile (inputFilePath: string) { | ||
1716 | const { videoFileResolution } = await getVideoFileResolution(inputFilePath) | ||
1717 | const { size } = await stat(inputFilePath) | ||
1718 | const fps = await getVideoFileFPS(inputFilePath) | ||
1719 | |||
1720 | let updatedVideoFile = new VideoFileModel({ | ||
1721 | resolution: videoFileResolution, | ||
1722 | extname: extname(inputFilePath), | ||
1723 | size, | ||
1724 | fps, | ||
1725 | videoId: this.id | ||
1726 | }) | ||
1727 | |||
1728 | const currentVideoFile = this.VideoFiles.find(videoFile => videoFile.resolution === updatedVideoFile.resolution) | ||
1729 | |||
1730 | if (currentVideoFile) { | ||
1731 | // Remove old file and old torrent | ||
1732 | await this.removeFile(currentVideoFile) | ||
1733 | await this.removeTorrent(currentVideoFile) | ||
1734 | // Remove the old video file from the array | ||
1735 | this.VideoFiles = this.VideoFiles.filter(f => f !== currentVideoFile) | ||
1736 | |||
1737 | // Update the database | ||
1738 | currentVideoFile.set('extname', updatedVideoFile.extname) | ||
1739 | currentVideoFile.set('size', updatedVideoFile.size) | ||
1740 | currentVideoFile.set('fps', updatedVideoFile.fps) | ||
1741 | |||
1742 | updatedVideoFile = currentVideoFile | ||
1743 | } | ||
1744 | |||
1745 | const outputPath = this.getVideoFilePath(updatedVideoFile) | ||
1746 | await copy(inputFilePath, outputPath) | ||
1747 | |||
1748 | await this.createTorrentAndSetInfoHash(updatedVideoFile) | ||
1749 | |||
1750 | await updatedVideoFile.save() | ||
1751 | |||
1752 | this.VideoFiles.push(updatedVideoFile) | ||
1753 | } | ||
1754 | |||
1755 | getOriginalFileResolution () { | 1413 | getOriginalFileResolution () { |
1756 | const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) | 1414 | const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) |
1757 | 1415 | ||
1758 | return getVideoFileResolution(originalFilePath) | 1416 | return getVideoFileResolution(originalFilePath) |
1759 | } | 1417 | } |
1760 | 1418 | ||
1761 | getDescriptionPath () { | 1419 | getDescriptionAPIPath () { |
1762 | return `/api/${API_VERSION}/videos/${this.uuid}/description` | 1420 | return `/api/${API_VERSION}/videos/${this.uuid}/description` |
1763 | } | 1421 | } |
1764 | 1422 | ||
@@ -1786,11 +1444,6 @@ export class VideoModel extends Model<VideoModel> { | |||
1786 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | 1444 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) |
1787 | } | 1445 | } |
1788 | 1446 | ||
1789 | getActivityStreamDuration () { | ||
1790 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
1791 | return 'PT' + this.duration + 'S' | ||
1792 | } | ||
1793 | |||
1794 | isOutdated () { | 1447 | isOutdated () { |
1795 | if (this.isOwned()) return false | 1448 | if (this.isOwned()) return false |
1796 | 1449 | ||
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index b2922c5da..f5a19c5ea 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts | |||
@@ -45,7 +45,9 @@ describe('Test jobs', function () { | |||
45 | expect(res.body.total).to.be.above(2) | 45 | expect(res.body.total).to.be.above(2) |
46 | expect(res.body.data).to.have.lengthOf(1) | 46 | expect(res.body.data).to.have.lengthOf(1) |
47 | 47 | ||
48 | const job = res.body.data[0] | 48 | let job = res.body.data[0] |
49 | // Skip repeat jobs | ||
50 | if (job.type === 'videos-views') job = res.body.data[1] | ||
49 | 51 | ||
50 | expect(job.state).to.equal('completed') | 52 | expect(job.state).to.equal('completed') |
51 | expect(job.type).to.equal('activitypub-follow') | 53 | expect(job.type).to.equal('activitypub-follow') |
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index c0ec75a45..6ce4b9dd1 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts | |||
@@ -6,15 +6,16 @@ import { VideoDetails } from '../../../../shared/models/videos' | |||
6 | import { | 6 | import { |
7 | doubleFollow, | 7 | doubleFollow, |
8 | flushAndRunMultipleServers, | 8 | flushAndRunMultipleServers, |
9 | flushTests, | ||
10 | getFollowingListPaginationAndSort, | 9 | getFollowingListPaginationAndSort, |
11 | getVideo, | 10 | getVideo, |
11 | immutableAssign, | ||
12 | killallServers, | 12 | killallServers, |
13 | root, | ||
13 | ServerInfo, | 14 | ServerInfo, |
14 | setAccessTokensToServers, | 15 | setAccessTokensToServers, |
15 | uploadVideo, | 16 | uploadVideo, |
16 | wait, | 17 | viewVideo, |
17 | root, viewVideo | 18 | wait |
18 | } from '../../utils' | 19 | } from '../../utils' |
19 | import { waitJobs } from '../../utils/server/jobs' | 20 | import { waitJobs } from '../../utils/server/jobs' |
20 | import * as magnetUtil from 'magnet-uri' | 21 | import * as magnetUtil from 'magnet-uri' |
@@ -22,9 +23,16 @@ import { updateRedundancy } from '../../utils/server/redundancy' | |||
22 | import { ActorFollow } from '../../../../shared/models/actors' | 23 | import { ActorFollow } from '../../../../shared/models/actors' |
23 | import { readdir } from 'fs-extra' | 24 | import { readdir } from 'fs-extra' |
24 | import { join } from 'path' | 25 | import { join } from 'path' |
26 | import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' | ||
27 | import { getStats } from '../../utils/server/stats' | ||
28 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' | ||
25 | 29 | ||
26 | const expect = chai.expect | 30 | const expect = chai.expect |
27 | 31 | ||
32 | let servers: ServerInfo[] = [] | ||
33 | let video1Server2UUID: string | ||
34 | let video2Server2UUID: string | ||
35 | |||
28 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { | 36 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { |
29 | const parsed = magnetUtil.decode(file.magnetUri) | 37 | const parsed = magnetUtil.decode(file.magnetUri) |
30 | 38 | ||
@@ -34,84 +42,105 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe | |||
34 | } | 42 | } |
35 | } | 43 | } |
36 | 44 | ||
37 | describe('Test videos redundancy', function () { | 45 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { |
38 | let servers: ServerInfo[] = [] | 46 | const config = { |
39 | let video1Server2UUID: string | 47 | redundancy: { |
40 | let video2Server2UUID: string | 48 | videos: { |
41 | 49 | check_interval: '5 seconds', | |
42 | before(async function () { | 50 | strategies: [ |
43 | this.timeout(120000) | 51 | immutableAssign({ |
44 | 52 | strategy: strategy, | |
45 | servers = await flushAndRunMultipleServers(3) | 53 | size: '100KB' |
54 | }, additionalParams) | ||
55 | ] | ||
56 | } | ||
57 | } | ||
58 | } | ||
59 | servers = await flushAndRunMultipleServers(3, config) | ||
46 | 60 | ||
47 | // Get the access tokens | 61 | // Get the access tokens |
48 | await setAccessTokensToServers(servers) | 62 | await setAccessTokensToServers(servers) |
49 | 63 | ||
50 | { | 64 | { |
51 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | 65 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) |
52 | video1Server2UUID = res.body.video.uuid | 66 | video1Server2UUID = res.body.video.uuid |
53 | 67 | ||
54 | await viewVideo(servers[1].url, video1Server2UUID) | 68 | await viewVideo(servers[ 1 ].url, video1Server2UUID) |
55 | } | 69 | } |
56 | 70 | ||
57 | { | 71 | { |
58 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | 72 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) |
59 | video2Server2UUID = res.body.video.uuid | 73 | video2Server2UUID = res.body.video.uuid |
60 | } | 74 | } |
61 | 75 | ||
62 | await waitJobs(servers) | 76 | await waitJobs(servers) |
63 | 77 | ||
64 | // Server 1 and server 2 follow each other | 78 | // Server 1 and server 2 follow each other |
65 | await doubleFollow(servers[0], servers[1]) | 79 | await doubleFollow(servers[ 0 ], servers[ 1 ]) |
66 | // Server 1 and server 3 follow each other | 80 | // Server 1 and server 3 follow each other |
67 | await doubleFollow(servers[0], servers[2]) | 81 | await doubleFollow(servers[ 0 ], servers[ 2 ]) |
68 | // Server 2 and server 3 follow each other | 82 | // Server 2 and server 3 follow each other |
69 | await doubleFollow(servers[1], servers[2]) | 83 | await doubleFollow(servers[ 1 ], servers[ 2 ]) |
70 | 84 | ||
71 | await waitJobs(servers) | 85 | await waitJobs(servers) |
72 | }) | 86 | } |
73 | 87 | ||
74 | it('Should have 1 webseed on the first video', async function () { | 88 | async function check1WebSeed (strategy: VideoRedundancyStrategy) { |
75 | const webseeds = [ | 89 | const webseeds = [ |
76 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | 90 | 'http://localhost:9002/static/webseed/' + video1Server2UUID |
77 | ] | 91 | ] |
78 | 92 | ||
79 | for (const server of servers) { | 93 | for (const server of servers) { |
94 | { | ||
80 | const res = await getVideo(server.url, video1Server2UUID) | 95 | const res = await getVideo(server.url, video1Server2UUID) |
81 | 96 | ||
82 | const video: VideoDetails = res.body | 97 | const video: VideoDetails = res.body |
83 | video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) | 98 | video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) |
84 | } | 99 | } |
85 | }) | ||
86 | 100 | ||
87 | it('Should enable redundancy on server 1', async function () { | 101 | { |
88 | await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) | 102 | const res = await getStats(server.url) |
103 | const data: ServerStats = res.body | ||
89 | 104 | ||
90 | const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') | 105 | expect(data.videosRedundancy).to.have.lengthOf(1) |
91 | const follows: ActorFollow[] = res.body.data | ||
92 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | ||
93 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | ||
94 | 106 | ||
95 | expect(server3).to.not.be.undefined | 107 | const stat = data.videosRedundancy[0] |
96 | expect(server3.following.hostRedundancyAllowed).to.be.false | 108 | expect(stat.strategy).to.equal(strategy) |
109 | expect(stat.totalSize).to.equal(102400) | ||
110 | expect(stat.totalUsed).to.equal(0) | ||
111 | expect(stat.totalVideoFiles).to.equal(0) | ||
112 | expect(stat.totalVideos).to.equal(0) | ||
113 | } | ||
114 | } | ||
115 | } | ||
97 | 116 | ||
98 | expect(server2).to.not.be.undefined | 117 | async function enableRedundancy () { |
99 | expect(server2.following.hostRedundancyAllowed).to.be.true | 118 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) |
100 | }) | ||
101 | 119 | ||
102 | it('Should have 2 webseed on the first video', async function () { | 120 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') |
103 | this.timeout(40000) | 121 | const follows: ActorFollow[] = res.body.data |
122 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | ||
123 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | ||
104 | 124 | ||
105 | await waitJobs(servers) | 125 | expect(server3).to.not.be.undefined |
106 | await wait(15000) | 126 | expect(server3.following.hostRedundancyAllowed).to.be.false |
107 | await waitJobs(servers) | ||
108 | 127 | ||
109 | const webseeds = [ | 128 | expect(server2).to.not.be.undefined |
110 | 'http://localhost:9001/static/webseed/' + video1Server2UUID, | 129 | expect(server2.following.hostRedundancyAllowed).to.be.true |
111 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | 130 | } |
112 | ] | 131 | |
132 | async function check2Webseeds (strategy: VideoRedundancyStrategy) { | ||
133 | await waitJobs(servers) | ||
134 | await wait(15000) | ||
135 | await waitJobs(servers) | ||
113 | 136 | ||
114 | for (const server of servers) { | 137 | const webseeds = [ |
138 | 'http://localhost:9001/static/webseed/' + video1Server2UUID, | ||
139 | 'http://localhost:9002/static/webseed/' + video1Server2UUID | ||
140 | ] | ||
141 | |||
142 | for (const server of servers) { | ||
143 | { | ||
115 | const res = await getVideo(server.url, video1Server2UUID) | 144 | const res = await getVideo(server.url, video1Server2UUID) |
116 | 145 | ||
117 | const video: VideoDetails = res.body | 146 | const video: VideoDetails = res.body |
@@ -120,21 +149,137 @@ describe('Test videos redundancy', function () { | |||
120 | checkMagnetWebseeds(file, webseeds) | 149 | checkMagnetWebseeds(file, webseeds) |
121 | } | 150 | } |
122 | } | 151 | } |
152 | } | ||
123 | 153 | ||
124 | const files = await readdir(join(root(), 'test1', 'videos')) | 154 | const files = await readdir(join(root(), 'test1', 'videos')) |
125 | expect(files).to.have.lengthOf(4) | 155 | expect(files).to.have.lengthOf(4) |
126 | 156 | ||
127 | for (const resolution of [ 240, 360, 480, 720 ]) { | 157 | for (const resolution of [ 240, 360, 480, 720 ]) { |
128 | expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined | 158 | expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined |
129 | } | 159 | } |
160 | |||
161 | { | ||
162 | const res = await getStats(servers[0].url) | ||
163 | const data: ServerStats = res.body | ||
164 | |||
165 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
166 | const stat = data.videosRedundancy[0] | ||
167 | |||
168 | expect(stat.strategy).to.equal(strategy) | ||
169 | expect(stat.totalSize).to.equal(102400) | ||
170 | expect(stat.totalUsed).to.be.at.least(1).and.below(102401) | ||
171 | expect(stat.totalVideoFiles).to.equal(4) | ||
172 | expect(stat.totalVideos).to.equal(1) | ||
173 | } | ||
174 | } | ||
175 | |||
176 | async function cleanServers () { | ||
177 | killallServers(servers) | ||
178 | } | ||
179 | |||
180 | describe('Test videos redundancy', function () { | ||
181 | |||
182 | describe('With most-views strategy', function () { | ||
183 | const strategy = 'most-views' | ||
184 | |||
185 | before(function () { | ||
186 | this.timeout(120000) | ||
187 | |||
188 | return runServers(strategy) | ||
189 | }) | ||
190 | |||
191 | it('Should have 1 webseed on the first video', function () { | ||
192 | return check1WebSeed(strategy) | ||
193 | }) | ||
194 | |||
195 | it('Should enable redundancy on server 1', function () { | ||
196 | return enableRedundancy() | ||
197 | }) | ||
198 | |||
199 | it('Should have 2 webseed on the first video', function () { | ||
200 | this.timeout(40000) | ||
201 | |||
202 | return check2Webseeds(strategy) | ||
203 | }) | ||
204 | |||
205 | after(function () { | ||
206 | return cleanServers() | ||
207 | }) | ||
130 | }) | 208 | }) |
131 | 209 | ||
132 | after(async function () { | 210 | describe('With trending strategy', function () { |
133 | killallServers(servers) | 211 | const strategy = 'trending' |
134 | 212 | ||
135 | // Keep the logs if the test failed | 213 | before(function () { |
136 | if (this['ok']) { | 214 | this.timeout(120000) |
137 | await flushTests() | 215 | |
138 | } | 216 | return runServers(strategy) |
217 | }) | ||
218 | |||
219 | it('Should have 1 webseed on the first video', function () { | ||
220 | return check1WebSeed(strategy) | ||
221 | }) | ||
222 | |||
223 | it('Should enable redundancy on server 1', function () { | ||
224 | return enableRedundancy() | ||
225 | }) | ||
226 | |||
227 | it('Should have 2 webseed on the first video', function () { | ||
228 | this.timeout(40000) | ||
229 | |||
230 | return check2Webseeds(strategy) | ||
231 | }) | ||
232 | |||
233 | after(function () { | ||
234 | return cleanServers() | ||
235 | }) | ||
236 | }) | ||
237 | |||
238 | describe('With recently added strategy', function () { | ||
239 | const strategy = 'recently-added' | ||
240 | |||
241 | before(function () { | ||
242 | this.timeout(120000) | ||
243 | |||
244 | return runServers(strategy, { minViews: 3 }) | ||
245 | }) | ||
246 | |||
247 | it('Should have 1 webseed on the first video', function () { | ||
248 | return check1WebSeed(strategy) | ||
249 | }) | ||
250 | |||
251 | it('Should enable redundancy on server 1', function () { | ||
252 | return enableRedundancy() | ||
253 | }) | ||
254 | |||
255 | it('Should still have 1 webseed on the first video', async function () { | ||
256 | this.timeout(40000) | ||
257 | |||
258 | await waitJobs(servers) | ||
259 | await wait(15000) | ||
260 | await waitJobs(servers) | ||
261 | |||
262 | return check1WebSeed(strategy) | ||
263 | }) | ||
264 | |||
265 | it('Should view 2 times the first video', async function () { | ||
266 | this.timeout(40000) | ||
267 | |||
268 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | ||
269 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | ||
270 | |||
271 | await wait(10000) | ||
272 | await waitJobs(servers) | ||
273 | }) | ||
274 | |||
275 | it('Should have 2 webseed on the first video', function () { | ||
276 | this.timeout(40000) | ||
277 | |||
278 | return check2Webseeds(strategy) | ||
279 | }) | ||
280 | |||
281 | after(function () { | ||
282 | return cleanServers() | ||
283 | }) | ||
139 | }) | 284 | }) |
140 | }) | 285 | }) |
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index fc9b88805..cb229e876 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts | |||
@@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs' | |||
21 | 21 | ||
22 | const expect = chai.expect | 22 | const expect = chai.expect |
23 | 23 | ||
24 | describe('Test stats', function () { | 24 | describe('Test stats (excluding redundancy)', function () { |
25 | let servers: ServerInfo[] = [] | 25 | let servers: ServerInfo[] = [] |
26 | 26 | ||
27 | before(async function () { | 27 | before(async function () { |
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts index 1372c03c3..26ab4e1bb 100644 --- a/server/tests/utils/server/servers.ts +++ b/server/tests/utils/server/servers.ts | |||
@@ -35,7 +35,7 @@ interface ServerInfo { | |||
35 | } | 35 | } |
36 | } | 36 | } |
37 | 37 | ||
38 | function flushAndRunMultipleServers (totalServers) { | 38 | function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) { |
39 | let apps = [] | 39 | let apps = [] |
40 | let i = 0 | 40 | let i = 0 |
41 | 41 | ||
@@ -51,10 +51,7 @@ function flushAndRunMultipleServers (totalServers) { | |||
51 | flushTests() | 51 | flushTests() |
52 | .then(() => { | 52 | .then(() => { |
53 | for (let j = 1; j <= totalServers; j++) { | 53 | for (let j = 1; j <= totalServers; j++) { |
54 | // For the virtual buffer | 54 | runServer(j, configOverride).then(app => anotherServerDone(j, app)) |
55 | setTimeout(() => { | ||
56 | runServer(j).then(app => anotherServerDone(j, app)) | ||
57 | }, 1000 * (j - 1)) | ||
58 | } | 55 | } |
59 | }) | 56 | }) |
60 | }) | 57 | }) |
diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts index 9cdec6cff..01989d952 100644 --- a/server/tests/utils/server/stats.ts +++ b/server/tests/utils/server/stats.ts | |||
@@ -1,11 +1,16 @@ | |||
1 | import { makeGetRequest } from '../' | 1 | import { makeGetRequest } from '../' |
2 | 2 | ||
3 | function getStats (url: string) { | 3 | function getStats (url: string, useCache = false) { |
4 | const path = '/api/v1/server/stats' | 4 | const path = '/api/v1/server/stats' |
5 | 5 | ||
6 | const query = { | ||
7 | t: useCache ? undefined : new Date().getTime() | ||
8 | } | ||
9 | |||
6 | return makeGetRequest({ | 10 | return makeGetRequest({ |
7 | url, | 11 | url, |
8 | path, | 12 | path, |
13 | query, | ||
9 | statusCodeExpected: 200 | 14 | statusCodeExpected: 200 |
10 | }) | 15 | }) |
11 | } | 16 | } |