diff options
53 files changed, 331 insertions, 1245 deletions
diff --git a/package.json b/package.json index a49b4d800..68df1cb9b 100644 --- a/package.json +++ b/package.json | |||
@@ -77,7 +77,6 @@ | |||
77 | "pg": "^6.4.2", | 77 | "pg": "^6.4.2", |
78 | "pg-hstore": "^2.3.2", | 78 | "pg-hstore": "^2.3.2", |
79 | "request": "^2.81.0", | 79 | "request": "^2.81.0", |
80 | "request-replay": "^1.0.2", | ||
81 | "rimraf": "^2.5.4", | 80 | "rimraf": "^2.5.4", |
82 | "safe-buffer": "^5.0.1", | 81 | "safe-buffer": "^5.0.1", |
83 | "scripty": "^1.5.0", | 82 | "scripty": "^1.5.0", |
diff --git a/scripts/update-host.ts b/scripts/update-host.ts index 06d84a658..7c46dc52b 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts | |||
@@ -1,11 +1,11 @@ | |||
1 | import * as Promise from 'bluebird' | ||
2 | |||
3 | import { database as db } from '../server/initializers/database' | 1 | import { database as db } from '../server/initializers/database' |
4 | import { hasFriends } from '../server/lib/friends' | 2 | // import { hasFriends } from '../server/lib/friends' |
5 | 3 | ||
6 | db.init(true) | 4 | db.init(true) |
7 | .then(() => { | 5 | .then(() => { |
8 | return hasFriends() | 6 | // FIXME: check if has followers |
7 | // return hasFriends() | ||
8 | return true | ||
9 | }) | 9 | }) |
10 | .then(itHasFriends => { | 10 | .then(itHasFriends => { |
11 | if (itHasFriends === true) { | 11 | if (itHasFriends === true) { |
@@ -46,7 +46,7 @@ db.init(false).then(() => onDatabaseInitDone()) | |||
46 | 46 | ||
47 | // ----------- PeerTube modules ----------- | 47 | // ----------- PeerTube modules ----------- |
48 | import { migrate, installApplication } from './server/initializers' | 48 | import { migrate, installApplication } from './server/initializers' |
49 | import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib' | 49 | import { httpRequestJobScheduler, transcodingJobScheduler, VideosPreviewCache } from './server/lib' |
50 | import { apiRouter, clientsRouter, staticRouter, servicesRouter } from './server/controllers' | 50 | import { apiRouter, clientsRouter, staticRouter, servicesRouter } from './server/controllers' |
51 | 51 | ||
52 | // ----------- Command line ----------- | 52 | // ----------- Command line ----------- |
@@ -146,19 +146,13 @@ function onDatabaseInitDone () { | |||
146 | const port = CONFIG.LISTEN.PORT | 146 | const port = CONFIG.LISTEN.PORT |
147 | // Run the migration scripts if needed | 147 | // Run the migration scripts if needed |
148 | migrate() | 148 | migrate() |
149 | .then(() => { | 149 | .then(() => installApplication()) |
150 | return installApplication() | ||
151 | }) | ||
152 | .then(() => { | 150 | .then(() => { |
153 | // ----------- Make the server listening ----------- | 151 | // ----------- Make the server listening ----------- |
154 | server.listen(port, function () { | 152 | server.listen(port, () => { |
155 | // Activate the communication with friends | ||
156 | activateSchedulers() | ||
157 | |||
158 | // Activate job scheduler | ||
159 | JobScheduler.Instance.activate() | ||
160 | |||
161 | VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) | 153 | VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) |
154 | httpRequestJobScheduler.activate() | ||
155 | transcodingJobScheduler.activate() | ||
162 | 156 | ||
163 | logger.info('Server listening on port %d', port) | 157 | logger.info('Server listening on port %d', port) |
164 | logger.info('Web server: %s', CONFIG.WEBSERVER.URL) | 158 | logger.info('Web server: %s', CONFIG.WEBSERVER.URL) |
diff --git a/server/controllers/activitypub/index.ts b/server/controllers/activitypub/index.ts index 7a4602b37..2b0e2a489 100644 --- a/server/controllers/activitypub/index.ts +++ b/server/controllers/activitypub/index.ts | |||
@@ -2,10 +2,12 @@ import * as express from 'express' | |||
2 | 2 | ||
3 | import { badRequest } from '../../helpers' | 3 | import { badRequest } from '../../helpers' |
4 | import { inboxRouter } from './inbox' | 4 | import { inboxRouter } from './inbox' |
5 | import { activityPubClientRouter } from './client' | ||
5 | 6 | ||
6 | const remoteRouter = express.Router() | 7 | const remoteRouter = express.Router() |
7 | 8 | ||
8 | remoteRouter.use('/inbox', inboxRouter) | 9 | remoteRouter.use('/inbox', inboxRouter) |
10 | remoteRouter.use('/', activityPubClientRouter) | ||
9 | remoteRouter.use('/*', badRequest) | 11 | remoteRouter.use('/*', badRequest) |
10 | 12 | ||
11 | // --------------------------------------------------------------------------- | 13 | // --------------------------------------------------------------------------- |
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index a9205b33c..2e949d531 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts | |||
@@ -5,8 +5,6 @@ import { badRequest } from '../../helpers' | |||
5 | import { oauthClientsRouter } from './oauth-clients' | 5 | import { oauthClientsRouter } from './oauth-clients' |
6 | import { configRouter } from './config' | 6 | import { configRouter } from './config' |
7 | import { podsRouter } from './pods' | 7 | import { podsRouter } from './pods' |
8 | import { remoteRouter } from './remote' | ||
9 | import { requestSchedulerRouter } from './request-schedulers' | ||
10 | import { usersRouter } from './users' | 8 | import { usersRouter } from './users' |
11 | import { videosRouter } from './videos' | 9 | import { videosRouter } from './videos' |
12 | 10 | ||
@@ -15,8 +13,6 @@ const apiRouter = express.Router() | |||
15 | apiRouter.use('/oauth-clients', oauthClientsRouter) | 13 | apiRouter.use('/oauth-clients', oauthClientsRouter) |
16 | apiRouter.use('/config', configRouter) | 14 | apiRouter.use('/config', configRouter) |
17 | apiRouter.use('/pods', podsRouter) | 15 | apiRouter.use('/pods', podsRouter) |
18 | apiRouter.use('/remote', remoteRouter) | ||
19 | apiRouter.use('/request-schedulers', requestSchedulerRouter) | ||
20 | apiRouter.use('/users', usersRouter) | 16 | apiRouter.use('/users', usersRouter) |
21 | apiRouter.use('/videos', videosRouter) | 17 | apiRouter.use('/videos', videosRouter) |
22 | apiRouter.use('/ping', pong) | 18 | apiRouter.use('/ping', pong) |
diff --git a/server/controllers/api/pods.ts b/server/controllers/api/pods.ts index b44cd6b83..43df3f66f 100644 --- a/server/controllers/api/pods.ts +++ b/server/controllers/api/pods.ts | |||
@@ -1,26 +1,7 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | 2 | import { getFormattedObjects } from '../../helpers' | |
3 | import { database as db } from '../../initializers/database' | 3 | import { database as db } from '../../initializers/database' |
4 | import { logger, getFormattedObjects } from '../../helpers' | 4 | import { asyncMiddleware, paginationValidator, podsSortValidator, setPagination, setPodsSort } from '../../middlewares' |
5 | import { | ||
6 | makeFriends, | ||
7 | quitFriends, | ||
8 | removeFriend | ||
9 | } from '../../lib' | ||
10 | import { | ||
11 | authenticate, | ||
12 | ensureUserHasRight, | ||
13 | makeFriendsValidator, | ||
14 | setBodyHostsPort, | ||
15 | podRemoveValidator, | ||
16 | paginationValidator, | ||
17 | setPagination, | ||
18 | setPodsSort, | ||
19 | podsSortValidator, | ||
20 | asyncMiddleware | ||
21 | } from '../../middlewares' | ||
22 | import { PodInstance } from '../../models' | ||
23 | import { UserRight } from '../../../shared' | ||
24 | 5 | ||
25 | const podsRouter = express.Router() | 6 | const podsRouter = express.Router() |
26 | 7 | ||
@@ -31,24 +12,6 @@ podsRouter.get('/', | |||
31 | setPagination, | 12 | setPagination, |
32 | asyncMiddleware(listPods) | 13 | asyncMiddleware(listPods) |
33 | ) | 14 | ) |
34 | podsRouter.post('/make-friends', | ||
35 | authenticate, | ||
36 | ensureUserHasRight(UserRight.MANAGE_PODS), | ||
37 | makeFriendsValidator, | ||
38 | setBodyHostsPort, | ||
39 | asyncMiddleware(makeFriendsController) | ||
40 | ) | ||
41 | podsRouter.get('/quit-friends', | ||
42 | authenticate, | ||
43 | ensureUserHasRight(UserRight.MANAGE_PODS), | ||
44 | asyncMiddleware(quitFriendsController) | ||
45 | ) | ||
46 | podsRouter.delete('/:id', | ||
47 | authenticate, | ||
48 | ensureUserHasRight(UserRight.MANAGE_PODS), | ||
49 | podRemoveValidator, | ||
50 | asyncMiddleware(removeFriendController) | ||
51 | ) | ||
52 | 15 | ||
53 | // --------------------------------------------------------------------------- | 16 | // --------------------------------------------------------------------------- |
54 | 17 | ||
@@ -63,28 +26,3 @@ async function listPods (req: express.Request, res: express.Response, next: expr | |||
63 | 26 | ||
64 | return res.json(getFormattedObjects(resultList.data, resultList.total)) | 27 | return res.json(getFormattedObjects(resultList.data, resultList.total)) |
65 | } | 28 | } |
66 | |||
67 | async function makeFriendsController (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
68 | const hosts = req.body.hosts as string[] | ||
69 | |||
70 | // Don't wait the process that could be long | ||
71 | makeFriends(hosts) | ||
72 | .then(() => logger.info('Made friends!')) | ||
73 | .catch(err => logger.error('Could not make friends.', err)) | ||
74 | |||
75 | return res.type('json').status(204).end() | ||
76 | } | ||
77 | |||
78 | async function quitFriendsController (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
79 | await quitFriends() | ||
80 | |||
81 | return res.type('json').status(204).end() | ||
82 | } | ||
83 | |||
84 | async function removeFriendController (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
85 | const pod = res.locals.pod as PodInstance | ||
86 | |||
87 | await removeFriend(pod) | ||
88 | |||
89 | return res.type('json').status(204).end() | ||
90 | } | ||
diff --git a/server/controllers/api/request-schedulers.ts b/server/controllers/api/request-schedulers.ts deleted file mode 100644 index 4c8fbe18b..000000000 --- a/server/controllers/api/request-schedulers.ts +++ /dev/null | |||
@@ -1,53 +0,0 @@ | |||
1 | import * as express from 'express' | ||
2 | import * as Bluebird from 'bluebird' | ||
3 | |||
4 | import { | ||
5 | AbstractRequestScheduler, | ||
6 | getRequestScheduler, | ||
7 | getRequestVideoQaduScheduler, | ||
8 | getRequestVideoEventScheduler | ||
9 | } from '../../lib' | ||
10 | import { authenticate, ensureUserHasRight, asyncMiddleware } from '../../middlewares' | ||
11 | import { RequestSchedulerStatsAttributes, UserRight } from '../../../shared' | ||
12 | |||
13 | const requestSchedulerRouter = express.Router() | ||
14 | |||
15 | requestSchedulerRouter.get('/stats', | ||
16 | authenticate, | ||
17 | ensureUserHasRight(UserRight.MANAGE_REQUEST_SCHEDULERS), | ||
18 | asyncMiddleware(getRequestSchedulersStats) | ||
19 | ) | ||
20 | |||
21 | // --------------------------------------------------------------------------- | ||
22 | |||
23 | export { | ||
24 | requestSchedulerRouter | ||
25 | } | ||
26 | |||
27 | // --------------------------------------------------------------------------- | ||
28 | |||
29 | async function getRequestSchedulersStats (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
30 | const result = await Bluebird.props({ | ||
31 | requestScheduler: buildRequestSchedulerStats(getRequestScheduler()), | ||
32 | requestVideoQaduScheduler: buildRequestSchedulerStats(getRequestVideoQaduScheduler()), | ||
33 | requestVideoEventScheduler: buildRequestSchedulerStats(getRequestVideoEventScheduler()) | ||
34 | }) | ||
35 | |||
36 | return res.json(result) | ||
37 | } | ||
38 | |||
39 | // --------------------------------------------------------------------------- | ||
40 | |||
41 | async function buildRequestSchedulerStats (requestScheduler: AbstractRequestScheduler<any>) { | ||
42 | const count = await requestScheduler.remainingRequestsCount() | ||
43 | |||
44 | const result: RequestSchedulerStatsAttributes = { | ||
45 | totalRequests: count, | ||
46 | requestsLimitPods: requestScheduler.limitPods, | ||
47 | requestsLimitPerPod: requestScheduler.limitPerPod, | ||
48 | remainingMilliSeconds: requestScheduler.remainingMilliSeconds(), | ||
49 | milliSecondsInterval: requestScheduler.requestInterval | ||
50 | } | ||
51 | |||
52 | return result | ||
53 | } | ||
diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts index 9ec6feb57..41ffb64cb 100644 --- a/server/controllers/api/users.ts +++ b/server/controllers/api/users.ts | |||
@@ -1,37 +1,29 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | 2 | import { UserCreate, UserRight, UserRole, UserUpdate, UserUpdateMe, UserVideoRate as FormattedUserVideoRate } from '../../../shared' | |
3 | import { database as db, CONFIG } from '../../initializers' | 3 | import { getFormattedObjects, logger, retryTransactionWrapper } from '../../helpers' |
4 | import { logger, getFormattedObjects, retryTransactionWrapper } from '../../helpers' | 4 | import { CONFIG, database as db } from '../../initializers' |
5 | import { createUserAccountAndChannel } from '../../lib' | ||
5 | import { | 6 | import { |
7 | asyncMiddleware, | ||
6 | authenticate, | 8 | authenticate, |
7 | ensureUserHasRight, | 9 | ensureUserHasRight, |
8 | ensureUserRegistrationAllowed, | 10 | ensureUserRegistrationAllowed, |
9 | usersAddValidator, | ||
10 | usersRegisterValidator, | ||
11 | usersUpdateValidator, | ||
12 | usersUpdateMeValidator, | ||
13 | usersRemoveValidator, | ||
14 | usersVideoRatingValidator, | ||
15 | usersGetValidator, | ||
16 | paginationValidator, | 11 | paginationValidator, |
17 | setPagination, | 12 | setPagination, |
18 | usersSortValidator, | ||
19 | setUsersSort, | 13 | setUsersSort, |
20 | token, | 14 | token, |
21 | asyncMiddleware | 15 | usersAddValidator, |
16 | usersGetValidator, | ||
17 | usersRegisterValidator, | ||
18 | usersRemoveValidator, | ||
19 | usersSortValidator, | ||
20 | usersUpdateMeValidator, | ||
21 | usersUpdateValidator, | ||
22 | usersVideoRatingValidator | ||
22 | } from '../../middlewares' | 23 | } from '../../middlewares' |
23 | import { | ||
24 | UserVideoRate as FormattedUserVideoRate, | ||
25 | UserCreate, | ||
26 | UserUpdate, | ||
27 | UserUpdateMe, | ||
28 | UserRole, | ||
29 | UserRight | ||
30 | } from '../../../shared' | ||
31 | import { createUserAccountAndChannel } from '../../lib' | ||
32 | import { UserInstance } from '../../models' | ||
33 | import { videosSortValidator } from '../../middlewares/validators/sort' | ||
34 | import { setVideosSort } from '../../middlewares/sort' | 24 | import { setVideosSort } from '../../middlewares/sort' |
25 | import { videosSortValidator } from '../../middlewares/validators/sort' | ||
26 | import { UserInstance } from '../../models' | ||
35 | 27 | ||
36 | const usersRouter = express.Router() | 28 | const usersRouter = express.Router() |
37 | 29 | ||
@@ -176,9 +168,9 @@ function getUser (req: express.Request, res: express.Response, next: express.Nex | |||
176 | 168 | ||
177 | async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { | 169 | async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { |
178 | const videoId = +req.params.videoId | 170 | const videoId = +req.params.videoId |
179 | const userId = +res.locals.oauth.token.User.id | 171 | const accountId = +res.locals.oauth.token.User.Account.id |
180 | 172 | ||
181 | const ratingObj = await db.UserVideoRate.load(userId, videoId, null) | 173 | const ratingObj = await db.AccountVideoRate.load(accountId, videoId, null) |
182 | const rating = ratingObj ? ratingObj.type : 'none' | 174 | const rating = ratingObj ? ratingObj.type : 'none' |
183 | 175 | ||
184 | const json: FormattedUserVideoRate = { | 176 | const json: FormattedUserVideoRate = { |
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 04349042b..7a3471116 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | 2 | ||
3 | import { database as db } from '../../../initializers/database' | 3 | import { database as db } from '../../../initializers/database' |
4 | import * as friends from '../../../lib/friends' | ||
5 | import { | 4 | import { |
6 | logger, | 5 | logger, |
7 | getFormattedObjects, | 6 | getFormattedObjects, |
@@ -84,7 +83,8 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { | |||
84 | videoUUID: videoInstance.uuid | 83 | videoUUID: videoInstance.uuid |
85 | } | 84 | } |
86 | 85 | ||
87 | await friends.reportAbuseVideoToFriend(reportData, videoInstance, t) | 86 | // await friends.reportAbuseVideoToFriend(reportData, videoInstance, t) |
87 | // TODO: send abuse to origin pod | ||
88 | } | 88 | } |
89 | }) | 89 | }) |
90 | 90 | ||
diff --git a/server/controllers/api/videos/channel.ts b/server/controllers/api/videos/channel.ts index 4d1f03903..656bc3129 100644 --- a/server/controllers/api/videos/channel.ts +++ b/server/controllers/api/videos/channel.ts | |||
@@ -1,31 +1,23 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | 2 | import { VideoChannelCreate, VideoChannelUpdate } from '../../../../shared' | |
3 | import { getFormattedObjects, logger, resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers' | ||
3 | import { database as db } from '../../../initializers' | 4 | import { database as db } from '../../../initializers' |
5 | import { createVideoChannel } from '../../../lib' | ||
4 | import { | 6 | import { |
5 | logger, | 7 | asyncMiddleware, |
6 | getFormattedObjects, | ||
7 | retryTransactionWrapper, | ||
8 | resetSequelizeInstance | ||
9 | } from '../../../helpers' | ||
10 | import { | ||
11 | authenticate, | 8 | authenticate, |
9 | listVideoAccountChannelsValidator, | ||
12 | paginationValidator, | 10 | paginationValidator, |
13 | videoChannelsSortValidator, | ||
14 | videoChannelsAddValidator, | ||
15 | setVideoChannelsSort, | ||
16 | setPagination, | 11 | setPagination, |
17 | videoChannelsRemoveValidator, | 12 | setVideoChannelsSort, |
18 | videoChannelGetValidator, | 13 | videoChannelGetValidator, |
19 | videoChannelsUpdateValidator, | 14 | videoChannelsAddValidator, |
20 | listVideoAccountChannelsValidator, | 15 | videoChannelsRemoveValidator, |
21 | asyncMiddleware | 16 | videoChannelsSortValidator, |
17 | videoChannelsUpdateValidator | ||
22 | } from '../../../middlewares' | 18 | } from '../../../middlewares' |
23 | import { | 19 | import { AccountInstance, VideoChannelInstance } from '../../../models' |
24 | createVideoChannel, | 20 | import { sendUpdateVideoChannel } from '../../../lib/activitypub/send-request' |
25 | updateVideoChannelToFriends | ||
26 | } from '../../../lib' | ||
27 | import { VideoChannelInstance, AccountInstance } from '../../../models' | ||
28 | import { VideoChannelCreate, VideoChannelUpdate } from '../../../../shared' | ||
29 | 21 | ||
30 | const videoChannelRouter = express.Router() | 22 | const videoChannelRouter = express.Router() |
31 | 23 | ||
@@ -137,11 +129,8 @@ async function updateVideoChannel (req: express.Request, res: express.Response) | |||
137 | if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.set('description', videoChannelInfoToUpdate.description) | 129 | if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.set('description', videoChannelInfoToUpdate.description) |
138 | 130 | ||
139 | await videoChannelInstance.save(sequelizeOptions) | 131 | await videoChannelInstance.save(sequelizeOptions) |
140 | const json = videoChannelInstance.toUpdateRemoteJSON() | ||
141 | |||
142 | // Now we'll update the video channel's meta data to our friends | ||
143 | return updateVideoChannelToFriends(json, t) | ||
144 | 132 | ||
133 | await sendUpdateVideoChannel(videoChannelInstance, t) | ||
145 | }) | 134 | }) |
146 | 135 | ||
147 | logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.uuid) | 136 | logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.uuid) |
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 9ad84609f..063839223 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -1,57 +1,41 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import * as multer from 'multer' | 2 | import * as multer from 'multer' |
3 | import { extname, join } from 'path' | 3 | import { extname, join } from 'path' |
4 | 4 | import { VideoCreate, VideoPrivacy, VideoUpdate } from '../../../../shared' | |
5 | import { database as db } from '../../../initializers/database' | ||
6 | import { | 5 | import { |
7 | CONFIG, | 6 | fetchRemoteVideoDescription, |
8 | REQUEST_VIDEO_QADU_TYPES, | 7 | generateRandomString, |
9 | REQUEST_VIDEO_EVENT_TYPES, | 8 | getFormattedObjects, |
10 | VIDEO_CATEGORIES, | 9 | getVideoFileHeight, |
11 | VIDEO_LICENCES, | 10 | logger, |
12 | VIDEO_LANGUAGES, | 11 | renamePromise, |
13 | VIDEO_PRIVACIES, | 12 | resetSequelizeInstance, |
14 | VIDEO_MIMETYPE_EXT | 13 | retryTransactionWrapper |
15 | } from '../../../initializers' | 14 | } from '../../../helpers' |
16 | import { | 15 | import { getActivityPubUrl } from '../../../helpers/activitypub' |
17 | addEventToRemoteVideo, | 16 | import { CONFIG, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_MIMETYPE_EXT, VIDEO_PRIVACIES } from '../../../initializers' |
18 | quickAndDirtyUpdateVideoToFriends, | 17 | import { database as db } from '../../../initializers/database' |
19 | addVideoToFriends, | 18 | import { sendAddVideo, sendUpdateVideoChannel } from '../../../lib/activitypub/send-request' |
20 | updateVideoToFriends, | 19 | import { transcodingJobScheduler } from '../../../lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler' |
21 | JobScheduler, | ||
22 | fetchRemoteDescription | ||
23 | } from '../../../lib' | ||
24 | import { | 20 | import { |
21 | asyncMiddleware, | ||
25 | authenticate, | 22 | authenticate, |
26 | paginationValidator, | 23 | paginationValidator, |
27 | videosSortValidator, | ||
28 | setVideosSort, | ||
29 | setPagination, | 24 | setPagination, |
30 | setVideosSearch, | 25 | setVideosSearch, |
31 | videosUpdateValidator, | 26 | setVideosSort, |
32 | videosSearchValidator, | ||
33 | videosAddValidator, | 27 | videosAddValidator, |
34 | videosGetValidator, | 28 | videosGetValidator, |
35 | videosRemoveValidator, | 29 | videosRemoveValidator, |
36 | asyncMiddleware | 30 | videosSearchValidator, |
31 | videosSortValidator, | ||
32 | videosUpdateValidator | ||
37 | } from '../../../middlewares' | 33 | } from '../../../middlewares' |
38 | import { | ||
39 | logger, | ||
40 | retryTransactionWrapper, | ||
41 | generateRandomString, | ||
42 | getFormattedObjects, | ||
43 | renamePromise, | ||
44 | getVideoFileHeight, | ||
45 | resetSequelizeInstance | ||
46 | } from '../../../helpers' | ||
47 | import { VideoInstance } from '../../../models' | 34 | import { VideoInstance } from '../../../models' |
48 | import { VideoCreate, VideoUpdate, VideoPrivacy } from '../../../../shared' | ||
49 | |||
50 | import { abuseVideoRouter } from './abuse' | 35 | import { abuseVideoRouter } from './abuse' |
51 | import { blacklistRouter } from './blacklist' | 36 | import { blacklistRouter } from './blacklist' |
52 | import { rateVideoRouter } from './rate' | ||
53 | import { videoChannelRouter } from './channel' | 37 | import { videoChannelRouter } from './channel' |
54 | import { getActivityPubUrl } from '../../../helpers/activitypub' | 38 | import { rateVideoRouter } from './rate' |
55 | 39 | ||
56 | const videosRouter = express.Router() | 40 | const videosRouter = express.Router() |
57 | 41 | ||
@@ -225,7 +209,7 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi | |||
225 | } | 209 | } |
226 | 210 | ||
227 | tasks.push( | 211 | tasks.push( |
228 | JobScheduler.Instance.createJob(t, 'videoFileOptimizer', dataInput) | 212 | transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput) |
229 | ) | 213 | ) |
230 | } | 214 | } |
231 | await Promise.all(tasks) | 215 | await Promise.all(tasks) |
@@ -252,9 +236,7 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi | |||
252 | // Don't send video to remote pods, it is private | 236 | // Don't send video to remote pods, it is private |
253 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | 237 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined |
254 | 238 | ||
255 | const remoteVideo = await video.toAddRemoteJSON() | 239 | await sendAddVideo(video, t) |
256 | // Now we'll add the video's meta data to our friends | ||
257 | return addVideoToFriends(remoteVideo, t) | ||
258 | }) | 240 | }) |
259 | 241 | ||
260 | logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID) | 242 | logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID) |
@@ -302,14 +284,12 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
302 | 284 | ||
303 | // Now we'll update the video's meta data to our friends | 285 | // Now we'll update the video's meta data to our friends |
304 | if (wasPrivateVideo === false) { | 286 | if (wasPrivateVideo === false) { |
305 | const json = videoInstance.toUpdateRemoteJSON() | 287 | await sendUpdateVideoChannel(videoInstance, t) |
306 | return updateVideoToFriends(json, t) | ||
307 | } | 288 | } |
308 | 289 | ||
309 | // Video is not private anymore, send a create action to remote pods | 290 | // Video is not private anymore, send a create action to remote pods |
310 | if (wasPrivateVideo === true && videoInstance.privacy !== VideoPrivacy.PRIVATE) { | 291 | if (wasPrivateVideo === true && videoInstance.privacy !== VideoPrivacy.PRIVATE) { |
311 | const remoteVideo = await videoInstance.toAddRemoteJSON() | 292 | await sendAddVideo(videoInstance, t) |
312 | return addVideoToFriends(remoteVideo, t) | ||
313 | } | 293 | } |
314 | }) | 294 | }) |
315 | 295 | ||
@@ -324,7 +304,7 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
324 | } | 304 | } |
325 | } | 305 | } |
326 | 306 | ||
327 | function getVideo (req: express.Request, res: express.Response) { | 307 | async function getVideo (req: express.Request, res: express.Response) { |
328 | const videoInstance = res.locals.video | 308 | const videoInstance = res.locals.video |
329 | 309 | ||
330 | if (videoInstance.isOwned()) { | 310 | if (videoInstance.isOwned()) { |
@@ -333,21 +313,11 @@ function getVideo (req: express.Request, res: express.Response) { | |||
333 | // For example, only add a view when a user watch a video during 30s etc | 313 | // For example, only add a view when a user watch a video during 30s etc |
334 | videoInstance.increment('views') | 314 | videoInstance.increment('views') |
335 | .then(() => { | 315 | .then(() => { |
336 | const qaduParams = { | 316 | // TODO: send to followers a notification |
337 | videoId: videoInstance.id, | ||
338 | type: REQUEST_VIDEO_QADU_TYPES.VIEWS | ||
339 | } | ||
340 | return quickAndDirtyUpdateVideoToFriends(qaduParams) | ||
341 | }) | 317 | }) |
342 | .catch(err => logger.error('Cannot add view to video %s.', videoInstance.uuid, err)) | 318 | .catch(err => logger.error('Cannot add view to video %s.', videoInstance.uuid, err)) |
343 | } else { | 319 | } else { |
344 | // Just send the event to our friends | 320 | // TODO: send view event to followers |
345 | const eventParams = { | ||
346 | videoId: videoInstance.id, | ||
347 | type: REQUEST_VIDEO_EVENT_TYPES.VIEWS | ||
348 | } | ||
349 | addEventToRemoteVideo(eventParams) | ||
350 | .catch(err => logger.error('Cannot add event to remote video %s.', videoInstance.uuid, err)) | ||
351 | } | 321 | } |
352 | 322 | ||
353 | // Do not wait the view system | 323 | // Do not wait the view system |
@@ -361,7 +331,7 @@ async function getVideoDescription (req: express.Request, res: express.Response) | |||
361 | if (videoInstance.isOwned()) { | 331 | if (videoInstance.isOwned()) { |
362 | description = videoInstance.description | 332 | description = videoInstance.description |
363 | } else { | 333 | } else { |
364 | description = await fetchRemoteDescription(videoInstance) | 334 | description = await fetchRemoteVideoDescription(videoInstance) |
365 | } | 335 | } |
366 | 336 | ||
367 | return res.json({ description }) | 337 | return res.json({ description }) |
diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts index 727984506..955277d25 100644 --- a/server/controllers/api/videos/rate.ts +++ b/server/controllers/api/videos/rate.ts | |||
@@ -1,25 +1,11 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | |||
3 | import { database as db } from '../../../initializers/database' | ||
4 | import { | ||
5 | logger, | ||
6 | retryTransactionWrapper | ||
7 | } from '../../../helpers' | ||
8 | import { | ||
9 | VIDEO_RATE_TYPES, | ||
10 | REQUEST_VIDEO_EVENT_TYPES, | ||
11 | REQUEST_VIDEO_QADU_TYPES | ||
12 | } from '../../../initializers' | ||
13 | import { | ||
14 | addEventsToRemoteVideo, | ||
15 | quickAndDirtyUpdatesVideoToFriends | ||
16 | } from '../../../lib' | ||
17 | import { | ||
18 | authenticate, | ||
19 | videoRateValidator, | ||
20 | asyncMiddleware | ||
21 | } from '../../../middlewares' | ||
22 | import { UserVideoRateUpdate } from '../../../../shared' | 2 | import { UserVideoRateUpdate } from '../../../../shared' |
3 | import { logger, retryTransactionWrapper } from '../../../helpers' | ||
4 | import { VIDEO_RATE_TYPES } from '../../../initializers' | ||
5 | import { database as db } from '../../../initializers/database' | ||
6 | import { asyncMiddleware, authenticate, videoRateValidator } from '../../../middlewares' | ||
7 | import { AccountInstance } from '../../../models/account/account-interface' | ||
8 | import { VideoInstance } from '../../../models/video/video-interface' | ||
23 | 9 | ||
24 | const rateVideoRouter = express.Router() | 10 | const rateVideoRouter = express.Router() |
25 | 11 | ||
@@ -51,12 +37,12 @@ async function rateVideoRetryWrapper (req: express.Request, res: express.Respons | |||
51 | async function rateVideo (req: express.Request, res: express.Response) { | 37 | async function rateVideo (req: express.Request, res: express.Response) { |
52 | const body: UserVideoRateUpdate = req.body | 38 | const body: UserVideoRateUpdate = req.body |
53 | const rateType = body.rating | 39 | const rateType = body.rating |
54 | const videoInstance = res.locals.video | 40 | const videoInstance: VideoInstance = res.locals.video |
55 | const userInstance = res.locals.oauth.token.User | 41 | const accountInstance: AccountInstance = res.locals.oauth.token.User.Account |
56 | 42 | ||
57 | await db.sequelize.transaction(async t => { | 43 | await db.sequelize.transaction(async t => { |
58 | const sequelizeOptions = { transaction: t } | 44 | const sequelizeOptions = { transaction: t } |
59 | const previousRate = await db.UserVideoRate.load(userInstance.id, videoInstance.id, t) | 45 | const previousRate = await db.AccountVideoRate.load(accountInstance.id, videoInstance.id, t) |
60 | 46 | ||
61 | let likesToIncrement = 0 | 47 | let likesToIncrement = 0 |
62 | let dislikesToIncrement = 0 | 48 | let dislikesToIncrement = 0 |
@@ -79,12 +65,12 @@ async function rateVideo (req: express.Request, res: express.Response) { | |||
79 | } | 65 | } |
80 | } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate | 66 | } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate |
81 | const query = { | 67 | const query = { |
82 | userId: userInstance.id, | 68 | accountId: accountInstance.id, |
83 | videoId: videoInstance.id, | 69 | videoId: videoInstance.id, |
84 | type: rateType | 70 | type: rateType |
85 | } | 71 | } |
86 | 72 | ||
87 | await db.UserVideoRate.create(query, sequelizeOptions) | 73 | await db.AccountVideoRate.create(query, sequelizeOptions) |
88 | } | 74 | } |
89 | 75 | ||
90 | const incrementQuery = { | 76 | const incrementQuery = { |
@@ -96,48 +82,12 @@ async function rateVideo (req: express.Request, res: express.Response) { | |||
96 | // It is useful for the user to have a feedback | 82 | // It is useful for the user to have a feedback |
97 | await videoInstance.increment(incrementQuery, sequelizeOptions) | 83 | await videoInstance.increment(incrementQuery, sequelizeOptions) |
98 | 84 | ||
99 | // Send a event to original pod | ||
100 | if (videoInstance.isOwned() === false) { | 85 | if (videoInstance.isOwned() === false) { |
101 | 86 | // TODO: Send a event to original pod | |
102 | const eventsParams = [] | 87 | } else { |
103 | 88 | // TODO: Send update to followers | |
104 | if (likesToIncrement !== 0) { | ||
105 | eventsParams.push({ | ||
106 | videoId: videoInstance.id, | ||
107 | type: REQUEST_VIDEO_EVENT_TYPES.LIKES, | ||
108 | count: likesToIncrement | ||
109 | }) | ||
110 | } | ||
111 | |||
112 | if (dislikesToIncrement !== 0) { | ||
113 | eventsParams.push({ | ||
114 | videoId: videoInstance.id, | ||
115 | type: REQUEST_VIDEO_EVENT_TYPES.DISLIKES, | ||
116 | count: dislikesToIncrement | ||
117 | }) | ||
118 | } | ||
119 | |||
120 | await addEventsToRemoteVideo(eventsParams, t) | ||
121 | } else { // We own the video, we need to send a quick and dirty update to friends to notify the counts changed | ||
122 | const qadusParams = [] | ||
123 | |||
124 | if (likesToIncrement !== 0) { | ||
125 | qadusParams.push({ | ||
126 | videoId: videoInstance.id, | ||
127 | type: REQUEST_VIDEO_QADU_TYPES.LIKES | ||
128 | }) | ||
129 | } | ||
130 | |||
131 | if (dislikesToIncrement !== 0) { | ||
132 | qadusParams.push({ | ||
133 | videoId: videoInstance.id, | ||
134 | type: REQUEST_VIDEO_QADU_TYPES.DISLIKES | ||
135 | }) | ||
136 | } | ||
137 | |||
138 | await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) | ||
139 | } | 89 | } |
140 | }) | 90 | }) |
141 | 91 | ||
142 | logger.info('User video rate for video %s of user %s updated.', videoInstance.name, userInstance.username) | 92 | logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) |
143 | } | 93 | } |
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 75de2278c..a1493e5c1 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -1,15 +1,15 @@ | |||
1 | import { join } from 'path' | ||
2 | import * as request from 'request' | ||
1 | import * as url from 'url' | 3 | import * as url from 'url' |
2 | 4 | import { ActivityIconObject } from '../../shared/index' | |
3 | import { database as db } from '../initializers' | ||
4 | import { logger } from './logger' | ||
5 | import { doRequest, doRequestAndSaveToFile } from './requests' | ||
6 | import { isRemoteAccountValid } from './custom-validators' | ||
7 | import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' | 5 | import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' |
8 | import { ResultList } from '../../shared/models/result-list.model' | 6 | import { ResultList } from '../../shared/models/result-list.model' |
9 | import { CONFIG } from '../initializers/constants' | 7 | import { database as db, REMOTE_SCHEME } from '../initializers' |
8 | import { CONFIG, STATIC_PATHS } from '../initializers/constants' | ||
10 | import { VideoInstance } from '../models/video/video-interface' | 9 | import { VideoInstance } from '../models/video/video-interface' |
11 | import { ActivityIconObject } from '../../shared/index' | 10 | import { isRemoteAccountValid } from './custom-validators' |
12 | import { join } from 'path' | 11 | import { logger } from './logger' |
12 | import { doRequest, doRequestAndSaveToFile } from './requests' | ||
13 | 13 | ||
14 | function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) { | 14 | function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) { |
15 | const thumbnailName = video.getThumbnailName() | 15 | const thumbnailName = video.getThumbnailName() |
@@ -22,9 +22,10 @@ function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObjec | |||
22 | return doRequestAndSaveToFile(options, thumbnailPath) | 22 | return doRequestAndSaveToFile(options, thumbnailPath) |
23 | } | 23 | } |
24 | 24 | ||
25 | function getActivityPubUrl (type: 'video' | 'videoChannel', uuid: string) { | 25 | function getActivityPubUrl (type: 'video' | 'videoChannel' | 'account', id: string) { |
26 | if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + uuid | 26 | if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + id |
27 | else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + uuid | 27 | else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + id |
28 | else if (type === 'account') return CONFIG.WEBSERVER.URL + '/account/' + id | ||
28 | 29 | ||
29 | return '' | 30 | return '' |
30 | } | 31 | } |
@@ -94,7 +95,24 @@ async function fetchRemoteAccountAndCreatePod (accountUrl: string) { | |||
94 | return { account, pod } | 95 | return { account, pod } |
95 | } | 96 | } |
96 | 97 | ||
97 | function activityPubContextify (data: object) { | 98 | function fetchRemoteVideoPreview (video: VideoInstance) { |
99 | // FIXME: use url | ||
100 | const host = video.VideoChannel.Account.Pod.host | ||
101 | const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) | ||
102 | |||
103 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) | ||
104 | } | ||
105 | |||
106 | async function fetchRemoteVideoDescription (video: VideoInstance) { | ||
107 | const options = { | ||
108 | uri: video.url | ||
109 | } | ||
110 | |||
111 | const { body } = await doRequest(options) | ||
112 | return body.description ? body.description : '' | ||
113 | } | ||
114 | |||
115 | function activityPubContextify <T> (data: T) { | ||
98 | return Object.assign(data,{ | 116 | return Object.assign(data,{ |
99 | '@context': [ | 117 | '@context': [ |
100 | 'https://www.w3.org/ns/activitystreams', | 118 | 'https://www.w3.org/ns/activitystreams', |
@@ -141,7 +159,9 @@ export { | |||
141 | activityPubCollectionPagination, | 159 | activityPubCollectionPagination, |
142 | getActivityPubUrl, | 160 | getActivityPubUrl, |
143 | generateThumbnailFromUrl, | 161 | generateThumbnailFromUrl, |
144 | getOrCreateAccount | 162 | getOrCreateAccount, |
163 | fetchRemoteVideoPreview, | ||
164 | fetchRemoteVideoDescription | ||
145 | } | 165 | } |
146 | 166 | ||
147 | // --------------------------------------------------------------------------- | 167 | // --------------------------------------------------------------------------- |
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 31cedd768..4b1deeadc 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts | |||
@@ -1,16 +1,6 @@ | |||
1 | import * as replay from 'request-replay' | ||
2 | import * as request from 'request' | ||
3 | import * as Promise from 'bluebird' | 1 | import * as Promise from 'bluebird' |
4 | |||
5 | import { | ||
6 | RETRY_REQUESTS, | ||
7 | REMOTE_SCHEME, | ||
8 | CONFIG | ||
9 | } from '../initializers' | ||
10 | import { PodInstance } from '../models' | ||
11 | import { PodSignature } from '../../shared' | ||
12 | import { signObject } from './peertube-crypto' | ||
13 | import { createWriteStream } from 'fs' | 2 | import { createWriteStream } from 'fs' |
3 | import * as request from 'request' | ||
14 | 4 | ||
15 | function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { | 5 | function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { |
16 | return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { | 6 | return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { |
@@ -27,78 +17,9 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U | |||
27 | }) | 17 | }) |
28 | } | 18 | } |
29 | 19 | ||
30 | type MakeRetryRequestParams = { | ||
31 | url: string, | ||
32 | method: 'GET' | 'POST', | ||
33 | json: Object | ||
34 | } | ||
35 | function makeRetryRequest (params: MakeRetryRequestParams) { | ||
36 | return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { | ||
37 | replay( | ||
38 | request(params, (err, response, body) => err ? rej(err) : res({ response, body })), | ||
39 | { | ||
40 | retries: RETRY_REQUESTS, | ||
41 | factor: 3, | ||
42 | maxTimeout: Infinity, | ||
43 | errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] | ||
44 | } | ||
45 | ) | ||
46 | }) | ||
47 | } | ||
48 | |||
49 | type MakeSecureRequestParams = { | ||
50 | toPod: PodInstance | ||
51 | path: string | ||
52 | data?: Object | ||
53 | } | ||
54 | function makeSecureRequest (params: MakeSecureRequestParams) { | ||
55 | const requestParams: { | ||
56 | method: 'POST', | ||
57 | uri: string, | ||
58 | json: { | ||
59 | signature: PodSignature, | ||
60 | data: any | ||
61 | } | ||
62 | } = { | ||
63 | method: 'POST', | ||
64 | uri: REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path, | ||
65 | json: { | ||
66 | signature: null, | ||
67 | data: null | ||
68 | } | ||
69 | } | ||
70 | |||
71 | const host = CONFIG.WEBSERVER.HOST | ||
72 | |||
73 | let dataToSign | ||
74 | if (params.data) { | ||
75 | dataToSign = params.data | ||
76 | } else { | ||
77 | // We do not have data to sign so we just take our host | ||
78 | // It is not ideal but the connection should be in HTTPS | ||
79 | dataToSign = host | ||
80 | } | ||
81 | |||
82 | sign(dataToSign).then(signature => { | ||
83 | requestParams.json.signature = { | ||
84 | host, // Which host we pretend to be | ||
85 | signature | ||
86 | } | ||
87 | |||
88 | // If there are data information | ||
89 | if (params.data) { | ||
90 | requestParams.json.data = params.data | ||
91 | } | ||
92 | |||
93 | return doRequest(requestParams) | ||
94 | }) | ||
95 | } | ||
96 | |||
97 | // --------------------------------------------------------------------------- | 20 | // --------------------------------------------------------------------------- |
98 | 21 | ||
99 | export { | 22 | export { |
100 | doRequest, | 23 | doRequest, |
101 | doRequestAndSaveToFile, | 24 | doRequestAndSaveToFile |
102 | makeRetryRequest, | ||
103 | makeSecureRequest | ||
104 | } | 25 | } |
diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts index 9586fa562..164ae4951 100644 --- a/server/helpers/webfinger.ts +++ b/server/helpers/webfinger.ts | |||
@@ -35,7 +35,7 @@ export { | |||
35 | 35 | ||
36 | function webfingerLookup (url: string) { | 36 | function webfingerLookup (url: string) { |
37 | return new Promise<WebFingerData>((res, rej) => { | 37 | return new Promise<WebFingerData>((res, rej) => { |
38 | webfinger.lookup('nick@silverbucket.net', (err, p) => { | 38 | webfinger.lookup(url, (err, p) => { |
39 | if (err) return rej(err) | 39 | if (err) return rej(err) |
40 | 40 | ||
41 | return p | 41 | return p |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e6fda88c2..2d61094bd 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -361,12 +361,6 @@ export { | |||
361 | PODS_SCORE, | 361 | PODS_SCORE, |
362 | PREVIEWS_SIZE, | 362 | PREVIEWS_SIZE, |
363 | REMOTE_SCHEME, | 363 | REMOTE_SCHEME, |
364 | REQUEST_ENDPOINT_ACTIONS, | ||
365 | REQUEST_ENDPOINTS, | ||
366 | REQUEST_VIDEO_EVENT_ENDPOINT, | ||
367 | REQUEST_VIDEO_EVENT_TYPES, | ||
368 | REQUEST_VIDEO_QADU_ENDPOINT, | ||
369 | REQUEST_VIDEO_QADU_TYPES, | ||
370 | REQUESTS_IN_PARALLEL, | 364 | REQUESTS_IN_PARALLEL, |
371 | REQUESTS_INTERVAL, | 365 | REQUESTS_INTERVAL, |
372 | REQUESTS_LIMIT_PER_POD, | 366 | REQUESTS_LIMIT_PER_POD, |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index aefb6da3a..1383bb33b 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -2,7 +2,6 @@ import { join } from 'path' | |||
2 | import { flattenDepth } from 'lodash' | 2 | import { flattenDepth } from 'lodash' |
3 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 3 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
4 | import * as Sequelize from 'sequelize' | 4 | import * as Sequelize from 'sequelize' |
5 | import * as Bluebird from 'bluebird' | ||
6 | 5 | ||
7 | import { CONFIG } from './constants' | 6 | import { CONFIG } from './constants' |
8 | // Do not use barrel, we need to load database first | 7 | // Do not use barrel, we need to load database first |
@@ -19,10 +18,6 @@ import { UserModel } from '../models/account/user-interface' | |||
19 | import { AccountVideoRateModel } from '../models/account/account-video-rate-interface' | 18 | import { AccountVideoRateModel } from '../models/account/account-video-rate-interface' |
20 | import { AccountFollowModel } from '../models/account/account-follow-interface' | 19 | import { AccountFollowModel } from '../models/account/account-follow-interface' |
21 | import { TagModel } from './../models/video/tag-interface' | 20 | import { TagModel } from './../models/video/tag-interface' |
22 | import { RequestModel } from './../models/request/request-interface' | ||
23 | import { RequestVideoQaduModel } from './../models/request/request-video-qadu-interface' | ||
24 | import { RequestVideoEventModel } from './../models/request/request-video-event-interface' | ||
25 | import { RequestToPodModel } from './../models/request/request-to-pod-interface' | ||
26 | import { PodModel } from './../models/pod/pod-interface' | 21 | import { PodModel } from './../models/pod/pod-interface' |
27 | import { OAuthTokenModel } from './../models/oauth/oauth-token-interface' | 22 | import { OAuthTokenModel } from './../models/oauth/oauth-token-interface' |
28 | import { OAuthClientModel } from './../models/oauth/oauth-client-interface' | 23 | import { OAuthClientModel } from './../models/oauth/oauth-client-interface' |
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index c8f6b3bc2..c617b16c9 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts | |||
@@ -1,20 +1,21 @@ | |||
1 | import * as passwordGenerator from 'password-generator' | 1 | import * as passwordGenerator from 'password-generator' |
2 | import * as Bluebird from 'bluebird' | 2 | import { UserRole } from '../../shared' |
3 | import { logger, mkdirpPromise, rimrafPromise } from '../helpers' | ||
4 | import { createPrivateAndPublicKeys } from '../helpers/peertube-crypto' | ||
5 | import { createUserAccountAndChannel } from '../lib' | ||
6 | import { clientsExist, usersExist } from './checker' | ||
7 | import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' | ||
3 | 8 | ||
4 | import { database as db } from './database' | 9 | import { database as db } from './database' |
5 | import { CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants' | 10 | import { createLocalAccount } from '../lib/user' |
6 | import { clientsExist, usersExist } from './checker' | ||
7 | import { logger, createCertsIfNotExist, mkdirpPromise, rimrafPromise } from '../helpers' | ||
8 | import { createUserAccountAndChannel } from '../lib' | ||
9 | import { UserRole } from '../../shared' | ||
10 | 11 | ||
11 | async function installApplication () { | 12 | async function installApplication () { |
12 | await db.sequelize.sync() | 13 | await db.sequelize.sync() |
13 | await removeCacheDirectories() | 14 | await removeCacheDirectories() |
14 | await createDirectoriesIfNotExist() | 15 | await createDirectoriesIfNotExist() |
15 | await createCertsIfNotExist() | ||
16 | await createOAuthClientIfNotExist() | 16 | await createOAuthClientIfNotExist() |
17 | await createOAuthAdminIfNotExist() | 17 | await createOAuthAdminIfNotExist() |
18 | await createApplicationIfNotExist() | ||
18 | } | 19 | } |
19 | 20 | ||
20 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
@@ -28,7 +29,7 @@ export { | |||
28 | function removeCacheDirectories () { | 29 | function removeCacheDirectories () { |
29 | const cacheDirectories = CACHE.DIRECTORIES | 30 | const cacheDirectories = CACHE.DIRECTORIES |
30 | 31 | ||
31 | const tasks: Bluebird<any>[] = [] | 32 | const tasks: Promise<any>[] = [] |
32 | 33 | ||
33 | // Cache directories | 34 | // Cache directories |
34 | for (const key of Object.keys(cacheDirectories)) { | 35 | for (const key of Object.keys(cacheDirectories)) { |
@@ -120,7 +121,12 @@ async function createOAuthAdminIfNotExist () { | |||
120 | await createUserAccountAndChannel(user, validatePassword) | 121 | await createUserAccountAndChannel(user, validatePassword) |
121 | logger.info('Username: ' + username) | 122 | logger.info('Username: ' + username) |
122 | logger.info('User password: ' + password) | 123 | logger.info('User password: ' + password) |
124 | } | ||
123 | 125 | ||
126 | async function createApplicationIfNotExist () { | ||
124 | logger.info('Creating Application table.') | 127 | logger.info('Creating Application table.') |
125 | await db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION }) | 128 | const applicationInstance = await db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION }) |
129 | |||
130 | logger.info('Creating application account.') | ||
131 | return createLocalAccount('peertube', null, applicationInstance.id, undefined) | ||
126 | } | 132 | } |
diff --git a/server/initializers/migrations/0075-video-resolutions.ts b/server/initializers/migrations/0075-video-resolutions.ts index e1d9fdacb..e7d8a2876 100644 --- a/server/initializers/migrations/0075-video-resolutions.ts +++ b/server/initializers/migrations/0075-video-resolutions.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import * as Promise from 'bluebird' | ||
3 | import { join } from 'path' | 2 | import { join } from 'path' |
4 | 3 | ||
5 | import { readdirPromise, renamePromise } from '../../helpers/core-utils' | 4 | import { readdirPromise, renamePromise } from '../../helpers/core-utils' |
diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts index 740800606..f8d56528a 100644 --- a/server/lib/activitypub/index.ts +++ b/server/lib/activitypub/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | export * from './process-create' | 1 | export * from './process-create' |
2 | export * from './process-flag' | 2 | export * from './process-flag' |
3 | export * from './process-update' | 3 | export * from './process-update' |
4 | export * from './send-request' | ||
diff --git a/server/lib/activitypub/misc.ts b/server/lib/activitypub/misc.ts index 05e77ebc3..2cf0c4fd1 100644 --- a/server/lib/activitypub/misc.ts +++ b/server/lib/activitypub/misc.ts | |||
@@ -8,7 +8,11 @@ import { VideoChannelInstance } from '../../models/video/video-channel-interface | |||
8 | import { VideoFileAttributes } from '../../models/video/video-file-interface' | 8 | import { VideoFileAttributes } from '../../models/video/video-file-interface' |
9 | import { VideoAttributes, VideoInstance } from '../../models/video/video-interface' | 9 | import { VideoAttributes, VideoInstance } from '../../models/video/video-interface' |
10 | 10 | ||
11 | async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelInstance, videoObject: VideoTorrentObject, t: Sequelize.Transaction) { | 11 | async function videoActivityObjectToDBAttributes ( |
12 | videoChannel: VideoChannelInstance, | ||
13 | videoObject: VideoTorrentObject, | ||
14 | t: Sequelize.Transaction | ||
15 | ) { | ||
12 | const videoFromDatabase = await db.Video.loadByUUIDOrURL(videoObject.uuid, videoObject.id, t) | 16 | const videoFromDatabase = await db.Video.loadByUUIDOrURL(videoObject.uuid, videoObject.id, t) |
13 | if (videoFromDatabase) throw new Error('Video with this UUID/Url already exists.') | 17 | if (videoFromDatabase) throw new Error('Video with this UUID/Url already exists.') |
14 | 18 | ||
diff --git a/server/lib/activitypub/process-flag.ts b/server/lib/activitypub/process-flag.ts index 6fa862ee9..b562dce4d 100644 --- a/server/lib/activitypub/process-flag.ts +++ b/server/lib/activitypub/process-flag.ts | |||
@@ -5,7 +5,7 @@ import { | |||
5 | } from '../../../shared' | 5 | } from '../../../shared' |
6 | 6 | ||
7 | function processFlagActivity (activity: ActivityCreate) { | 7 | function processFlagActivity (activity: ActivityCreate) { |
8 | // empty | 8 | return Promise.resolve(undefined) |
9 | } | 9 | } |
10 | 10 | ||
11 | // --------------------------------------------------------------------------- | 11 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/send-request.ts b/server/lib/activitypub/send-request.ts index 6a31c226d..91101f5ad 100644 --- a/server/lib/activitypub/send-request.ts +++ b/server/lib/activitypub/send-request.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | 2 | ||
3 | import { database as db } from '../../initializers' | ||
3 | import { | 4 | import { |
4 | AccountInstance, | 5 | AccountInstance, |
5 | VideoInstance, | 6 | VideoInstance, |
@@ -13,54 +14,66 @@ function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Sequeliz | |||
13 | const videoChannelObject = videoChannel.toActivityPubObject() | 14 | const videoChannelObject = videoChannel.toActivityPubObject() |
14 | const data = createActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) | 15 | const data = createActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) |
15 | 16 | ||
16 | return broadcastToFollowers(data, t) | 17 | return broadcastToFollowers(data, videoChannel.Account, t) |
17 | } | 18 | } |
18 | 19 | ||
19 | function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { | 20 | function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { |
20 | const videoChannelObject = videoChannel.toActivityPubObject() | 21 | const videoChannelObject = videoChannel.toActivityPubObject() |
21 | const data = updateActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) | 22 | const data = updateActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) |
22 | 23 | ||
23 | return broadcastToFollowers(data, t) | 24 | return broadcastToFollowers(data, videoChannel.Account, t) |
24 | } | 25 | } |
25 | 26 | ||
26 | function sendDeleteVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { | 27 | function sendDeleteVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { |
27 | const videoChannelObject = videoChannel.toActivityPubObject() | 28 | const videoChannelObject = videoChannel.toActivityPubObject() |
28 | const data = deleteActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) | 29 | const data = deleteActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) |
29 | 30 | ||
30 | return broadcastToFollowers(data, t) | 31 | return broadcastToFollowers(data, videoChannel.Account, t) |
31 | } | 32 | } |
32 | 33 | ||
33 | function sendAddVideo (video: VideoInstance, t: Sequelize.Transaction) { | 34 | function sendAddVideo (video: VideoInstance, t: Sequelize.Transaction) { |
34 | const videoObject = video.toActivityPubObject() | 35 | const videoObject = video.toActivityPubObject() |
35 | const data = addActivityData(video.url, video.VideoChannel.Account, video.VideoChannel.url, videoObject) | 36 | const data = addActivityData(video.url, video.VideoChannel.Account, video.VideoChannel.url, videoObject) |
36 | 37 | ||
37 | return broadcastToFollowers(data, t) | 38 | return broadcastToFollowers(data, video.VideoChannel.Account, t) |
38 | } | 39 | } |
39 | 40 | ||
40 | function sendUpdateVideo (video: VideoInstance, t: Sequelize.Transaction) { | 41 | function sendUpdateVideo (video: VideoInstance, t: Sequelize.Transaction) { |
41 | const videoObject = video.toActivityPubObject() | 42 | const videoObject = video.toActivityPubObject() |
42 | const data = updateActivityData(video.url, video.VideoChannel.Account, videoObject) | 43 | const data = updateActivityData(video.url, video.VideoChannel.Account, videoObject) |
43 | 44 | ||
44 | return broadcastToFollowers(data, t) | 45 | return broadcastToFollowers(data, video.VideoChannel.Account, t) |
45 | } | 46 | } |
46 | 47 | ||
47 | function sendDeleteVideo (video: VideoInstance, t: Sequelize.Transaction) { | 48 | function sendDeleteVideo (video: VideoInstance, t: Sequelize.Transaction) { |
48 | const videoObject = video.toActivityPubObject() | 49 | const videoObject = video.toActivityPubObject() |
49 | const data = deleteActivityData(video.url, video.VideoChannel.Account, videoObject) | 50 | const data = deleteActivityData(video.url, video.VideoChannel.Account, videoObject) |
50 | 51 | ||
51 | return broadcastToFollowers(data, t) | 52 | return broadcastToFollowers(data, video.VideoChannel.Account, t) |
52 | } | 53 | } |
53 | 54 | ||
54 | // --------------------------------------------------------------------------- | 55 | // --------------------------------------------------------------------------- |
55 | 56 | ||
56 | export { | 57 | export { |
57 | 58 | sendCreateVideoChannel, | |
59 | sendUpdateVideoChannel, | ||
60 | sendDeleteVideoChannel, | ||
61 | sendAddVideo, | ||
62 | sendUpdateVideo, | ||
63 | sendDeleteVideo | ||
58 | } | 64 | } |
59 | 65 | ||
60 | // --------------------------------------------------------------------------- | 66 | // --------------------------------------------------------------------------- |
61 | 67 | ||
62 | function broadcastToFollowers (data: any, t: Sequelize.Transaction) { | 68 | async function broadcastToFollowers (data: any, fromAccount: AccountInstance, t: Sequelize.Transaction) { |
63 | return httpRequestJobScheduler.createJob(t, 'http-request', 'httpRequestBroadcastHandler', data) | 69 | const result = await db.Account.listFollowerUrlsForApi(fromAccount.name, 0) |
70 | |||
71 | const jobPayload = { | ||
72 | uris: result.data, | ||
73 | body: data | ||
74 | } | ||
75 | |||
76 | return httpRequestJobScheduler.createJob(t, 'httpRequestBroadcastHandler', jobPayload) | ||
64 | } | 77 | } |
65 | 78 | ||
66 | function buildSignedActivity (byAccount: AccountInstance, data: Object) { | 79 | function buildSignedActivity (byAccount: AccountInstance, data: Object) { |
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts index 791ad1cbf..776f647a0 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/cache/videos-preview-cache.ts | |||
@@ -3,9 +3,8 @@ import { join } from 'path' | |||
3 | import { createWriteStream } from 'fs' | 3 | import { createWriteStream } from 'fs' |
4 | 4 | ||
5 | import { database as db, CONFIG, CACHE } from '../../initializers' | 5 | import { database as db, CONFIG, CACHE } from '../../initializers' |
6 | import { logger, unlinkPromise } from '../../helpers' | 6 | import { logger, unlinkPromise, fetchRemoteVideoPreview } from '../../helpers' |
7 | import { VideoInstance } from '../../models' | 7 | import { VideoInstance } from '../../models' |
8 | import { fetchRemotePreview } from '../../lib' | ||
9 | 8 | ||
10 | class VideosPreviewCache { | 9 | class VideosPreviewCache { |
11 | 10 | ||
@@ -54,7 +53,7 @@ class VideosPreviewCache { | |||
54 | } | 53 | } |
55 | 54 | ||
56 | private saveRemotePreviewAndReturnPath (video: VideoInstance) { | 55 | private saveRemotePreviewAndReturnPath (video: VideoInstance) { |
57 | const req = fetchRemotePreview(video) | 56 | const req = fetchRemoteVideoPreview(video) |
58 | 57 | ||
59 | return new Promise<string>((res, rej) => { | 58 | return new Promise<string>((res, rej) => { |
60 | const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName()) | 59 | const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName()) |
diff --git a/server/lib/index.ts b/server/lib/index.ts index bfb415ad2..d22ecb665 100644 --- a/server/lib/index.ts +++ b/server/lib/index.ts | |||
@@ -1,8 +1,6 @@ | |||
1 | export * from './activitypub' | 1 | export * from './activitypub' |
2 | export * from './cache' | 2 | export * from './cache' |
3 | export * from './jobs' | 3 | export * from './jobs' |
4 | export * from './request' | ||
5 | export * from './friends' | ||
6 | export * from './oauth-model' | 4 | export * from './oauth-model' |
7 | export * from './user' | 5 | export * from './user' |
8 | export * from './video-channel' | 6 | export * from './video-channel' |
diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts index 6b6946d02..799b86e1c 100644 --- a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts +++ b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts | |||
@@ -1,19 +1,28 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | |||
3 | import { database as db } from '../../../initializers/database' | ||
4 | import { logger } from '../../../helpers' | 1 | import { logger } from '../../../helpers' |
2 | import { doRequest } from '../../../helpers/requests' | ||
3 | import { HTTPRequestPayload } from './http-request-job-scheduler' | ||
4 | |||
5 | async function process (payload: HTTPRequestPayload, jobId: number) { | ||
6 | logger.info('Processing broadcast in job %d.', jobId) | ||
5 | 7 | ||
6 | async function process (data: { videoUUID: string }, jobId: number) { | 8 | const options = { |
9 | uri: '', | ||
10 | json: payload.body | ||
11 | } | ||
7 | 12 | ||
13 | for (const uri of payload.uris) { | ||
14 | options.uri = uri | ||
15 | await doRequest(options) | ||
16 | } | ||
8 | } | 17 | } |
9 | 18 | ||
10 | function onError (err: Error, jobId: number) { | 19 | function onError (err: Error, jobId: number) { |
11 | logger.error('Error when optimized video file in job %d.', jobId, err) | 20 | logger.error('Error when broadcasting request in job %d.', jobId, err) |
12 | return Promise.resolve() | 21 | return Promise.resolve() |
13 | } | 22 | } |
14 | 23 | ||
15 | async function onSuccess (jobId: number) { | 24 | async function onSuccess (jobId: number) { |
16 | 25 | logger.info('Job %d is a success.', jobId) | |
17 | } | 26 | } |
18 | 27 | ||
19 | // --------------------------------------------------------------------------- | 28 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts index 42cb9139c..ad3349866 100644 --- a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts +++ b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts | |||
@@ -4,7 +4,11 @@ import * as httpRequestBroadcastHandler from './http-request-broadcast-handler' | |||
4 | import * as httpRequestUnicastHandler from './http-request-unicast-handler' | 4 | import * as httpRequestUnicastHandler from './http-request-unicast-handler' |
5 | import { JobCategory } from '../../../../shared' | 5 | import { JobCategory } from '../../../../shared' |
6 | 6 | ||
7 | const jobHandlers: { [ handlerName: string ]: JobHandler<any> } = { | 7 | type HTTPRequestPayload = { |
8 | uris: string[] | ||
9 | body: any | ||
10 | } | ||
11 | const jobHandlers: { [ handlerName: string ]: JobHandler<HTTPRequestPayload, void> } = { | ||
8 | httpRequestBroadcastHandler, | 12 | httpRequestBroadcastHandler, |
9 | httpRequestUnicastHandler | 13 | httpRequestUnicastHandler |
10 | } | 14 | } |
@@ -13,5 +17,6 @@ const jobCategory: JobCategory = 'http-request' | |||
13 | const httpRequestJobScheduler = new JobScheduler(jobCategory, jobHandlers) | 17 | const httpRequestJobScheduler = new JobScheduler(jobCategory, jobHandlers) |
14 | 18 | ||
15 | export { | 19 | export { |
20 | HTTPRequestPayload, | ||
16 | httpRequestJobScheduler | 21 | httpRequestJobScheduler |
17 | } | 22 | } |
diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts index 6b6946d02..13451f042 100644 --- a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts +++ b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts | |||
@@ -1,19 +1,26 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | |||
3 | import { database as db } from '../../../initializers/database' | ||
4 | import { logger } from '../../../helpers' | 1 | import { logger } from '../../../helpers' |
2 | import { doRequest } from '../../../helpers/requests' | ||
3 | import { HTTPRequestPayload } from './http-request-job-scheduler' | ||
4 | |||
5 | async function process (payload: HTTPRequestPayload, jobId: number) { | ||
6 | logger.info('Processing unicast in job %d.', jobId) | ||
5 | 7 | ||
6 | async function process (data: { videoUUID: string }, jobId: number) { | 8 | const uri = payload.uris[0] |
9 | const options = { | ||
10 | uri, | ||
11 | json: payload.body | ||
12 | } | ||
7 | 13 | ||
14 | await doRequest(options) | ||
8 | } | 15 | } |
9 | 16 | ||
10 | function onError (err: Error, jobId: number) { | 17 | function onError (err: Error, jobId: number) { |
11 | logger.error('Error when optimized video file in job %d.', jobId, err) | 18 | logger.error('Error when sending request in job %d.', jobId, err) |
12 | return Promise.resolve() | 19 | return Promise.resolve() |
13 | } | 20 | } |
14 | 21 | ||
15 | async function onSuccess (jobId: number) { | 22 | async function onSuccess (jobId: number) { |
16 | 23 | logger.info('Job %d is a success.', jobId) | |
17 | } | 24 | } |
18 | 25 | ||
19 | // --------------------------------------------------------------------------- | 26 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/jobs/job-scheduler.ts b/server/lib/jobs/job-scheduler.ts index 89a4bca88..f10f745b3 100644 --- a/server/lib/jobs/job-scheduler.ts +++ b/server/lib/jobs/job-scheduler.ts | |||
@@ -1,28 +1,22 @@ | |||
1 | import { AsyncQueue, forever, queue } from 'async' | 1 | import { AsyncQueue, forever, queue } from 'async' |
2 | import * as Sequelize from 'sequelize' | 2 | import * as Sequelize from 'sequelize' |
3 | 3 | import { JobCategory } from '../../../shared' | |
4 | import { | ||
5 | database as db, | ||
6 | JOBS_FETCHING_INTERVAL, | ||
7 | JOBS_FETCH_LIMIT_PER_CYCLE, | ||
8 | JOB_STATES | ||
9 | } from '../../initializers' | ||
10 | import { logger } from '../../helpers' | 4 | import { logger } from '../../helpers' |
5 | import { database as db, JOB_STATES, JOBS_FETCH_LIMIT_PER_CYCLE, JOBS_FETCHING_INTERVAL } from '../../initializers' | ||
11 | import { JobInstance } from '../../models' | 6 | import { JobInstance } from '../../models' |
12 | import { JobCategory } from '../../../shared' | ||
13 | 7 | ||
14 | export interface JobHandler<T> { | 8 | export interface JobHandler<P, T> { |
15 | process (data: object, jobId: number): T | 9 | process (data: object, jobId: number): Promise<T> |
16 | onError (err: Error, jobId: number) | 10 | onError (err: Error, jobId: number) |
17 | onSuccess (jobId: number, jobResult: T) | 11 | onSuccess (jobId: number, jobResult: T, jobScheduler: JobScheduler<P, T>) |
18 | } | 12 | } |
19 | type JobQueueCallback = (err: Error) => void | 13 | type JobQueueCallback = (err: Error) => void |
20 | 14 | ||
21 | class JobScheduler<T> { | 15 | class JobScheduler<P, T> { |
22 | 16 | ||
23 | constructor ( | 17 | constructor ( |
24 | private jobCategory: JobCategory, | 18 | private jobCategory: JobCategory, |
25 | private jobHandlers: { [ id: string ]: JobHandler<T> } | 19 | private jobHandlers: { [ id: string ]: JobHandler<P, T> } |
26 | ) {} | 20 | ) {} |
27 | 21 | ||
28 | async activate () { | 22 | async activate () { |
@@ -66,13 +60,14 @@ class JobScheduler<T> { | |||
66 | ) | 60 | ) |
67 | } | 61 | } |
68 | 62 | ||
69 | createJob (transaction: Sequelize.Transaction, category: JobCategory, handlerName: string, handlerInputData: object) { | 63 | createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: P) { |
70 | const createQuery = { | 64 | const createQuery = { |
71 | state: JOB_STATES.PENDING, | 65 | state: JOB_STATES.PENDING, |
72 | category, | 66 | category: this.jobCategory, |
73 | handlerName, | 67 | handlerName, |
74 | handlerInputData | 68 | handlerInputData |
75 | } | 69 | } |
70 | |||
76 | const options = { transaction } | 71 | const options = { transaction } |
77 | 72 | ||
78 | return db.Job.create(createQuery, options) | 73 | return db.Job.create(createQuery, options) |
@@ -95,7 +90,7 @@ class JobScheduler<T> { | |||
95 | await job.save() | 90 | await job.save() |
96 | 91 | ||
97 | try { | 92 | try { |
98 | const result = await jobHandler.process(job.handlerInputData, job.id) | 93 | const result: T = await jobHandler.process(job.handlerInputData, job.id) |
99 | await this.onJobSuccess(jobHandler, job, result) | 94 | await this.onJobSuccess(jobHandler, job, result) |
100 | } catch (err) { | 95 | } catch (err) { |
101 | logger.error('Error in job handler %s.', job.handlerName, err) | 96 | logger.error('Error in job handler %s.', job.handlerName, err) |
@@ -111,7 +106,7 @@ class JobScheduler<T> { | |||
111 | callback(null) | 106 | callback(null) |
112 | } | 107 | } |
113 | 108 | ||
114 | private async onJobError (jobHandler: JobHandler<any>, job: JobInstance, err: Error) { | 109 | private async onJobError (jobHandler: JobHandler<P, T>, job: JobInstance, err: Error) { |
115 | job.state = JOB_STATES.ERROR | 110 | job.state = JOB_STATES.ERROR |
116 | 111 | ||
117 | try { | 112 | try { |
@@ -122,12 +117,12 @@ class JobScheduler<T> { | |||
122 | } | 117 | } |
123 | } | 118 | } |
124 | 119 | ||
125 | private async onJobSuccess (jobHandler: JobHandler<any>, job: JobInstance, jobResult: any) { | 120 | private async onJobSuccess (jobHandler: JobHandler<P, T>, job: JobInstance, jobResult: T) { |
126 | job.state = JOB_STATES.SUCCESS | 121 | job.state = JOB_STATES.SUCCESS |
127 | 122 | ||
128 | try { | 123 | try { |
129 | await job.save() | 124 | await job.save() |
130 | jobHandler.onSuccess(job.id, jobResult) | 125 | jobHandler.onSuccess(job.id, jobResult, this) |
131 | } catch (err) { | 126 | } catch (err) { |
132 | this.cannotSaveJobError(err) | 127 | this.cannotSaveJobError(err) |
133 | } | 128 | } |
diff --git a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts index d7c614fb8..c5efe8eeb 100644 --- a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts | |||
@@ -1,10 +1,14 @@ | |||
1 | import { JobScheduler, JobHandler } from '../job-scheduler' | 1 | import { JobCategory } from '../../../../shared' |
2 | 2 | import { JobHandler, JobScheduler } from '../job-scheduler' | |
3 | import * as videoFileOptimizer from './video-file-optimizer-handler' | 3 | import * as videoFileOptimizer from './video-file-optimizer-handler' |
4 | import * as videoFileTranscoder from './video-file-transcoder-handler' | 4 | import * as videoFileTranscoder from './video-file-transcoder-handler' |
5 | import { JobCategory } from '../../../../shared' | 5 | import { VideoInstance } from '../../../models/video/video-interface' |
6 | 6 | ||
7 | const jobHandlers: { [ handlerName: string ]: JobHandler<any> } = { | 7 | type TranscodingJobPayload = { |
8 | videoUUID: string | ||
9 | resolution?: number | ||
10 | } | ||
11 | const jobHandlers: { [ handlerName: string ]: JobHandler<TranscodingJobPayload, VideoInstance> } = { | ||
8 | videoFileOptimizer, | 12 | videoFileOptimizer, |
9 | videoFileTranscoder | 13 | videoFileTranscoder |
10 | } | 14 | } |
@@ -13,5 +17,6 @@ const jobCategory: JobCategory = 'transcoding' | |||
13 | const transcodingJobScheduler = new JobScheduler(jobCategory, jobHandlers) | 17 | const transcodingJobScheduler = new JobScheduler(jobCategory, jobHandlers) |
14 | 18 | ||
15 | export { | 19 | export { |
20 | TranscodingJobPayload, | ||
16 | transcodingJobScheduler | 21 | transcodingJobScheduler |
17 | } | 22 | } |
diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts index f019c28bc..47603a66c 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts | |||
@@ -1,12 +1,13 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { computeResolutionsToTranscode, logger } from '../../../helpers' | ||
2 | 3 | ||
3 | import { database as db } from '../../../initializers/database' | 4 | import { database as db } from '../../../initializers/database' |
4 | import { logger, computeResolutionsToTranscode } from '../../../helpers' | ||
5 | import { VideoInstance } from '../../../models' | 5 | import { VideoInstance } from '../../../models' |
6 | import { addVideoToFriends } from '../../friends' | 6 | import { sendAddVideo } from '../../activitypub/send-request' |
7 | import { JobScheduler } from '../job-scheduler' | 7 | import { JobScheduler } from '../job-scheduler' |
8 | import { TranscodingJobPayload } from './transcoding-job-scheduler' | ||
8 | 9 | ||
9 | async function process (data: { videoUUID: string }, jobId: number) { | 10 | async function process (data: TranscodingJobPayload, jobId: number) { |
10 | const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) | 11 | const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) |
11 | // No video, maybe deleted? | 12 | // No video, maybe deleted? |
12 | if (!video) { | 13 | if (!video) { |
@@ -24,7 +25,7 @@ function onError (err: Error, jobId: number) { | |||
24 | return Promise.resolve() | 25 | return Promise.resolve() |
25 | } | 26 | } |
26 | 27 | ||
27 | async function onSuccess (jobId: number, video: VideoInstance) { | 28 | async function onSuccess (jobId: number, video: VideoInstance, jobScheduler: JobScheduler<TranscodingJobPayload, VideoInstance>) { |
28 | if (video === undefined) return undefined | 29 | if (video === undefined) return undefined |
29 | 30 | ||
30 | logger.info('Job %d is a success.', jobId) | 31 | logger.info('Job %d is a success.', jobId) |
@@ -34,10 +35,8 @@ async function onSuccess (jobId: number, video: VideoInstance) { | |||
34 | // Video does not exist anymore | 35 | // Video does not exist anymore |
35 | if (!videoDatabase) return undefined | 36 | if (!videoDatabase) return undefined |
36 | 37 | ||
37 | const remoteVideo = await videoDatabase.toAddRemoteJSON() | 38 | // Now we'll add the video's meta data to our followers |
38 | 39 | await sendAddVideo(video, undefined) | |
39 | // Now we'll add the video's meta data to our friends | ||
40 | await addVideoToFriends(remoteVideo, null) | ||
41 | 40 | ||
42 | const originalFileHeight = await videoDatabase.getOriginalFileHeight() | 41 | const originalFileHeight = await videoDatabase.getOriginalFileHeight() |
43 | // Create transcoding jobs if there are enabled resolutions | 42 | // Create transcoding jobs if there are enabled resolutions |
@@ -59,7 +58,7 @@ async function onSuccess (jobId: number, video: VideoInstance) { | |||
59 | resolution | 58 | resolution |
60 | } | 59 | } |
61 | 60 | ||
62 | const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput) | 61 | const p = jobScheduler.createJob(t, 'videoFileTranscoder', dataInput) |
63 | tasks.push(p) | 62 | tasks.push(p) |
64 | } | 63 | } |
65 | 64 | ||
diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts index 397b95795..77e5d9f7f 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import { database as db } from '../../../initializers/database' | 1 | import { VideoResolution } from '../../../../shared' |
2 | import { updateVideoToFriends } from '../../friends' | ||
3 | import { logger } from '../../../helpers' | 2 | import { logger } from '../../../helpers' |
3 | import { database as db } from '../../../initializers/database' | ||
4 | import { VideoInstance } from '../../../models' | 4 | import { VideoInstance } from '../../../models' |
5 | import { VideoResolution } from '../../../../shared' | 5 | import { sendUpdateVideo } from '../../activitypub/send-request' |
6 | 6 | ||
7 | async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { | 7 | async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { |
8 | const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) | 8 | const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) |
@@ -32,10 +32,7 @@ async function onSuccess (jobId: number, video: VideoInstance) { | |||
32 | // Video does not exist anymore | 32 | // Video does not exist anymore |
33 | if (!videoDatabase) return undefined | 33 | if (!videoDatabase) return undefined |
34 | 34 | ||
35 | const remoteVideo = videoDatabase.toUpdateRemoteJSON() | 35 | await sendUpdateVideo(video, undefined) |
36 | |||
37 | // Now we'll add the video's meta data to our friends | ||
38 | await updateVideoToFriends(remoteVideo, null) | ||
39 | 36 | ||
40 | return undefined | 37 | return undefined |
41 | } | 38 | } |
diff --git a/server/lib/request/abstract-request-scheduler.ts b/server/lib/request/abstract-request-scheduler.ts deleted file mode 100644 index f838c47f2..000000000 --- a/server/lib/request/abstract-request-scheduler.ts +++ /dev/null | |||
@@ -1,168 +0,0 @@ | |||
1 | import { isEmpty } from 'lodash' | ||
2 | import * as Bluebird from 'bluebird' | ||
3 | |||
4 | import { database as db } from '../../initializers/database' | ||
5 | import { logger, makeSecureRequest } from '../../helpers' | ||
6 | import { AbstractRequestClass, AbstractRequestToPodClass, PodInstance } from '../../models' | ||
7 | import { | ||
8 | API_VERSION, | ||
9 | REQUESTS_IN_PARALLEL, | ||
10 | REQUESTS_INTERVAL | ||
11 | } from '../../initializers' | ||
12 | |||
13 | interface RequestsObjects<U> { | ||
14 | [ id: string ]: { | ||
15 | toPod: PodInstance | ||
16 | endpoint: string | ||
17 | ids: number[] // ids | ||
18 | datas: U[] | ||
19 | } | ||
20 | } | ||
21 | |||
22 | abstract class AbstractRequestScheduler <T> { | ||
23 | requestInterval: number | ||
24 | limitPods: number | ||
25 | limitPerPod: number | ||
26 | |||
27 | protected lastRequestTimestamp: number | ||
28 | protected timer: NodeJS.Timer | ||
29 | protected description: string | ||
30 | |||
31 | constructor () { | ||
32 | this.lastRequestTimestamp = 0 | ||
33 | this.timer = null | ||
34 | this.requestInterval = REQUESTS_INTERVAL | ||
35 | } | ||
36 | |||
37 | abstract getRequestModel (): AbstractRequestClass<T> | ||
38 | abstract getRequestToPodModel (): AbstractRequestToPodClass | ||
39 | abstract buildRequestsObjects (requestsGrouped: T): RequestsObjects<any> | ||
40 | |||
41 | activate () { | ||
42 | logger.info('Requests scheduler activated.') | ||
43 | this.lastRequestTimestamp = Date.now() | ||
44 | |||
45 | this.timer = setInterval(() => { | ||
46 | this.lastRequestTimestamp = Date.now() | ||
47 | this.makeRequests() | ||
48 | }, this.requestInterval) | ||
49 | } | ||
50 | |||
51 | deactivate () { | ||
52 | logger.info('Requests scheduler deactivated.') | ||
53 | clearInterval(this.timer) | ||
54 | this.timer = null | ||
55 | } | ||
56 | |||
57 | forceSend () { | ||
58 | logger.info('Force requests scheduler sending.') | ||
59 | this.makeRequests() | ||
60 | } | ||
61 | |||
62 | remainingMilliSeconds () { | ||
63 | if (this.timer === null) return -1 | ||
64 | |||
65 | return REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) | ||
66 | } | ||
67 | |||
68 | remainingRequestsCount () { | ||
69 | return this.getRequestModel().countTotalRequests() | ||
70 | } | ||
71 | |||
72 | flush () { | ||
73 | return this.getRequestModel().removeAll() | ||
74 | } | ||
75 | |||
76 | // --------------------------------------------------------------------------- | ||
77 | |||
78 | // Make a requests to friends of a certain type | ||
79 | protected async makeRequest (toPod: PodInstance, requestEndpoint: string, requestsToMake: any) { | ||
80 | const params = { | ||
81 | toPod: toPod, | ||
82 | method: 'POST' as 'POST', | ||
83 | path: '/api/' + API_VERSION + '/remote/' + requestEndpoint, | ||
84 | data: requestsToMake // Requests we need to make | ||
85 | } | ||
86 | |||
87 | // Make multiple retry requests to all of pods | ||
88 | // The function fire some useful callbacks | ||
89 | try { | ||
90 | const { response } = await makeSecureRequest(params) | ||
91 | |||
92 | // 400 because if the other pod is not up to date, it may not understand our request | ||
93 | if ([ 200, 201, 204, 400 ].indexOf(response.statusCode) === -1) { | ||
94 | throw new Error('Status code not 20x or 400 : ' + response.statusCode) | ||
95 | } | ||
96 | } catch (err) { | ||
97 | logger.error('Error sending secure request to %s pod.', toPod.host, err) | ||
98 | |||
99 | throw err | ||
100 | } | ||
101 | } | ||
102 | |||
103 | // Make all the requests of the scheduler | ||
104 | protected async makeRequests () { | ||
105 | let requestsGrouped: T | ||
106 | |||
107 | try { | ||
108 | requestsGrouped = await this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod) | ||
109 | } catch (err) { | ||
110 | logger.error('Cannot get the list of "%s".', this.description, { error: err.stack }) | ||
111 | throw err | ||
112 | } | ||
113 | |||
114 | // We want to group requests by destinations pod and endpoint | ||
115 | const requestsToMake = this.buildRequestsObjects(requestsGrouped) | ||
116 | |||
117 | // If there are no requests, abort | ||
118 | if (isEmpty(requestsToMake) === true) { | ||
119 | logger.info('No "%s" to make.', this.description) | ||
120 | return { goodPods: [], badPods: [] } | ||
121 | } | ||
122 | |||
123 | logger.info('Making "%s" to friends.', this.description) | ||
124 | |||
125 | const goodPods: number[] = [] | ||
126 | const badPods: number[] = [] | ||
127 | |||
128 | await Bluebird.map(Object.keys(requestsToMake), async hashKey => { | ||
129 | const requestToMake = requestsToMake[hashKey] | ||
130 | const toPod: PodInstance = requestToMake.toPod | ||
131 | |||
132 | try { | ||
133 | await this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas) | ||
134 | logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) | ||
135 | goodPods.push(requestToMake.toPod.id) | ||
136 | |||
137 | this.afterRequestHook() | ||
138 | |||
139 | // Remove the pod id of these request ids | ||
140 | await this.getRequestToPodModel() | ||
141 | .removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id) | ||
142 | } catch (err) { | ||
143 | badPods.push(requestToMake.toPod.id) | ||
144 | logger.info('Cannot make request to %s.', toPod.host, err) | ||
145 | } | ||
146 | }, { concurrency: REQUESTS_IN_PARALLEL }) | ||
147 | |||
148 | this.afterRequestsHook() | ||
149 | |||
150 | // All the requests were made, we update the pods score | ||
151 | db.Pod.updatePodsScore(goodPods, badPods) | ||
152 | } | ||
153 | |||
154 | protected afterRequestHook () { | ||
155 | // Nothing to do, let children re-implement it | ||
156 | } | ||
157 | |||
158 | protected afterRequestsHook () { | ||
159 | // Nothing to do, let children re-implement it | ||
160 | } | ||
161 | } | ||
162 | |||
163 | // --------------------------------------------------------------------------- | ||
164 | |||
165 | export { | ||
166 | AbstractRequestScheduler, | ||
167 | RequestsObjects | ||
168 | } | ||
diff --git a/server/lib/request/index.ts b/server/lib/request/index.ts deleted file mode 100644 index 47d60e5b4..000000000 --- a/server/lib/request/index.ts +++ /dev/null | |||
@@ -1,4 +0,0 @@ | |||
1 | export * from './abstract-request-scheduler' | ||
2 | export * from './request-scheduler' | ||
3 | export * from './request-video-event-scheduler' | ||
4 | export * from './request-video-qadu-scheduler' | ||
diff --git a/server/lib/request/request-scheduler.ts b/server/lib/request/request-scheduler.ts deleted file mode 100644 index c3f7f6429..000000000 --- a/server/lib/request/request-scheduler.ts +++ /dev/null | |||
@@ -1,96 +0,0 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | import { database as db } from '../../initializers/database' | ||
4 | import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler' | ||
5 | import { logger } from '../../helpers' | ||
6 | import { REQUESTS_LIMIT_PODS, REQUESTS_LIMIT_PER_POD } from '../../initializers' | ||
7 | import { RequestsGrouped } from '../../models' | ||
8 | import { RequestEndpoint, RemoteVideoRequest } from '../../../shared' | ||
9 | |||
10 | export type RequestSchedulerOptions = { | ||
11 | type: string | ||
12 | endpoint: RequestEndpoint | ||
13 | data: Object | ||
14 | toIds: number[] | ||
15 | transaction: Sequelize.Transaction | ||
16 | } | ||
17 | |||
18 | class RequestScheduler extends AbstractRequestScheduler<RequestsGrouped> { | ||
19 | constructor () { | ||
20 | super() | ||
21 | |||
22 | // We limit the size of the requests | ||
23 | this.limitPods = REQUESTS_LIMIT_PODS | ||
24 | this.limitPerPod = REQUESTS_LIMIT_PER_POD | ||
25 | |||
26 | this.description = 'requests' | ||
27 | } | ||
28 | |||
29 | getRequestModel () { | ||
30 | return db.Request | ||
31 | } | ||
32 | |||
33 | getRequestToPodModel () { | ||
34 | return db.RequestToPod | ||
35 | } | ||
36 | |||
37 | buildRequestsObjects (requestsGrouped: RequestsGrouped) { | ||
38 | const requestsToMakeGrouped: RequestsObjects<RemoteVideoRequest> = {} | ||
39 | |||
40 | for (const toPodId of Object.keys(requestsGrouped)) { | ||
41 | for (const data of requestsGrouped[toPodId]) { | ||
42 | const request = data.request | ||
43 | const pod = data.pod | ||
44 | const hashKey = toPodId + request.endpoint | ||
45 | |||
46 | if (!requestsToMakeGrouped[hashKey]) { | ||
47 | requestsToMakeGrouped[hashKey] = { | ||
48 | toPod: pod, | ||
49 | endpoint: request.endpoint, | ||
50 | ids: [], // request ids, to delete them from the DB in the future | ||
51 | datas: [] // requests data, | ||
52 | } | ||
53 | } | ||
54 | |||
55 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
56 | requestsToMakeGrouped[hashKey].datas.push(request.request) | ||
57 | } | ||
58 | } | ||
59 | |||
60 | return requestsToMakeGrouped | ||
61 | } | ||
62 | |||
63 | async createRequest ({ type, endpoint, data, toIds, transaction }: RequestSchedulerOptions) { | ||
64 | // If there are no destination pods abort | ||
65 | if (toIds.length === 0) return undefined | ||
66 | |||
67 | const createQuery = { | ||
68 | endpoint, | ||
69 | request: { | ||
70 | type: type, | ||
71 | data: data | ||
72 | } | ||
73 | } | ||
74 | |||
75 | const dbRequestOptions: Sequelize.CreateOptions = { | ||
76 | transaction | ||
77 | } | ||
78 | |||
79 | const request = await db.Request.create(createQuery, dbRequestOptions) | ||
80 | await request.setPods(toIds, dbRequestOptions) | ||
81 | } | ||
82 | |||
83 | // --------------------------------------------------------------------------- | ||
84 | |||
85 | afterRequestsHook () { | ||
86 | // Flush requests with no pod | ||
87 | this.getRequestModel().removeWithEmptyTo() | ||
88 | .catch(err => logger.error('Error when removing requests with no pods.', err)) | ||
89 | } | ||
90 | } | ||
91 | |||
92 | // --------------------------------------------------------------------------- | ||
93 | |||
94 | export { | ||
95 | RequestScheduler | ||
96 | } | ||
diff --git a/server/lib/request/request-video-event-scheduler.ts b/server/lib/request/request-video-event-scheduler.ts deleted file mode 100644 index 5f21287f0..000000000 --- a/server/lib/request/request-video-event-scheduler.ts +++ /dev/null | |||
@@ -1,129 +0,0 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | import { database as db } from '../../initializers/database' | ||
4 | import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler' | ||
5 | import { | ||
6 | REQUESTS_VIDEO_EVENT_LIMIT_PODS, | ||
7 | REQUESTS_VIDEO_EVENT_LIMIT_PER_POD, | ||
8 | REQUEST_VIDEO_EVENT_ENDPOINT | ||
9 | } from '../../initializers' | ||
10 | import { RequestsVideoEventGrouped } from '../../models' | ||
11 | import { RequestVideoEventType, RemoteVideoEventRequest, RemoteVideoEventType } from '../../../shared' | ||
12 | |||
13 | export type RequestVideoEventSchedulerOptions = { | ||
14 | type: RequestVideoEventType | ||
15 | videoId: number | ||
16 | count?: number | ||
17 | transaction?: Sequelize.Transaction | ||
18 | } | ||
19 | |||
20 | class RequestVideoEventScheduler extends AbstractRequestScheduler<RequestsVideoEventGrouped> { | ||
21 | constructor () { | ||
22 | super() | ||
23 | |||
24 | // We limit the size of the requests | ||
25 | this.limitPods = REQUESTS_VIDEO_EVENT_LIMIT_PODS | ||
26 | this.limitPerPod = REQUESTS_VIDEO_EVENT_LIMIT_PER_POD | ||
27 | |||
28 | this.description = 'video event requests' | ||
29 | } | ||
30 | |||
31 | getRequestModel () { | ||
32 | return db.RequestVideoEvent | ||
33 | } | ||
34 | |||
35 | getRequestToPodModel () { | ||
36 | return db.RequestVideoEvent | ||
37 | } | ||
38 | |||
39 | buildRequestsObjects (eventRequests: RequestsVideoEventGrouped) { | ||
40 | const requestsToMakeGrouped: RequestsObjects<RemoteVideoEventRequest> = {} | ||
41 | |||
42 | /* Example: | ||
43 | { | ||
44 | pod1: { | ||
45 | video1: { views: 4, likes: 5 }, | ||
46 | video2: { likes: 5 } | ||
47 | } | ||
48 | } | ||
49 | */ | ||
50 | const eventsPerVideoPerPod: { | ||
51 | [ podId: string ]: { | ||
52 | [ videoUUID: string ]: { | ||
53 | views?: number | ||
54 | likes?: number | ||
55 | dislikes?: number | ||
56 | } | ||
57 | } | ||
58 | } = {} | ||
59 | |||
60 | // We group video events per video and per pod | ||
61 | // We add the counts of the same event types | ||
62 | for (const toPodId of Object.keys(eventRequests)) { | ||
63 | for (const eventToProcess of eventRequests[toPodId]) { | ||
64 | if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {} | ||
65 | |||
66 | if (!requestsToMakeGrouped[toPodId]) { | ||
67 | requestsToMakeGrouped[toPodId] = { | ||
68 | toPod: eventToProcess.pod, | ||
69 | endpoint: REQUEST_VIDEO_EVENT_ENDPOINT, | ||
70 | ids: [], // request ids, to delete them from the DB in the future | ||
71 | datas: [] // requests data | ||
72 | } | ||
73 | } | ||
74 | requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id) | ||
75 | |||
76 | const eventsPerVideo = eventsPerVideoPerPod[toPodId] | ||
77 | const uuid = eventToProcess.video.uuid | ||
78 | if (!eventsPerVideo[uuid]) eventsPerVideo[uuid] = {} | ||
79 | |||
80 | const events = eventsPerVideo[uuid] | ||
81 | if (!events[eventToProcess.type]) events[eventToProcess.type] = 0 | ||
82 | |||
83 | events[eventToProcess.type] += eventToProcess.count | ||
84 | } | ||
85 | } | ||
86 | |||
87 | // Now we build our requests array per pod | ||
88 | for (const toPodId of Object.keys(eventsPerVideoPerPod)) { | ||
89 | const eventsForPod = eventsPerVideoPerPod[toPodId] | ||
90 | |||
91 | for (const uuid of Object.keys(eventsForPod)) { | ||
92 | const eventsForVideo = eventsForPod[uuid] | ||
93 | |||
94 | for (const eventType of Object.keys(eventsForVideo)) { | ||
95 | requestsToMakeGrouped[toPodId].datas.push({ | ||
96 | data: { | ||
97 | uuid, | ||
98 | eventType: eventType as RemoteVideoEventType, | ||
99 | count: +eventsForVideo[eventType] | ||
100 | } | ||
101 | }) | ||
102 | } | ||
103 | } | ||
104 | } | ||
105 | |||
106 | return requestsToMakeGrouped | ||
107 | } | ||
108 | |||
109 | createRequest ({ type, videoId, count, transaction }: RequestVideoEventSchedulerOptions) { | ||
110 | if (count === undefined) count = 1 | ||
111 | |||
112 | const dbRequestOptions: Sequelize.CreateOptions = {} | ||
113 | if (transaction) dbRequestOptions.transaction = transaction | ||
114 | |||
115 | const createQuery = { | ||
116 | type, | ||
117 | count, | ||
118 | videoId | ||
119 | } | ||
120 | |||
121 | return db.RequestVideoEvent.create(createQuery, dbRequestOptions) | ||
122 | } | ||
123 | } | ||
124 | |||
125 | // --------------------------------------------------------------------------- | ||
126 | |||
127 | export { | ||
128 | RequestVideoEventScheduler | ||
129 | } | ||
diff --git a/server/lib/request/request-video-qadu-scheduler.ts b/server/lib/request/request-video-qadu-scheduler.ts deleted file mode 100644 index 24ee59d29..000000000 --- a/server/lib/request/request-video-qadu-scheduler.ts +++ /dev/null | |||
@@ -1,148 +0,0 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | import { database as db } from '../../initializers/database' | ||
4 | import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler' | ||
5 | import { logger } from '../../helpers' | ||
6 | import { | ||
7 | REQUESTS_VIDEO_QADU_LIMIT_PODS, | ||
8 | REQUESTS_VIDEO_QADU_LIMIT_PER_POD, | ||
9 | REQUEST_VIDEO_QADU_ENDPOINT, | ||
10 | REQUEST_VIDEO_QADU_TYPES | ||
11 | } from '../../initializers' | ||
12 | import { RequestsVideoQaduGrouped, PodInstance } from '../../models' | ||
13 | import { RemoteQaduVideoRequest, RequestVideoQaduType } from '../../../shared' | ||
14 | |||
15 | // We create a custom interface because we need "videos" attribute for our computations | ||
16 | interface RequestsObjectsCustom<U> extends RequestsObjects<U> { | ||
17 | [ id: string ]: { | ||
18 | toPod: PodInstance | ||
19 | endpoint: string | ||
20 | ids: number[] // ids | ||
21 | datas: U[] | ||
22 | |||
23 | videos: { | ||
24 | [ uuid: string ]: { | ||
25 | uuid: string | ||
26 | likes?: number | ||
27 | dislikes?: number | ||
28 | views?: number | ||
29 | } | ||
30 | } | ||
31 | } | ||
32 | } | ||
33 | |||
34 | export type RequestVideoQaduSchedulerOptions = { | ||
35 | type: RequestVideoQaduType | ||
36 | videoId: number | ||
37 | transaction?: Sequelize.Transaction | ||
38 | } | ||
39 | |||
40 | class RequestVideoQaduScheduler extends AbstractRequestScheduler<RequestsVideoQaduGrouped> { | ||
41 | constructor () { | ||
42 | super() | ||
43 | |||
44 | // We limit the size of the requests | ||
45 | this.limitPods = REQUESTS_VIDEO_QADU_LIMIT_PODS | ||
46 | this.limitPerPod = REQUESTS_VIDEO_QADU_LIMIT_PER_POD | ||
47 | |||
48 | this.description = 'video QADU requests' | ||
49 | } | ||
50 | |||
51 | getRequestModel () { | ||
52 | return db.RequestVideoQadu | ||
53 | } | ||
54 | |||
55 | getRequestToPodModel () { | ||
56 | return db.RequestVideoQadu | ||
57 | } | ||
58 | |||
59 | buildRequestsObjects (requests: RequestsVideoQaduGrouped) { | ||
60 | const requestsToMakeGrouped: RequestsObjectsCustom<RemoteQaduVideoRequest> = {} | ||
61 | |||
62 | for (const toPodId of Object.keys(requests)) { | ||
63 | for (const data of requests[toPodId]) { | ||
64 | const request = data.request | ||
65 | const video = data.video | ||
66 | const pod = data.pod | ||
67 | const hashKey = toPodId | ||
68 | |||
69 | if (!requestsToMakeGrouped[hashKey]) { | ||
70 | requestsToMakeGrouped[hashKey] = { | ||
71 | toPod: pod, | ||
72 | endpoint: REQUEST_VIDEO_QADU_ENDPOINT, | ||
73 | ids: [], // request ids, to delete them from the DB in the future | ||
74 | datas: [], // requests data | ||
75 | videos: {} | ||
76 | } | ||
77 | } | ||
78 | |||
79 | // Maybe another attribute was filled for this video | ||
80 | let videoData = requestsToMakeGrouped[hashKey].videos[video.id] | ||
81 | if (!videoData) videoData = { uuid: null } | ||
82 | |||
83 | switch (request.type) { | ||
84 | case REQUEST_VIDEO_QADU_TYPES.LIKES: | ||
85 | videoData.likes = video.likes | ||
86 | break | ||
87 | |||
88 | case REQUEST_VIDEO_QADU_TYPES.DISLIKES: | ||
89 | videoData.dislikes = video.dislikes | ||
90 | break | ||
91 | |||
92 | case REQUEST_VIDEO_QADU_TYPES.VIEWS: | ||
93 | videoData.views = video.views | ||
94 | break | ||
95 | |||
96 | default: | ||
97 | logger.error('Unknown request video QADU type %s.', request.type) | ||
98 | return undefined | ||
99 | } | ||
100 | |||
101 | // Do not forget the uuid so the remote pod can identify the video | ||
102 | videoData.uuid = video.uuid | ||
103 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
104 | |||
105 | // Maybe there are multiple quick and dirty update for the same video | ||
106 | // We use this hash map to dedupe them | ||
107 | requestsToMakeGrouped[hashKey].videos[video.id] = videoData | ||
108 | } | ||
109 | } | ||
110 | |||
111 | // Now we deduped similar quick and dirty updates, we can build our requests data | ||
112 | for (const hashKey of Object.keys(requestsToMakeGrouped)) { | ||
113 | for (const videoUUID of Object.keys(requestsToMakeGrouped[hashKey].videos)) { | ||
114 | const videoData = requestsToMakeGrouped[hashKey].videos[videoUUID] | ||
115 | |||
116 | requestsToMakeGrouped[hashKey].datas.push({ | ||
117 | data: videoData | ||
118 | }) | ||
119 | } | ||
120 | |||
121 | // We don't need it anymore, it was just to build our data array | ||
122 | delete requestsToMakeGrouped[hashKey].videos | ||
123 | } | ||
124 | |||
125 | return requestsToMakeGrouped | ||
126 | } | ||
127 | |||
128 | async createRequest ({ type, videoId, transaction }: RequestVideoQaduSchedulerOptions) { | ||
129 | const dbRequestOptions: Sequelize.BulkCreateOptions = {} | ||
130 | if (transaction) dbRequestOptions.transaction = transaction | ||
131 | |||
132 | // Send the update to all our friends | ||
133 | const podIds = await db.Pod.listAllIds(transaction) | ||
134 | const queries = [] | ||
135 | for (const podId of podIds) { | ||
136 | queries.push({ type, videoId, podId }) | ||
137 | } | ||
138 | |||
139 | await db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions) | ||
140 | return undefined | ||
141 | } | ||
142 | } | ||
143 | |||
144 | // --------------------------------------------------------------------------- | ||
145 | |||
146 | export { | ||
147 | RequestVideoQaduScheduler | ||
148 | } | ||
diff --git a/server/lib/user.ts b/server/lib/user.ts index 57c653e55..1094c2401 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts | |||
@@ -1,6 +1,9 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { getActivityPubUrl } from '../helpers/activitypub' | ||
3 | import { createPrivateAndPublicKeys } from '../helpers/peertube-crypto' | ||
1 | import { database as db } from '../initializers' | 4 | import { database as db } from '../initializers' |
5 | import { CONFIG } from '../initializers/constants' | ||
2 | import { UserInstance } from '../models' | 6 | import { UserInstance } from '../models' |
3 | import { addVideoAccountToFriends } from './friends' | ||
4 | import { createVideoChannel } from './video-channel' | 7 | import { createVideoChannel } from './video-channel' |
5 | 8 | ||
6 | async function createUserAccountAndChannel (user: UserInstance, validateUser = true) { | 9 | async function createUserAccountAndChannel (user: UserInstance, validateUser = true) { |
@@ -11,32 +14,46 @@ async function createUserAccountAndChannel (user: UserInstance, validateUser = t | |||
11 | } | 14 | } |
12 | 15 | ||
13 | const userCreated = await user.save(userOptions) | 16 | const userCreated = await user.save(userOptions) |
14 | const accountInstance = db.Account.build({ | 17 | const accountCreated = await createLocalAccount(user.username, user.id, null, t) |
15 | name: userCreated.username, | ||
16 | podId: null, // It is our pod | ||
17 | userId: userCreated.id | ||
18 | }) | ||
19 | |||
20 | const accountCreated = await accountInstance.save({ transaction: t }) | ||
21 | |||
22 | const remoteVideoAccount = accountCreated.toAddRemoteJSON() | ||
23 | |||
24 | // Now we'll add the video channel's meta data to our friends | ||
25 | const account = await addVideoAccountToFriends(remoteVideoAccount, t) | ||
26 | 18 | ||
27 | const videoChannelInfo = { | 19 | const videoChannelInfo = { |
28 | name: `Default ${userCreated.username} channel` | 20 | name: `Default ${userCreated.username} channel` |
29 | } | 21 | } |
30 | const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) | 22 | const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) |
31 | 23 | ||
32 | return { account, videoChannel } | 24 | return { account: accountCreated, videoChannel } |
33 | }) | 25 | }) |
34 | 26 | ||
35 | return res | 27 | return res |
36 | } | 28 | } |
37 | 29 | ||
30 | async function createLocalAccount (name: string, userId: number, applicationId: number, t: Sequelize.Transaction) { | ||
31 | const { publicKey, privateKey } = await createPrivateAndPublicKeys() | ||
32 | const url = getActivityPubUrl('account', name) | ||
33 | |||
34 | const accountInstance = db.Account.build({ | ||
35 | name, | ||
36 | url, | ||
37 | publicKey, | ||
38 | privateKey, | ||
39 | followersCount: 0, | ||
40 | followingCount: 0, | ||
41 | inboxUrl: url + '/inbox', | ||
42 | outboxUrl: url + '/outbox', | ||
43 | sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox', | ||
44 | followersUrl: url + '/followers', | ||
45 | followingUrl: url + '/following', | ||
46 | userId, | ||
47 | applicationId, | ||
48 | podId: null // It is our pod | ||
49 | }) | ||
50 | |||
51 | return accountInstance.save({ transaction: t }) | ||
52 | } | ||
53 | |||
38 | // --------------------------------------------------------------------------- | 54 | // --------------------------------------------------------------------------- |
39 | 55 | ||
40 | export { | 56 | export { |
41 | createUserAccountAndChannel | 57 | createUserAccountAndChannel, |
58 | createLocalAccount | ||
42 | } | 59 | } |
diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index f81383ce8..459d9d4a8 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts | |||
@@ -1,10 +1,10 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | 2 | ||
3 | import { addVideoChannelToFriends } from './friends' | ||
4 | import { database as db } from '../initializers' | 3 | import { database as db } from '../initializers' |
5 | import { logger } from '../helpers' | 4 | import { logger } from '../helpers' |
6 | import { AccountInstance } from '../models' | 5 | import { AccountInstance } from '../models' |
7 | import { VideoChannelCreate } from '../../shared/models' | 6 | import { VideoChannelCreate } from '../../shared/models' |
7 | import { sendCreateVideoChannel } from './activitypub/send-request' | ||
8 | 8 | ||
9 | async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account: AccountInstance, t: Sequelize.Transaction) { | 9 | async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account: AccountInstance, t: Sequelize.Transaction) { |
10 | const videoChannelData = { | 10 | const videoChannelData = { |
@@ -22,10 +22,7 @@ async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account | |||
22 | // Do not forget to add Account information to the created video channel | 22 | // Do not forget to add Account information to the created video channel |
23 | videoChannelCreated.Account = account | 23 | videoChannelCreated.Account = account |
24 | 24 | ||
25 | const remoteVideoChannel = videoChannelCreated.toAddRemoteJSON() | 25 | sendCreateVideoChannel(videoChannelCreated, t) |
26 | |||
27 | // Now we'll add the video channel's meta data to our friends | ||
28 | await addVideoChannelToFriends(remoteVideoChannel, t) | ||
29 | 26 | ||
30 | return videoChannelCreated | 27 | return videoChannelCreated |
31 | } | 28 | } |
diff --git a/server/middlewares/validators/activitypub/index.ts b/server/middlewares/validators/activitypub/index.ts index f1f26043e..84d1107fc 100644 --- a/server/middlewares/validators/activitypub/index.ts +++ b/server/middlewares/validators/activitypub/index.ts | |||
@@ -1,3 +1,2 @@ | |||
1 | export * from './pods' | 1 | export * from './activity' |
2 | export * from './signature' | 2 | export * from './signature' |
3 | export * from './videos' | ||
diff --git a/server/middlewares/validators/activitypub/pods.ts b/server/middlewares/validators/activitypub/pods.ts deleted file mode 100644 index f917b61ee..000000000 --- a/server/middlewares/validators/activitypub/pods.ts +++ /dev/null | |||
@@ -1,38 +0,0 @@ | |||
1 | import { body } from 'express-validator/check' | ||
2 | import * as express from 'express' | ||
3 | |||
4 | import { database as db } from '../../../initializers' | ||
5 | import { isHostValid, logger } from '../../../helpers' | ||
6 | import { checkErrors } from '../utils' | ||
7 | |||
8 | const remotePodsAddValidator = [ | ||
9 | body('host').custom(isHostValid).withMessage('Should have a host'), | ||
10 | body('email').isEmail().withMessage('Should have an email'), | ||
11 | body('publicKey').not().isEmpty().withMessage('Should have a public key'), | ||
12 | |||
13 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
14 | logger.debug('Checking podsAdd parameters', { parameters: req.body }) | ||
15 | |||
16 | checkErrors(req, res, () => { | ||
17 | db.Pod.loadByHost(req.body.host) | ||
18 | .then(pod => { | ||
19 | // Pod with this host already exists | ||
20 | if (pod) { | ||
21 | return res.sendStatus(409) | ||
22 | } | ||
23 | |||
24 | return next() | ||
25 | }) | ||
26 | .catch(err => { | ||
27 | logger.error('Cannot load pod by host.', err) | ||
28 | res.sendStatus(500) | ||
29 | }) | ||
30 | }) | ||
31 | } | ||
32 | ] | ||
33 | |||
34 | // --------------------------------------------------------------------------- | ||
35 | |||
36 | export { | ||
37 | remotePodsAddValidator | ||
38 | } | ||
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 46c00d679..0b7573d4f 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts | |||
@@ -2,7 +2,6 @@ export * from './account' | |||
2 | export * from './oembed' | 2 | export * from './oembed' |
3 | export * from './activitypub' | 3 | export * from './activitypub' |
4 | export * from './pagination' | 4 | export * from './pagination' |
5 | export * from './pods' | ||
6 | export * from './sort' | 5 | export * from './sort' |
7 | export * from './users' | 6 | export * from './users' |
8 | export * from './videos' | 7 | export * from './videos' |
diff --git a/server/middlewares/validators/pods.ts b/server/middlewares/validators/pods.ts deleted file mode 100644 index 8465fea53..000000000 --- a/server/middlewares/validators/pods.ts +++ /dev/null | |||
@@ -1,73 +0,0 @@ | |||
1 | import { body, param } from 'express-validator/check' | ||
2 | import * as express from 'express' | ||
3 | |||
4 | import { database as db } from '../../initializers/database' | ||
5 | import { checkErrors } from './utils' | ||
6 | import { logger, isEachUniqueHostValid, isTestInstance } from '../../helpers' | ||
7 | import { CONFIG } from '../../initializers' | ||
8 | import { hasFriends } from '../../lib' | ||
9 | |||
10 | const makeFriendsValidator = [ | ||
11 | body('hosts').custom(isEachUniqueHostValid).withMessage('Should have an array of unique hosts'), | ||
12 | |||
13 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
14 | // Force https if the administrator wants to make friends | ||
15 | if (isTestInstance() === false && CONFIG.WEBSERVER.SCHEME === 'http') { | ||
16 | return res.status(400) | ||
17 | .json({ | ||
18 | error: 'Cannot make friends with a non HTTPS web server.' | ||
19 | }) | ||
20 | .end() | ||
21 | } | ||
22 | |||
23 | logger.debug('Checking makeFriends parameters', { parameters: req.body }) | ||
24 | |||
25 | checkErrors(req, res, () => { | ||
26 | hasFriends() | ||
27 | .then(heHasFriends => { | ||
28 | if (heHasFriends === true) { | ||
29 | // We need to quit our friends before make new ones | ||
30 | return res.sendStatus(409) | ||
31 | } | ||
32 | |||
33 | return next() | ||
34 | }) | ||
35 | .catch(err => { | ||
36 | logger.error('Cannot know if we have friends.', err) | ||
37 | res.sendStatus(500) | ||
38 | }) | ||
39 | }) | ||
40 | } | ||
41 | ] | ||
42 | |||
43 | const podRemoveValidator = [ | ||
44 | param('id').isNumeric().not().isEmpty().withMessage('Should have a valid id'), | ||
45 | |||
46 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
47 | logger.debug('Checking podRemoveValidator parameters', { parameters: req.params }) | ||
48 | |||
49 | checkErrors(req, res, () => { | ||
50 | db.Pod.load(req.params.id) | ||
51 | .then(pod => { | ||
52 | if (!pod) { | ||
53 | logger.error('Cannot find pod %d.', req.params.id) | ||
54 | return res.sendStatus(404) | ||
55 | } | ||
56 | |||
57 | res.locals.pod = pod | ||
58 | return next() | ||
59 | }) | ||
60 | .catch(err => { | ||
61 | logger.error('Cannot load pod %d.', req.params.id, err) | ||
62 | res.sendStatus(500) | ||
63 | }) | ||
64 | }) | ||
65 | } | ||
66 | ] | ||
67 | |||
68 | // --------------------------------------------------------------------------- | ||
69 | |||
70 | export { | ||
71 | makeFriendsValidator, | ||
72 | podRemoveValidator | ||
73 | } | ||
diff --git a/server/models/account/account-interface.ts b/server/models/account/account-interface.ts index 2ef3e2246..a662eb992 100644 --- a/server/models/account/account-interface.ts +++ b/server/models/account/account-interface.ts | |||
@@ -13,8 +13,8 @@ export namespace AccountMethods { | |||
13 | export type LoadAccountByPodAndUUID = (uuid: string, podId: number, transaction: Sequelize.Transaction) => Bluebird<AccountInstance> | 13 | export type LoadAccountByPodAndUUID = (uuid: string, podId: number, transaction: Sequelize.Transaction) => Bluebird<AccountInstance> |
14 | export type LoadLocalAccountByName = (name: string) => Bluebird<AccountInstance> | 14 | export type LoadLocalAccountByName = (name: string) => Bluebird<AccountInstance> |
15 | export type ListOwned = () => Bluebird<AccountInstance[]> | 15 | export type ListOwned = () => Bluebird<AccountInstance[]> |
16 | export type ListFollowerUrlsForApi = (name: string, start: number, count: number) => Promise< ResultList<string> > | 16 | export type ListFollowerUrlsForApi = (name: string, start: number, count?: number) => Promise< ResultList<string> > |
17 | export type ListFollowingUrlsForApi = (name: string, start: number, count: number) => Promise< ResultList<string> > | 17 | export type ListFollowingUrlsForApi = (name: string, start: number, count?: number) => Promise< ResultList<string> > |
18 | 18 | ||
19 | export type ToActivityPubObject = (this: AccountInstance) => ActivityPubActor | 19 | export type ToActivityPubObject = (this: AccountInstance) => ActivityPubActor |
20 | export type IsOwned = (this: AccountInstance) => boolean | 20 | export type IsOwned = (this: AccountInstance) => boolean |
diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 00c0aefd4..a79e13880 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts | |||
@@ -268,14 +268,15 @@ function afterDestroy (account: AccountInstance) { | |||
268 | uuid: account.uuid | 268 | uuid: account.uuid |
269 | } | 269 | } |
270 | 270 | ||
271 | return removeVideoAccountToFriends(removeVideoAccountToFriendsParams) | 271 | // FIXME: remove account in followers |
272 | // return removeVideoAccountToFriends(removeVideoAccountToFriendsParams) | ||
272 | } | 273 | } |
273 | 274 | ||
274 | return undefined | 275 | return undefined |
275 | } | 276 | } |
276 | 277 | ||
277 | toActivityPubObject = function (this: AccountInstance) { | 278 | toActivityPubObject = function (this: AccountInstance) { |
278 | const type = this.podId ? 'Application' : 'Person' | 279 | const type = this.podId ? 'Application' as 'Application' : 'Person' as 'Person' |
279 | 280 | ||
280 | const json = { | 281 | const json = { |
281 | type, | 282 | type, |
@@ -346,11 +347,11 @@ listOwned = function () { | |||
346 | return Account.findAll(query) | 347 | return Account.findAll(query) |
347 | } | 348 | } |
348 | 349 | ||
349 | listFollowerUrlsForApi = function (name: string, start: number, count: number) { | 350 | listFollowerUrlsForApi = function (name: string, start: number, count?: number) { |
350 | return createListFollowForApiQuery('followers', name, start, count) | 351 | return createListFollowForApiQuery('followers', name, start, count) |
351 | } | 352 | } |
352 | 353 | ||
353 | listFollowingUrlsForApi = function (name: string, start: number, count: number) { | 354 | listFollowingUrlsForApi = function (name: string, start: number, count?: number) { |
354 | return createListFollowForApiQuery('following', name, start, count) | 355 | return createListFollowForApiQuery('following', name, start, count) |
355 | } | 356 | } |
356 | 357 | ||
@@ -405,7 +406,7 @@ loadAccountByPodAndUUID = function (uuid: string, podId: number, transaction: Se | |||
405 | 406 | ||
406 | // ------------------------------ UTILS ------------------------------ | 407 | // ------------------------------ UTILS ------------------------------ |
407 | 408 | ||
408 | async function createListFollowForApiQuery (type: 'followers' | 'following', name: string, start: number, count: number) { | 409 | async function createListFollowForApiQuery (type: 'followers' | 'following', name: string, start: number, count?: number) { |
409 | let firstJoin: string | 410 | let firstJoin: string |
410 | let secondJoin: string | 411 | let secondJoin: string |
411 | 412 | ||
@@ -421,11 +422,13 @@ async function createListFollowForApiQuery (type: 'followers' | 'following', nam | |||
421 | const tasks: Promise<any>[] = [] | 422 | const tasks: Promise<any>[] = [] |
422 | 423 | ||
423 | for (const selection of selections) { | 424 | for (const selection of selections) { |
424 | const query = 'SELECT ' + selection + ' FROM "Account" ' + | 425 | let query = 'SELECT ' + selection + ' FROM "Account" ' + |
425 | 'INNER JOIN "AccountFollower" ON "AccountFollower"."' + firstJoin + '" = "Account"."id" ' + | 426 | 'INNER JOIN "AccountFollower" ON "AccountFollower"."' + firstJoin + '" = "Account"."id" ' + |
426 | 'INNER JOIN "Account" AS "Followers" ON "Followers"."id" = "AccountFollower"."' + secondJoin + '" ' + | 427 | 'INNER JOIN "Account" AS "Followers" ON "Followers"."id" = "AccountFollower"."' + secondJoin + '" ' + |
427 | 'WHERE "Account"."name" = \'$name\' ' + | 428 | 'WHERE "Account"."name" = \'$name\' ' + |
428 | 'LIMIT ' + start + ', ' + count | 429 | 'LIMIT ' + start |
430 | |||
431 | if (count !== undefined) query += ', ' + count | ||
429 | 432 | ||
430 | const options = { | 433 | const options = { |
431 | bind: { name }, | 434 | bind: { name }, |
diff --git a/server/models/job/job-interface.ts b/server/models/job/job-interface.ts index 163930a4f..411a05029 100644 --- a/server/models/job/job-interface.ts +++ b/server/models/job/job-interface.ts | |||
@@ -14,7 +14,7 @@ export interface JobClass { | |||
14 | export interface JobAttributes { | 14 | export interface JobAttributes { |
15 | state: JobState | 15 | state: JobState |
16 | handlerName: string | 16 | handlerName: string |
17 | handlerInputData: object | 17 | handlerInputData: any |
18 | } | 18 | } |
19 | 19 | ||
20 | export interface JobInstance extends JobClass, JobAttributes, Sequelize.Instance<JobAttributes> { | 20 | export interface JobInstance extends JobClass, JobAttributes, Sequelize.Instance<JobAttributes> { |
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 93a611fa0..183ff3436 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | 2 | ||
3 | import { isVideoChannelNameValid, isVideoChannelDescriptionValid } from '../../helpers' | 3 | import { isVideoChannelNameValid, isVideoChannelDescriptionValid } from '../../helpers' |
4 | import { removeVideoChannelToFriends } from '../../lib' | ||
5 | 4 | ||
6 | import { addMethodsToModel, getSort } from '../utils' | 5 | import { addMethodsToModel, getSort } from '../utils' |
7 | import { | 6 | import { |
@@ -143,12 +142,13 @@ toFormattedJSON = function (this: VideoChannelInstance) { | |||
143 | 142 | ||
144 | toActivityPubObject = function (this: VideoChannelInstance) { | 143 | toActivityPubObject = function (this: VideoChannelInstance) { |
145 | const json = { | 144 | const json = { |
145 | type: 'VideoChannel' as 'VideoChannel', | ||
146 | id: this.url, | ||
146 | uuid: this.uuid, | 147 | uuid: this.uuid, |
148 | content: this.description, | ||
147 | name: this.name, | 149 | name: this.name, |
148 | description: this.description, | 150 | published: this.createdAt, |
149 | createdAt: this.createdAt, | 151 | updated: this.updatedAt |
150 | updatedAt: this.updatedAt, | ||
151 | ownerUUID: this.Account.uuid | ||
152 | } | 152 | } |
153 | 153 | ||
154 | return json | 154 | return json |
@@ -180,7 +180,7 @@ function afterDestroy (videoChannel: VideoChannelInstance) { | |||
180 | uuid: videoChannel.uuid | 180 | uuid: videoChannel.uuid |
181 | } | 181 | } |
182 | 182 | ||
183 | return removeVideoChannelToFriends(removeVideoChannelToFriendsParams) | 183 | // FIXME: send remove event to followers |
184 | } | 184 | } |
185 | 185 | ||
186 | return undefined | 186 | return undefined |
@@ -277,7 +277,7 @@ loadByUUIDOrUrl = function (uuid: string, url: string, t?: Sequelize.Transaction | |||
277 | { uuid }, | 277 | { uuid }, |
278 | { url } | 278 | { url } |
279 | ] | 279 | ] |
280 | }, | 280 | } |
281 | } | 281 | } |
282 | 282 | ||
283 | if (t !== undefined) query.transaction = t | 283 | if (t !== undefined) query.transaction = t |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index b5d333347..10ae5097c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -1,58 +1,52 @@ | |||
1 | import * as safeBuffer from 'safe-buffer' | ||
2 | const Buffer = safeBuffer.Buffer | ||
3 | import * as magnetUtil from 'magnet-uri' | ||
4 | import { map, maxBy, truncate } from 'lodash' | 1 | import { map, maxBy, truncate } from 'lodash' |
2 | import * as magnetUtil from 'magnet-uri' | ||
5 | import * as parseTorrent from 'parse-torrent' | 3 | import * as parseTorrent from 'parse-torrent' |
6 | import { join } from 'path' | 4 | import { join } from 'path' |
5 | import * as safeBuffer from 'safe-buffer' | ||
7 | import * as Sequelize from 'sequelize' | 6 | import * as Sequelize from 'sequelize' |
8 | 7 | import { VideoPrivacy, VideoResolution } from '../../../shared' | |
9 | import { TagInstance } from './tag-interface' | 8 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects/video-torrent-object' |
10 | import { | 9 | import { |
11 | logger, | 10 | createTorrentPromise, |
12 | isVideoNameValid, | 11 | generateImageFromVideoFile, |
12 | getActivityPubUrl, | ||
13 | getVideoFileHeight, | ||
13 | isVideoCategoryValid, | 14 | isVideoCategoryValid, |
14 | isVideoLicenceValid, | ||
15 | isVideoLanguageValid, | ||
16 | isVideoNSFWValid, | ||
17 | isVideoDescriptionValid, | 15 | isVideoDescriptionValid, |
18 | isVideoDurationValid, | 16 | isVideoDurationValid, |
17 | isVideoLanguageValid, | ||
18 | isVideoLicenceValid, | ||
19 | isVideoNameValid, | ||
20 | isVideoNSFWValid, | ||
19 | isVideoPrivacyValid, | 21 | isVideoPrivacyValid, |
20 | readFileBufferPromise, | 22 | logger, |
21 | unlinkPromise, | ||
22 | renamePromise, | 23 | renamePromise, |
23 | writeFilePromise, | ||
24 | createTorrentPromise, | ||
25 | statPromise, | 24 | statPromise, |
26 | generateImageFromVideoFile, | ||
27 | transcode, | 25 | transcode, |
28 | getVideoFileHeight, | 26 | unlinkPromise, |
29 | getActivityPubUrl | 27 | writeFilePromise |
30 | } from '../../helpers' | 28 | } from '../../helpers' |
31 | import { | 29 | import { |
30 | API_VERSION, | ||
32 | CONFIG, | 31 | CONFIG, |
32 | CONSTRAINTS_FIELDS, | ||
33 | PREVIEWS_SIZE, | ||
33 | REMOTE_SCHEME, | 34 | REMOTE_SCHEME, |
34 | STATIC_PATHS, | 35 | STATIC_PATHS, |
36 | THUMBNAILS_SIZE, | ||
35 | VIDEO_CATEGORIES, | 37 | VIDEO_CATEGORIES, |
36 | VIDEO_LICENCES, | ||
37 | VIDEO_LANGUAGES, | 38 | VIDEO_LANGUAGES, |
38 | THUMBNAILS_SIZE, | 39 | VIDEO_LICENCES, |
39 | PREVIEWS_SIZE, | ||
40 | CONSTRAINTS_FIELDS, | ||
41 | API_VERSION, | ||
42 | VIDEO_PRIVACIES | 40 | VIDEO_PRIVACIES |
43 | } from '../../initializers' | 41 | } from '../../initializers' |
44 | import { removeVideoToFriends } from '../../lib' | ||
45 | import { VideoResolution, VideoPrivacy } from '../../../shared' | ||
46 | import { VideoFileInstance, VideoFileModel } from './video-file-interface' | ||
47 | 42 | ||
48 | import { addMethodsToModel, getSort } from '../utils' | 43 | import { addMethodsToModel, getSort } from '../utils' |
49 | import { | ||
50 | VideoInstance, | ||
51 | VideoAttributes, | ||
52 | 44 | ||
53 | VideoMethods | 45 | import { TagInstance } from './tag-interface' |
54 | } from './video-interface' | 46 | import { VideoFileInstance, VideoFileModel } from './video-file-interface' |
55 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects/video-torrent-object' | 47 | import { VideoAttributes, VideoInstance, VideoMethods } from './video-interface' |
48 | |||
49 | const Buffer = safeBuffer.Buffer | ||
56 | 50 | ||
57 | let Video: Sequelize.Model<VideoInstance, VideoAttributes> | 51 | let Video: Sequelize.Model<VideoInstance, VideoAttributes> |
58 | let getOriginalFile: VideoMethods.GetOriginalFile | 52 | let getOriginalFile: VideoMethods.GetOriginalFile |
@@ -374,8 +368,8 @@ function afterDestroy (video: VideoInstance) { | |||
374 | } | 368 | } |
375 | 369 | ||
376 | tasks.push( | 370 | tasks.push( |
377 | video.removePreview(), | 371 | video.removePreview() |
378 | removeVideoToFriends(removeVideoToFriendsParams) | 372 | // FIXME: remove video for followers |
379 | ) | 373 | ) |
380 | 374 | ||
381 | // Remove physical files and torrents | 375 | // Remove physical files and torrents |
@@ -566,7 +560,7 @@ toActivityPubObject = function (this: VideoInstance) { | |||
566 | const { baseUrlHttp, baseUrlWs } = getBaseUrls(this) | 560 | const { baseUrlHttp, baseUrlWs } = getBaseUrls(this) |
567 | 561 | ||
568 | const tag = this.Tags.map(t => ({ | 562 | const tag = this.Tags.map(t => ({ |
569 | type: 'Hashtag', | 563 | type: 'Hashtag' as 'Hashtag', |
570 | name: t.name | 564 | name: t.name |
571 | })) | 565 | })) |
572 | 566 | ||
@@ -596,7 +590,7 @@ toActivityPubObject = function (this: VideoInstance) { | |||
596 | } | 590 | } |
597 | 591 | ||
598 | const videoObject: VideoTorrentObject = { | 592 | const videoObject: VideoTorrentObject = { |
599 | type: 'Video', | 593 | type: 'Video' as 'Video', |
600 | id: getActivityPubUrl('video', this.uuid), | 594 | id: getActivityPubUrl('video', this.uuid), |
601 | name: this.name, | 595 | name: this.name, |
602 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | 596 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration |
@@ -604,15 +598,15 @@ toActivityPubObject = function (this: VideoInstance) { | |||
604 | uuid: this.uuid, | 598 | uuid: this.uuid, |
605 | tag, | 599 | tag, |
606 | category: { | 600 | category: { |
607 | id: this.category, | 601 | identifier: this.category + '', |
608 | label: this.getCategoryLabel() | 602 | name: this.getCategoryLabel() |
609 | }, | 603 | }, |
610 | licence: { | 604 | licence: { |
611 | id: this.licence, | 605 | identifier: this.licence + '', |
612 | name: this.getLicenceLabel() | 606 | name: this.getLicenceLabel() |
613 | }, | 607 | }, |
614 | language: { | 608 | language: { |
615 | id: this.language, | 609 | identifier: this.language + '', |
616 | name: this.getLanguageLabel() | 610 | name: this.getLanguageLabel() |
617 | }, | 611 | }, |
618 | views: this.views, | 612 | views: this.views, |
diff --git a/server/tests/api/video-channels.ts b/server/tests/api/video-channels.ts index 601b527ef..b851d7185 100644 --- a/server/tests/api/video-channels.ts +++ b/server/tests/api/video-channels.ts | |||
@@ -65,7 +65,7 @@ describe('Test a video channels', function () { | |||
65 | }) | 65 | }) |
66 | 66 | ||
67 | it('Should have two video channels when getting author channels', async () => { | 67 | it('Should have two video channels when getting author channels', async () => { |
68 | const res = await getAuthorVideoChannelsList(server.url, userInfo.author.uuid) | 68 | const res = await getAuthorVideoChannelsList(server.url, userInfo.account.uuid) |
69 | 69 | ||
70 | expect(res.body.total).to.equal(2) | 70 | expect(res.body.total).to.equal(2) |
71 | expect(res.body.data).to.be.an('array') | 71 | expect(res.body.data).to.be.an('array') |
diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts index dc562c00a..a2494da25 100644 --- a/shared/models/activitypub/activity.ts +++ b/shared/models/activitypub/activity.ts | |||
@@ -4,7 +4,7 @@ import { | |||
4 | } from './objects' | 4 | } from './objects' |
5 | import { ActivityPubSignature } from './activitypub-signature' | 5 | import { ActivityPubSignature } from './activitypub-signature' |
6 | 6 | ||
7 | export type Activity = ActivityCreate | ActivityUpdate | ActivityFlag | 7 | export type Activity = ActivityCreate | ActivityAdd | ActivityUpdate | ActivityFlag |
8 | 8 | ||
9 | // Flag -> report abuse | 9 | // Flag -> report abuse |
10 | export type ActivityType = 'Create' | 'Add' | 'Update' | 'Flag' | 10 | export type ActivityType = 'Create' | 'Add' | 'Update' | 'Flag' |
diff --git a/shared/models/pods/pod.model.ts b/shared/models/pods/pod.model.ts index d25421936..ff9e8f2f5 100644 --- a/shared/models/pods/pod.model.ts +++ b/shared/models/pods/pod.model.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | export interface Pod { | 1 | export interface Pod { |
2 | id: number, | 2 | id: number, |
3 | host: string, | 3 | host: string, |
4 | email: string, | ||
5 | score: number, | 4 | score: number, |
6 | createdAt: Date | 5 | createdAt: Date |
7 | } | 6 | } |
diff --git a/shared/models/users/user.model.ts b/shared/models/users/user.model.ts index ee2147590..a8012734c 100644 --- a/shared/models/users/user.model.ts +++ b/shared/models/users/user.model.ts | |||
@@ -9,7 +9,7 @@ export interface User { | |||
9 | role: UserRole | 9 | role: UserRole |
10 | videoQuota: number | 10 | videoQuota: number |
11 | createdAt: Date, | 11 | createdAt: Date, |
12 | author: { | 12 | account: { |
13 | id: number | 13 | id: number |
14 | uuid: string | 14 | uuid: string |
15 | } | 15 | } |
@@ -3473,12 +3473,6 @@ repeat-string@^1.5.2: | |||
3473 | version "1.6.1" | 3473 | version "1.6.1" |
3474 | resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" | 3474 | resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" |
3475 | 3475 | ||
3476 | request-replay@^1.0.2: | ||
3477 | version "1.0.4" | ||
3478 | resolved "https://registry.yarnpkg.com/request-replay/-/request-replay-1.0.4.tgz#b6e5953a7eb39fc8a48e8111c277d35355adfe06" | ||
3479 | dependencies: | ||
3480 | retry "^0.10.0" | ||
3481 | |||
3482 | request@2.81.0: | 3476 | request@2.81.0: |
3483 | version "2.81.0" | 3477 | version "2.81.0" |
3484 | resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" | 3478 | resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" |
@@ -3564,10 +3558,6 @@ retry-as-promised@^2.3.1: | |||
3564 | bluebird "^3.4.6" | 3558 | bluebird "^3.4.6" |
3565 | debug "^2.6.9" | 3559 | debug "^2.6.9" |
3566 | 3560 | ||
3567 | retry@^0.10.0: | ||
3568 | version "0.10.1" | ||
3569 | resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4" | ||
3570 | |||
3571 | rimraf@2, rimraf@^2.2.8, rimraf@^2.4.2, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.1: | 3561 | rimraf@2, rimraf@^2.2.8, rimraf@^2.4.2, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.1: |
3572 | version "2.6.2" | 3562 | version "2.6.2" |
3573 | resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" | 3563 | resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" |