From 571389d43b8fc8aaf27e77c06f19b320b08dbbc9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Nov 2017 17:27:49 +0100 Subject: [PATCH] Make it compile at least --- package.json | 1 - scripts/update-host.ts | 8 +- server.ts | 16 +- server/controllers/activitypub/index.ts | 2 + server/controllers/api/index.ts | 4 - server/controllers/api/pods.ts | 66 +------ server/controllers/api/request-schedulers.ts | 53 ------ server/controllers/api/users.ts | 42 ++--- server/controllers/api/videos/abuse.ts | 4 +- server/controllers/api/videos/channel.ts | 37 ++-- server/controllers/api/videos/index.ts | 88 +++------ server/controllers/api/videos/rate.ts | 80 ++------- server/helpers/activitypub.ts | 46 +++-- server/helpers/requests.ts | 83 +-------- server/helpers/webfinger.ts | 2 +- server/initializers/constants.ts | 6 - server/initializers/database.ts | 5 - server/initializers/installer.ts | 24 ++- .../migrations/0075-video-resolutions.ts | 1 - server/lib/activitypub/index.ts | 1 + server/lib/activitypub/misc.ts | 6 +- server/lib/activitypub/process-flag.ts | 2 +- server/lib/activitypub/send-request.ts | 31 +++- server/lib/cache/videos-preview-cache.ts | 5 +- server/lib/index.ts | 2 - .../http-request-broadcast-handler.ts | 21 ++- .../http-request-job-scheduler.ts | 7 +- .../http-request-unicast-handler.ts | 19 +- server/lib/jobs/job-scheduler.ts | 33 ++-- .../transcoding-job-scheduler.ts | 13 +- .../video-file-optimizer-handler.ts | 17 +- .../video-file-transcoder-handler.ts | 11 +- .../lib/request/abstract-request-scheduler.ts | 168 ------------------ server/lib/request/index.ts | 4 - server/lib/request/request-scheduler.ts | 96 ---------- .../request/request-video-event-scheduler.ts | 129 -------------- .../request/request-video-qadu-scheduler.ts | 148 --------------- server/lib/user.ts | 47 +++-- server/lib/video-channel.ts | 7 +- .../validators/activitypub/index.ts | 3 +- .../validators/activitypub/pods.ts | 38 ---- server/middlewares/validators/index.ts | 1 - server/middlewares/validators/pods.ts | 73 -------- server/models/account/account-interface.ts | 4 +- server/models/account/account.ts | 17 +- server/models/job/job-interface.ts | 2 +- server/models/video/video-channel.ts | 14 +- server/models/video/video.ts | 72 ++++---- server/tests/api/video-channels.ts | 2 +- shared/models/activitypub/activity.ts | 2 +- shared/models/pods/pod.model.ts | 1 - shared/models/users/user.model.ts | 2 +- yarn.lock | 10 -- 53 files changed, 331 insertions(+), 1245 deletions(-) delete mode 100644 server/controllers/api/request-schedulers.ts delete mode 100644 server/lib/request/abstract-request-scheduler.ts delete mode 100644 server/lib/request/index.ts delete mode 100644 server/lib/request/request-scheduler.ts delete mode 100644 server/lib/request/request-video-event-scheduler.ts delete mode 100644 server/lib/request/request-video-qadu-scheduler.ts delete mode 100644 server/middlewares/validators/activitypub/pods.ts delete mode 100644 server/middlewares/validators/pods.ts diff --git a/package.json b/package.json index a49b4d800..68df1cb9b 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "pg": "^6.4.2", "pg-hstore": "^2.3.2", "request": "^2.81.0", - "request-replay": "^1.0.2", "rimraf": "^2.5.4", "safe-buffer": "^5.0.1", "scripty": "^1.5.0", diff --git a/scripts/update-host.ts b/scripts/update-host.ts index 06d84a658..7c46dc52b 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts @@ -1,11 +1,11 @@ -import * as Promise from 'bluebird' - import { database as db } from '../server/initializers/database' -import { hasFriends } from '../server/lib/friends' +// import { hasFriends } from '../server/lib/friends' db.init(true) .then(() => { - return hasFriends() + // FIXME: check if has followers + // return hasFriends() + return true }) .then(itHasFriends => { if (itHasFriends === true) { diff --git a/server.ts b/server.ts index f50e5bad4..0878fe757 100644 --- a/server.ts +++ b/server.ts @@ -46,7 +46,7 @@ db.init(false).then(() => onDatabaseInitDone()) // ----------- PeerTube modules ----------- import { migrate, installApplication } from './server/initializers' -import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib' +import { httpRequestJobScheduler, transcodingJobScheduler, VideosPreviewCache } from './server/lib' import { apiRouter, clientsRouter, staticRouter, servicesRouter } from './server/controllers' // ----------- Command line ----------- @@ -146,19 +146,13 @@ function onDatabaseInitDone () { const port = CONFIG.LISTEN.PORT // Run the migration scripts if needed migrate() - .then(() => { - return installApplication() - }) + .then(() => installApplication()) .then(() => { // ----------- Make the server listening ----------- - server.listen(port, function () { - // Activate the communication with friends - activateSchedulers() - - // Activate job scheduler - JobScheduler.Instance.activate() - + server.listen(port, () => { VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) + httpRequestJobScheduler.activate() + transcodingJobScheduler.activate() logger.info('Server listening on port %d', port) logger.info('Web server: %s', CONFIG.WEBSERVER.URL) diff --git a/server/controllers/activitypub/index.ts b/server/controllers/activitypub/index.ts index 7a4602b37..2b0e2a489 100644 --- a/server/controllers/activitypub/index.ts +++ b/server/controllers/activitypub/index.ts @@ -2,10 +2,12 @@ import * as express from 'express' import { badRequest } from '../../helpers' import { inboxRouter } from './inbox' +import { activityPubClientRouter } from './client' const remoteRouter = express.Router() remoteRouter.use('/inbox', inboxRouter) +remoteRouter.use('/', activityPubClientRouter) remoteRouter.use('/*', badRequest) // --------------------------------------------------------------------------- diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index a9205b33c..2e949d531 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts @@ -5,8 +5,6 @@ import { badRequest } from '../../helpers' import { oauthClientsRouter } from './oauth-clients' import { configRouter } from './config' import { podsRouter } from './pods' -import { remoteRouter } from './remote' -import { requestSchedulerRouter } from './request-schedulers' import { usersRouter } from './users' import { videosRouter } from './videos' @@ -15,8 +13,6 @@ const apiRouter = express.Router() apiRouter.use('/oauth-clients', oauthClientsRouter) apiRouter.use('/config', configRouter) apiRouter.use('/pods', podsRouter) -apiRouter.use('/remote', remoteRouter) -apiRouter.use('/request-schedulers', requestSchedulerRouter) apiRouter.use('/users', usersRouter) apiRouter.use('/videos', videosRouter) apiRouter.use('/ping', pong) diff --git a/server/controllers/api/pods.ts b/server/controllers/api/pods.ts index b44cd6b83..43df3f66f 100644 --- a/server/controllers/api/pods.ts +++ b/server/controllers/api/pods.ts @@ -1,26 +1,7 @@ import * as express from 'express' - +import { getFormattedObjects } from '../../helpers' import { database as db } from '../../initializers/database' -import { logger, getFormattedObjects } from '../../helpers' -import { - makeFriends, - quitFriends, - removeFriend -} from '../../lib' -import { - authenticate, - ensureUserHasRight, - makeFriendsValidator, - setBodyHostsPort, - podRemoveValidator, - paginationValidator, - setPagination, - setPodsSort, - podsSortValidator, - asyncMiddleware -} from '../../middlewares' -import { PodInstance } from '../../models' -import { UserRight } from '../../../shared' +import { asyncMiddleware, paginationValidator, podsSortValidator, setPagination, setPodsSort } from '../../middlewares' const podsRouter = express.Router() @@ -31,24 +12,6 @@ podsRouter.get('/', setPagination, asyncMiddleware(listPods) ) -podsRouter.post('/make-friends', - authenticate, - ensureUserHasRight(UserRight.MANAGE_PODS), - makeFriendsValidator, - setBodyHostsPort, - asyncMiddleware(makeFriendsController) -) -podsRouter.get('/quit-friends', - authenticate, - ensureUserHasRight(UserRight.MANAGE_PODS), - asyncMiddleware(quitFriendsController) -) -podsRouter.delete('/:id', - authenticate, - ensureUserHasRight(UserRight.MANAGE_PODS), - podRemoveValidator, - asyncMiddleware(removeFriendController) -) // --------------------------------------------------------------------------- @@ -63,28 +26,3 @@ async function listPods (req: express.Request, res: express.Response, next: expr return res.json(getFormattedObjects(resultList.data, resultList.total)) } - -async function makeFriendsController (req: express.Request, res: express.Response, next: express.NextFunction) { - const hosts = req.body.hosts as string[] - - // Don't wait the process that could be long - makeFriends(hosts) - .then(() => logger.info('Made friends!')) - .catch(err => logger.error('Could not make friends.', err)) - - return res.type('json').status(204).end() -} - -async function quitFriendsController (req: express.Request, res: express.Response, next: express.NextFunction) { - await quitFriends() - - return res.type('json').status(204).end() -} - -async function removeFriendController (req: express.Request, res: express.Response, next: express.NextFunction) { - const pod = res.locals.pod as PodInstance - - await removeFriend(pod) - - return res.type('json').status(204).end() -} diff --git a/server/controllers/api/request-schedulers.ts b/server/controllers/api/request-schedulers.ts deleted file mode 100644 index 4c8fbe18b..000000000 --- a/server/controllers/api/request-schedulers.ts +++ /dev/null @@ -1,53 +0,0 @@ -import * as express from 'express' -import * as Bluebird from 'bluebird' - -import { - AbstractRequestScheduler, - getRequestScheduler, - getRequestVideoQaduScheduler, - getRequestVideoEventScheduler -} from '../../lib' -import { authenticate, ensureUserHasRight, asyncMiddleware } from '../../middlewares' -import { RequestSchedulerStatsAttributes, UserRight } from '../../../shared' - -const requestSchedulerRouter = express.Router() - -requestSchedulerRouter.get('/stats', - authenticate, - ensureUserHasRight(UserRight.MANAGE_REQUEST_SCHEDULERS), - asyncMiddleware(getRequestSchedulersStats) -) - -// --------------------------------------------------------------------------- - -export { - requestSchedulerRouter -} - -// --------------------------------------------------------------------------- - -async function getRequestSchedulersStats (req: express.Request, res: express.Response, next: express.NextFunction) { - const result = await Bluebird.props({ - requestScheduler: buildRequestSchedulerStats(getRequestScheduler()), - requestVideoQaduScheduler: buildRequestSchedulerStats(getRequestVideoQaduScheduler()), - requestVideoEventScheduler: buildRequestSchedulerStats(getRequestVideoEventScheduler()) - }) - - return res.json(result) -} - -// --------------------------------------------------------------------------- - -async function buildRequestSchedulerStats (requestScheduler: AbstractRequestScheduler) { - const count = await requestScheduler.remainingRequestsCount() - - const result: RequestSchedulerStatsAttributes = { - totalRequests: count, - requestsLimitPods: requestScheduler.limitPods, - requestsLimitPerPod: requestScheduler.limitPerPod, - remainingMilliSeconds: requestScheduler.remainingMilliSeconds(), - milliSecondsInterval: requestScheduler.requestInterval - } - - return result -} diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts index 9ec6feb57..41ffb64cb 100644 --- a/server/controllers/api/users.ts +++ b/server/controllers/api/users.ts @@ -1,37 +1,29 @@ import * as express from 'express' - -import { database as db, CONFIG } from '../../initializers' -import { logger, getFormattedObjects, retryTransactionWrapper } from '../../helpers' +import { UserCreate, UserRight, UserRole, UserUpdate, UserUpdateMe, UserVideoRate as FormattedUserVideoRate } from '../../../shared' +import { getFormattedObjects, logger, retryTransactionWrapper } from '../../helpers' +import { CONFIG, database as db } from '../../initializers' +import { createUserAccountAndChannel } from '../../lib' import { + asyncMiddleware, authenticate, ensureUserHasRight, ensureUserRegistrationAllowed, - usersAddValidator, - usersRegisterValidator, - usersUpdateValidator, - usersUpdateMeValidator, - usersRemoveValidator, - usersVideoRatingValidator, - usersGetValidator, paginationValidator, setPagination, - usersSortValidator, setUsersSort, token, - asyncMiddleware + usersAddValidator, + usersGetValidator, + usersRegisterValidator, + usersRemoveValidator, + usersSortValidator, + usersUpdateMeValidator, + usersUpdateValidator, + usersVideoRatingValidator } from '../../middlewares' -import { - UserVideoRate as FormattedUserVideoRate, - UserCreate, - UserUpdate, - UserUpdateMe, - UserRole, - UserRight -} from '../../../shared' -import { createUserAccountAndChannel } from '../../lib' -import { UserInstance } from '../../models' -import { videosSortValidator } from '../../middlewares/validators/sort' import { setVideosSort } from '../../middlewares/sort' +import { videosSortValidator } from '../../middlewares/validators/sort' +import { UserInstance } from '../../models' const usersRouter = express.Router() @@ -176,9 +168,9 @@ function getUser (req: express.Request, res: express.Response, next: express.Nex async function getUserVideoRating (req: express.Request, res: express.Response, next: express.NextFunction) { const videoId = +req.params.videoId - const userId = +res.locals.oauth.token.User.id + const accountId = +res.locals.oauth.token.User.Account.id - const ratingObj = await db.UserVideoRate.load(userId, videoId, null) + const ratingObj = await db.AccountVideoRate.load(accountId, videoId, null) const rating = ratingObj ? ratingObj.type : 'none' const json: FormattedUserVideoRate = { diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 04349042b..7a3471116 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts @@ -1,7 +1,6 @@ import * as express from 'express' import { database as db } from '../../../initializers/database' -import * as friends from '../../../lib/friends' import { logger, getFormattedObjects, @@ -84,7 +83,8 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { videoUUID: videoInstance.uuid } - await friends.reportAbuseVideoToFriend(reportData, videoInstance, t) + // await friends.reportAbuseVideoToFriend(reportData, videoInstance, t) + // TODO: send abuse to origin pod } }) diff --git a/server/controllers/api/videos/channel.ts b/server/controllers/api/videos/channel.ts index 4d1f03903..656bc3129 100644 --- a/server/controllers/api/videos/channel.ts +++ b/server/controllers/api/videos/channel.ts @@ -1,31 +1,23 @@ import * as express from 'express' - +import { VideoChannelCreate, VideoChannelUpdate } from '../../../../shared' +import { getFormattedObjects, logger, resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers' import { database as db } from '../../../initializers' +import { createVideoChannel } from '../../../lib' import { - logger, - getFormattedObjects, - retryTransactionWrapper, - resetSequelizeInstance -} from '../../../helpers' -import { + asyncMiddleware, authenticate, + listVideoAccountChannelsValidator, paginationValidator, - videoChannelsSortValidator, - videoChannelsAddValidator, - setVideoChannelsSort, setPagination, - videoChannelsRemoveValidator, + setVideoChannelsSort, videoChannelGetValidator, - videoChannelsUpdateValidator, - listVideoAccountChannelsValidator, - asyncMiddleware + videoChannelsAddValidator, + videoChannelsRemoveValidator, + videoChannelsSortValidator, + videoChannelsUpdateValidator } from '../../../middlewares' -import { - createVideoChannel, - updateVideoChannelToFriends -} from '../../../lib' -import { VideoChannelInstance, AccountInstance } from '../../../models' -import { VideoChannelCreate, VideoChannelUpdate } from '../../../../shared' +import { AccountInstance, VideoChannelInstance } from '../../../models' +import { sendUpdateVideoChannel } from '../../../lib/activitypub/send-request' const videoChannelRouter = express.Router() @@ -137,11 +129,8 @@ async function updateVideoChannel (req: express.Request, res: express.Response) if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.set('description', videoChannelInfoToUpdate.description) await videoChannelInstance.save(sequelizeOptions) - const json = videoChannelInstance.toUpdateRemoteJSON() - - // Now we'll update the video channel's meta data to our friends - return updateVideoChannelToFriends(json, t) + await sendUpdateVideoChannel(videoChannelInstance, t) }) logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.uuid) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 9ad84609f..063839223 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -1,57 +1,41 @@ import * as express from 'express' import * as multer from 'multer' import { extname, join } from 'path' - -import { database as db } from '../../../initializers/database' +import { VideoCreate, VideoPrivacy, VideoUpdate } from '../../../../shared' import { - CONFIG, - REQUEST_VIDEO_QADU_TYPES, - REQUEST_VIDEO_EVENT_TYPES, - VIDEO_CATEGORIES, - VIDEO_LICENCES, - VIDEO_LANGUAGES, - VIDEO_PRIVACIES, - VIDEO_MIMETYPE_EXT -} from '../../../initializers' -import { - addEventToRemoteVideo, - quickAndDirtyUpdateVideoToFriends, - addVideoToFriends, - updateVideoToFriends, - JobScheduler, - fetchRemoteDescription -} from '../../../lib' + fetchRemoteVideoDescription, + generateRandomString, + getFormattedObjects, + getVideoFileHeight, + logger, + renamePromise, + resetSequelizeInstance, + retryTransactionWrapper +} from '../../../helpers' +import { getActivityPubUrl } from '../../../helpers/activitypub' +import { CONFIG, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_MIMETYPE_EXT, VIDEO_PRIVACIES } from '../../../initializers' +import { database as db } from '../../../initializers/database' +import { sendAddVideo, sendUpdateVideoChannel } from '../../../lib/activitypub/send-request' +import { transcodingJobScheduler } from '../../../lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler' import { + asyncMiddleware, authenticate, paginationValidator, - videosSortValidator, - setVideosSort, setPagination, setVideosSearch, - videosUpdateValidator, - videosSearchValidator, + setVideosSort, videosAddValidator, videosGetValidator, videosRemoveValidator, - asyncMiddleware + videosSearchValidator, + videosSortValidator, + videosUpdateValidator } from '../../../middlewares' -import { - logger, - retryTransactionWrapper, - generateRandomString, - getFormattedObjects, - renamePromise, - getVideoFileHeight, - resetSequelizeInstance -} from '../../../helpers' import { VideoInstance } from '../../../models' -import { VideoCreate, VideoUpdate, VideoPrivacy } from '../../../../shared' - import { abuseVideoRouter } from './abuse' import { blacklistRouter } from './blacklist' -import { rateVideoRouter } from './rate' import { videoChannelRouter } from './channel' -import { getActivityPubUrl } from '../../../helpers/activitypub' +import { rateVideoRouter } from './rate' const videosRouter = express.Router() @@ -225,7 +209,7 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi } tasks.push( - JobScheduler.Instance.createJob(t, 'videoFileOptimizer', dataInput) + transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput) ) } await Promise.all(tasks) @@ -252,9 +236,7 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi // Don't send video to remote pods, it is private if (video.privacy === VideoPrivacy.PRIVATE) return undefined - const remoteVideo = await video.toAddRemoteJSON() - // Now we'll add the video's meta data to our friends - return addVideoToFriends(remoteVideo, t) + await sendAddVideo(video, t) }) logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID) @@ -302,14 +284,12 @@ async function updateVideo (req: express.Request, res: express.Response) { // Now we'll update the video's meta data to our friends if (wasPrivateVideo === false) { - const json = videoInstance.toUpdateRemoteJSON() - return updateVideoToFriends(json, t) + await sendUpdateVideoChannel(videoInstance, t) } // Video is not private anymore, send a create action to remote pods if (wasPrivateVideo === true && videoInstance.privacy !== VideoPrivacy.PRIVATE) { - const remoteVideo = await videoInstance.toAddRemoteJSON() - return addVideoToFriends(remoteVideo, t) + await sendAddVideo(videoInstance, t) } }) @@ -324,7 +304,7 @@ async function updateVideo (req: express.Request, res: express.Response) { } } -function getVideo (req: express.Request, res: express.Response) { +async function getVideo (req: express.Request, res: express.Response) { const videoInstance = res.locals.video if (videoInstance.isOwned()) { @@ -333,21 +313,11 @@ function getVideo (req: express.Request, res: express.Response) { // For example, only add a view when a user watch a video during 30s etc videoInstance.increment('views') .then(() => { - const qaduParams = { - videoId: videoInstance.id, - type: REQUEST_VIDEO_QADU_TYPES.VIEWS - } - return quickAndDirtyUpdateVideoToFriends(qaduParams) + // TODO: send to followers a notification }) .catch(err => logger.error('Cannot add view to video %s.', videoInstance.uuid, err)) } else { - // Just send the event to our friends - const eventParams = { - videoId: videoInstance.id, - type: REQUEST_VIDEO_EVENT_TYPES.VIEWS - } - addEventToRemoteVideo(eventParams) - .catch(err => logger.error('Cannot add event to remote video %s.', videoInstance.uuid, err)) + // TODO: send view event to followers } // Do not wait the view system @@ -361,7 +331,7 @@ async function getVideoDescription (req: express.Request, res: express.Response) if (videoInstance.isOwned()) { description = videoInstance.description } else { - description = await fetchRemoteDescription(videoInstance) + description = await fetchRemoteVideoDescription(videoInstance) } return res.json({ description }) diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts index 727984506..955277d25 100644 --- a/server/controllers/api/videos/rate.ts +++ b/server/controllers/api/videos/rate.ts @@ -1,25 +1,11 @@ import * as express from 'express' - -import { database as db } from '../../../initializers/database' -import { - logger, - retryTransactionWrapper -} from '../../../helpers' -import { - VIDEO_RATE_TYPES, - REQUEST_VIDEO_EVENT_TYPES, - REQUEST_VIDEO_QADU_TYPES -} from '../../../initializers' -import { - addEventsToRemoteVideo, - quickAndDirtyUpdatesVideoToFriends -} from '../../../lib' -import { - authenticate, - videoRateValidator, - asyncMiddleware -} from '../../../middlewares' import { UserVideoRateUpdate } from '../../../../shared' +import { logger, retryTransactionWrapper } from '../../../helpers' +import { VIDEO_RATE_TYPES } from '../../../initializers' +import { database as db } from '../../../initializers/database' +import { asyncMiddleware, authenticate, videoRateValidator } from '../../../middlewares' +import { AccountInstance } from '../../../models/account/account-interface' +import { VideoInstance } from '../../../models/video/video-interface' const rateVideoRouter = express.Router() @@ -51,12 +37,12 @@ async function rateVideoRetryWrapper (req: express.Request, res: express.Respons async function rateVideo (req: express.Request, res: express.Response) { const body: UserVideoRateUpdate = req.body const rateType = body.rating - const videoInstance = res.locals.video - const userInstance = res.locals.oauth.token.User + const videoInstance: VideoInstance = res.locals.video + const accountInstance: AccountInstance = res.locals.oauth.token.User.Account await db.sequelize.transaction(async t => { const sequelizeOptions = { transaction: t } - const previousRate = await db.UserVideoRate.load(userInstance.id, videoInstance.id, t) + const previousRate = await db.AccountVideoRate.load(accountInstance.id, videoInstance.id, t) let likesToIncrement = 0 let dislikesToIncrement = 0 @@ -79,12 +65,12 @@ async function rateVideo (req: express.Request, res: express.Response) { } } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate const query = { - userId: userInstance.id, + accountId: accountInstance.id, videoId: videoInstance.id, type: rateType } - await db.UserVideoRate.create(query, sequelizeOptions) + await db.AccountVideoRate.create(query, sequelizeOptions) } const incrementQuery = { @@ -96,48 +82,12 @@ async function rateVideo (req: express.Request, res: express.Response) { // It is useful for the user to have a feedback await videoInstance.increment(incrementQuery, sequelizeOptions) - // Send a event to original pod if (videoInstance.isOwned() === false) { - - const eventsParams = [] - - if (likesToIncrement !== 0) { - eventsParams.push({ - videoId: videoInstance.id, - type: REQUEST_VIDEO_EVENT_TYPES.LIKES, - count: likesToIncrement - }) - } - - if (dislikesToIncrement !== 0) { - eventsParams.push({ - videoId: videoInstance.id, - type: REQUEST_VIDEO_EVENT_TYPES.DISLIKES, - count: dislikesToIncrement - }) - } - - await addEventsToRemoteVideo(eventsParams, t) - } else { // We own the video, we need to send a quick and dirty update to friends to notify the counts changed - const qadusParams = [] - - if (likesToIncrement !== 0) { - qadusParams.push({ - videoId: videoInstance.id, - type: REQUEST_VIDEO_QADU_TYPES.LIKES - }) - } - - if (dislikesToIncrement !== 0) { - qadusParams.push({ - videoId: videoInstance.id, - type: REQUEST_VIDEO_QADU_TYPES.DISLIKES - }) - } - - await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) + // TODO: Send a event to original pod + } else { + // TODO: Send update to followers } }) - logger.info('User video rate for video %s of user %s updated.', videoInstance.name, userInstance.username) + logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) } diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 75de2278c..a1493e5c1 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -1,15 +1,15 @@ +import { join } from 'path' +import * as request from 'request' import * as url from 'url' - -import { database as db } from '../initializers' -import { logger } from './logger' -import { doRequest, doRequestAndSaveToFile } from './requests' -import { isRemoteAccountValid } from './custom-validators' +import { ActivityIconObject } from '../../shared/index' import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' import { ResultList } from '../../shared/models/result-list.model' -import { CONFIG } from '../initializers/constants' +import { database as db, REMOTE_SCHEME } from '../initializers' +import { CONFIG, STATIC_PATHS } from '../initializers/constants' import { VideoInstance } from '../models/video/video-interface' -import { ActivityIconObject } from '../../shared/index' -import { join } from 'path' +import { isRemoteAccountValid } from './custom-validators' +import { logger } from './logger' +import { doRequest, doRequestAndSaveToFile } from './requests' function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) { const thumbnailName = video.getThumbnailName() @@ -22,9 +22,10 @@ function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObjec return doRequestAndSaveToFile(options, thumbnailPath) } -function getActivityPubUrl (type: 'video' | 'videoChannel', uuid: string) { - if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + uuid - else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + uuid +function getActivityPubUrl (type: 'video' | 'videoChannel' | 'account', id: string) { + if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + id + else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + id + else if (type === 'account') return CONFIG.WEBSERVER.URL + '/account/' + id return '' } @@ -94,7 +95,24 @@ async function fetchRemoteAccountAndCreatePod (accountUrl: string) { return { account, pod } } -function activityPubContextify (data: object) { +function fetchRemoteVideoPreview (video: VideoInstance) { + // FIXME: use url + const host = video.VideoChannel.Account.Pod.host + const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) + + return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) +} + +async function fetchRemoteVideoDescription (video: VideoInstance) { + const options = { + uri: video.url + } + + const { body } = await doRequest(options) + return body.description ? body.description : '' +} + +function activityPubContextify (data: T) { return Object.assign(data,{ '@context': [ 'https://www.w3.org/ns/activitystreams', @@ -141,7 +159,9 @@ export { activityPubCollectionPagination, getActivityPubUrl, generateThumbnailFromUrl, - getOrCreateAccount + getOrCreateAccount, + fetchRemoteVideoPreview, + fetchRemoteVideoDescription } // --------------------------------------------------------------------------- diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 31cedd768..4b1deeadc 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,16 +1,6 @@ -import * as replay from 'request-replay' -import * as request from 'request' import * as Promise from 'bluebird' - -import { - RETRY_REQUESTS, - REMOTE_SCHEME, - CONFIG -} from '../initializers' -import { PodInstance } from '../models' -import { PodSignature } from '../../shared' -import { signObject } from './peertube-crypto' import { createWriteStream } from 'fs' +import * as request from 'request' function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { @@ -27,78 +17,9 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U }) } -type MakeRetryRequestParams = { - url: string, - method: 'GET' | 'POST', - json: Object -} -function makeRetryRequest (params: MakeRetryRequestParams) { - return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { - replay( - request(params, (err, response, body) => err ? rej(err) : res({ response, body })), - { - retries: RETRY_REQUESTS, - factor: 3, - maxTimeout: Infinity, - errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] - } - ) - }) -} - -type MakeSecureRequestParams = { - toPod: PodInstance - path: string - data?: Object -} -function makeSecureRequest (params: MakeSecureRequestParams) { - const requestParams: { - method: 'POST', - uri: string, - json: { - signature: PodSignature, - data: any - } - } = { - method: 'POST', - uri: REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path, - json: { - signature: null, - data: null - } - } - - const host = CONFIG.WEBSERVER.HOST - - let dataToSign - if (params.data) { - dataToSign = params.data - } else { - // We do not have data to sign so we just take our host - // It is not ideal but the connection should be in HTTPS - dataToSign = host - } - - sign(dataToSign).then(signature => { - requestParams.json.signature = { - host, // Which host we pretend to be - signature - } - - // If there are data information - if (params.data) { - requestParams.json.data = params.data - } - - return doRequest(requestParams) - }) -} - // --------------------------------------------------------------------------- export { doRequest, - doRequestAndSaveToFile, - makeRetryRequest, - makeSecureRequest + doRequestAndSaveToFile } diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts index 9586fa562..164ae4951 100644 --- a/server/helpers/webfinger.ts +++ b/server/helpers/webfinger.ts @@ -35,7 +35,7 @@ export { function webfingerLookup (url: string) { return new Promise((res, rej) => { - webfinger.lookup('nick@silverbucket.net', (err, p) => { + webfinger.lookup(url, (err, p) => { if (err) return rej(err) return p diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e6fda88c2..2d61094bd 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -361,12 +361,6 @@ export { PODS_SCORE, PREVIEWS_SIZE, REMOTE_SCHEME, - REQUEST_ENDPOINT_ACTIONS, - REQUEST_ENDPOINTS, - REQUEST_VIDEO_EVENT_ENDPOINT, - REQUEST_VIDEO_EVENT_TYPES, - REQUEST_VIDEO_QADU_ENDPOINT, - REQUEST_VIDEO_QADU_TYPES, REQUESTS_IN_PARALLEL, REQUESTS_INTERVAL, REQUESTS_LIMIT_PER_POD, diff --git a/server/initializers/database.ts b/server/initializers/database.ts index aefb6da3a..1383bb33b 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -2,7 +2,6 @@ import { join } from 'path' import { flattenDepth } from 'lodash' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string import * as Sequelize from 'sequelize' -import * as Bluebird from 'bluebird' import { CONFIG } from './constants' // Do not use barrel, we need to load database first @@ -19,10 +18,6 @@ import { UserModel } from '../models/account/user-interface' import { AccountVideoRateModel } from '../models/account/account-video-rate-interface' import { AccountFollowModel } from '../models/account/account-follow-interface' import { TagModel } from './../models/video/tag-interface' -import { RequestModel } from './../models/request/request-interface' -import { RequestVideoQaduModel } from './../models/request/request-video-qadu-interface' -import { RequestVideoEventModel } from './../models/request/request-video-event-interface' -import { RequestToPodModel } from './../models/request/request-to-pod-interface' import { PodModel } from './../models/pod/pod-interface' import { OAuthTokenModel } from './../models/oauth/oauth-token-interface' import { OAuthClientModel } from './../models/oauth/oauth-client-interface' diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index c8f6b3bc2..c617b16c9 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -1,20 +1,21 @@ import * as passwordGenerator from 'password-generator' -import * as Bluebird from 'bluebird' +import { UserRole } from '../../shared' +import { logger, mkdirpPromise, rimrafPromise } from '../helpers' +import { createPrivateAndPublicKeys } from '../helpers/peertube-crypto' +import { createUserAccountAndChannel } from '../lib' +import { clientsExist, usersExist } from './checker' +import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' import { database as db } from './database' -import { CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants' -import { clientsExist, usersExist } from './checker' -import { logger, createCertsIfNotExist, mkdirpPromise, rimrafPromise } from '../helpers' -import { createUserAccountAndChannel } from '../lib' -import { UserRole } from '../../shared' +import { createLocalAccount } from '../lib/user' async function installApplication () { await db.sequelize.sync() await removeCacheDirectories() await createDirectoriesIfNotExist() - await createCertsIfNotExist() await createOAuthClientIfNotExist() await createOAuthAdminIfNotExist() + await createApplicationIfNotExist() } // --------------------------------------------------------------------------- @@ -28,7 +29,7 @@ export { function removeCacheDirectories () { const cacheDirectories = CACHE.DIRECTORIES - const tasks: Bluebird[] = [] + const tasks: Promise[] = [] // Cache directories for (const key of Object.keys(cacheDirectories)) { @@ -120,7 +121,12 @@ async function createOAuthAdminIfNotExist () { await createUserAccountAndChannel(user, validatePassword) logger.info('Username: ' + username) logger.info('User password: ' + password) +} +async function createApplicationIfNotExist () { logger.info('Creating Application table.') - await db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION }) + const applicationInstance = await db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION }) + + logger.info('Creating application account.') + return createLocalAccount('peertube', null, applicationInstance.id, undefined) } diff --git a/server/initializers/migrations/0075-video-resolutions.ts b/server/initializers/migrations/0075-video-resolutions.ts index e1d9fdacb..e7d8a2876 100644 --- a/server/initializers/migrations/0075-video-resolutions.ts +++ b/server/initializers/migrations/0075-video-resolutions.ts @@ -1,5 +1,4 @@ import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' import { join } from 'path' import { readdirPromise, renamePromise } from '../../helpers/core-utils' diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts index 740800606..f8d56528a 100644 --- a/server/lib/activitypub/index.ts +++ b/server/lib/activitypub/index.ts @@ -1,3 +1,4 @@ export * from './process-create' export * from './process-flag' export * from './process-update' +export * from './send-request' diff --git a/server/lib/activitypub/misc.ts b/server/lib/activitypub/misc.ts index 05e77ebc3..2cf0c4fd1 100644 --- a/server/lib/activitypub/misc.ts +++ b/server/lib/activitypub/misc.ts @@ -8,7 +8,11 @@ import { VideoChannelInstance } from '../../models/video/video-channel-interface import { VideoFileAttributes } from '../../models/video/video-file-interface' import { VideoAttributes, VideoInstance } from '../../models/video/video-interface' -async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelInstance, videoObject: VideoTorrentObject, t: Sequelize.Transaction) { +async function videoActivityObjectToDBAttributes ( + videoChannel: VideoChannelInstance, + videoObject: VideoTorrentObject, + t: Sequelize.Transaction +) { const videoFromDatabase = await db.Video.loadByUUIDOrURL(videoObject.uuid, videoObject.id, t) if (videoFromDatabase) throw new Error('Video with this UUID/Url already exists.') diff --git a/server/lib/activitypub/process-flag.ts b/server/lib/activitypub/process-flag.ts index 6fa862ee9..b562dce4d 100644 --- a/server/lib/activitypub/process-flag.ts +++ b/server/lib/activitypub/process-flag.ts @@ -5,7 +5,7 @@ import { } from '../../../shared' function processFlagActivity (activity: ActivityCreate) { - // empty + return Promise.resolve(undefined) } // --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/send-request.ts b/server/lib/activitypub/send-request.ts index 6a31c226d..91101f5ad 100644 --- a/server/lib/activitypub/send-request.ts +++ b/server/lib/activitypub/send-request.ts @@ -1,5 +1,6 @@ import * as Sequelize from 'sequelize' +import { database as db } from '../../initializers' import { AccountInstance, VideoInstance, @@ -13,54 +14,66 @@ function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Sequeliz const videoChannelObject = videoChannel.toActivityPubObject() const data = createActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) - return broadcastToFollowers(data, t) + return broadcastToFollowers(data, videoChannel.Account, t) } function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { const videoChannelObject = videoChannel.toActivityPubObject() const data = updateActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) - return broadcastToFollowers(data, t) + return broadcastToFollowers(data, videoChannel.Account, t) } function sendDeleteVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { const videoChannelObject = videoChannel.toActivityPubObject() const data = deleteActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) - return broadcastToFollowers(data, t) + return broadcastToFollowers(data, videoChannel.Account, t) } function sendAddVideo (video: VideoInstance, t: Sequelize.Transaction) { const videoObject = video.toActivityPubObject() const data = addActivityData(video.url, video.VideoChannel.Account, video.VideoChannel.url, videoObject) - return broadcastToFollowers(data, t) + return broadcastToFollowers(data, video.VideoChannel.Account, t) } function sendUpdateVideo (video: VideoInstance, t: Sequelize.Transaction) { const videoObject = video.toActivityPubObject() const data = updateActivityData(video.url, video.VideoChannel.Account, videoObject) - return broadcastToFollowers(data, t) + return broadcastToFollowers(data, video.VideoChannel.Account, t) } function sendDeleteVideo (video: VideoInstance, t: Sequelize.Transaction) { const videoObject = video.toActivityPubObject() const data = deleteActivityData(video.url, video.VideoChannel.Account, videoObject) - return broadcastToFollowers(data, t) + return broadcastToFollowers(data, video.VideoChannel.Account, t) } // --------------------------------------------------------------------------- export { - + sendCreateVideoChannel, + sendUpdateVideoChannel, + sendDeleteVideoChannel, + sendAddVideo, + sendUpdateVideo, + sendDeleteVideo } // --------------------------------------------------------------------------- -function broadcastToFollowers (data: any, t: Sequelize.Transaction) { - return httpRequestJobScheduler.createJob(t, 'http-request', 'httpRequestBroadcastHandler', data) +async function broadcastToFollowers (data: any, fromAccount: AccountInstance, t: Sequelize.Transaction) { + const result = await db.Account.listFollowerUrlsForApi(fromAccount.name, 0) + + const jobPayload = { + uris: result.data, + body: data + } + + return httpRequestJobScheduler.createJob(t, 'httpRequestBroadcastHandler', jobPayload) } function buildSignedActivity (byAccount: AccountInstance, data: Object) { diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts index 791ad1cbf..776f647a0 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/cache/videos-preview-cache.ts @@ -3,9 +3,8 @@ import { join } from 'path' import { createWriteStream } from 'fs' import { database as db, CONFIG, CACHE } from '../../initializers' -import { logger, unlinkPromise } from '../../helpers' +import { logger, unlinkPromise, fetchRemoteVideoPreview } from '../../helpers' import { VideoInstance } from '../../models' -import { fetchRemotePreview } from '../../lib' class VideosPreviewCache { @@ -54,7 +53,7 @@ class VideosPreviewCache { } private saveRemotePreviewAndReturnPath (video: VideoInstance) { - const req = fetchRemotePreview(video) + const req = fetchRemoteVideoPreview(video) return new Promise((res, rej) => { const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName()) diff --git a/server/lib/index.ts b/server/lib/index.ts index bfb415ad2..d22ecb665 100644 --- a/server/lib/index.ts +++ b/server/lib/index.ts @@ -1,8 +1,6 @@ export * from './activitypub' export * from './cache' export * from './jobs' -export * from './request' -export * from './friends' export * from './oauth-model' export * from './user' export * from './video-channel' diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts index 6b6946d02..799b86e1c 100644 --- a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts +++ b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts @@ -1,19 +1,28 @@ -import * as Bluebird from 'bluebird' - -import { database as db } from '../../../initializers/database' import { logger } from '../../../helpers' +import { doRequest } from '../../../helpers/requests' +import { HTTPRequestPayload } from './http-request-job-scheduler' + +async function process (payload: HTTPRequestPayload, jobId: number) { + logger.info('Processing broadcast in job %d.', jobId) -async function process (data: { videoUUID: string }, jobId: number) { + const options = { + uri: '', + json: payload.body + } + for (const uri of payload.uris) { + options.uri = uri + await doRequest(options) + } } function onError (err: Error, jobId: number) { - logger.error('Error when optimized video file in job %d.', jobId, err) + logger.error('Error when broadcasting request in job %d.', jobId, err) return Promise.resolve() } async function onSuccess (jobId: number) { - + logger.info('Job %d is a success.', jobId) } // --------------------------------------------------------------------------- diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts index 42cb9139c..ad3349866 100644 --- a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts +++ b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts @@ -4,7 +4,11 @@ import * as httpRequestBroadcastHandler from './http-request-broadcast-handler' import * as httpRequestUnicastHandler from './http-request-unicast-handler' import { JobCategory } from '../../../../shared' -const jobHandlers: { [ handlerName: string ]: JobHandler } = { +type HTTPRequestPayload = { + uris: string[] + body: any +} +const jobHandlers: { [ handlerName: string ]: JobHandler } = { httpRequestBroadcastHandler, httpRequestUnicastHandler } @@ -13,5 +17,6 @@ const jobCategory: JobCategory = 'http-request' const httpRequestJobScheduler = new JobScheduler(jobCategory, jobHandlers) export { + HTTPRequestPayload, httpRequestJobScheduler } diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts index 6b6946d02..13451f042 100644 --- a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts +++ b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts @@ -1,19 +1,26 @@ -import * as Bluebird from 'bluebird' - -import { database as db } from '../../../initializers/database' import { logger } from '../../../helpers' +import { doRequest } from '../../../helpers/requests' +import { HTTPRequestPayload } from './http-request-job-scheduler' + +async function process (payload: HTTPRequestPayload, jobId: number) { + logger.info('Processing unicast in job %d.', jobId) -async function process (data: { videoUUID: string }, jobId: number) { + const uri = payload.uris[0] + const options = { + uri, + json: payload.body + } + await doRequest(options) } function onError (err: Error, jobId: number) { - logger.error('Error when optimized video file in job %d.', jobId, err) + logger.error('Error when sending request in job %d.', jobId, err) return Promise.resolve() } async function onSuccess (jobId: number) { - + logger.info('Job %d is a success.', jobId) } // --------------------------------------------------------------------------- diff --git a/server/lib/jobs/job-scheduler.ts b/server/lib/jobs/job-scheduler.ts index 89a4bca88..f10f745b3 100644 --- a/server/lib/jobs/job-scheduler.ts +++ b/server/lib/jobs/job-scheduler.ts @@ -1,28 +1,22 @@ import { AsyncQueue, forever, queue } from 'async' import * as Sequelize from 'sequelize' - -import { - database as db, - JOBS_FETCHING_INTERVAL, - JOBS_FETCH_LIMIT_PER_CYCLE, - JOB_STATES -} from '../../initializers' +import { JobCategory } from '../../../shared' import { logger } from '../../helpers' +import { database as db, JOB_STATES, JOBS_FETCH_LIMIT_PER_CYCLE, JOBS_FETCHING_INTERVAL } from '../../initializers' import { JobInstance } from '../../models' -import { JobCategory } from '../../../shared' -export interface JobHandler { - process (data: object, jobId: number): T +export interface JobHandler { + process (data: object, jobId: number): Promise onError (err: Error, jobId: number) - onSuccess (jobId: number, jobResult: T) + onSuccess (jobId: number, jobResult: T, jobScheduler: JobScheduler) } type JobQueueCallback = (err: Error) => void -class JobScheduler { +class JobScheduler { constructor ( private jobCategory: JobCategory, - private jobHandlers: { [ id: string ]: JobHandler } + private jobHandlers: { [ id: string ]: JobHandler } ) {} async activate () { @@ -66,13 +60,14 @@ class JobScheduler { ) } - createJob (transaction: Sequelize.Transaction, category: JobCategory, handlerName: string, handlerInputData: object) { + createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: P) { const createQuery = { state: JOB_STATES.PENDING, - category, + category: this.jobCategory, handlerName, handlerInputData } + const options = { transaction } return db.Job.create(createQuery, options) @@ -95,7 +90,7 @@ class JobScheduler { await job.save() try { - const result = await jobHandler.process(job.handlerInputData, job.id) + const result: T = await jobHandler.process(job.handlerInputData, job.id) await this.onJobSuccess(jobHandler, job, result) } catch (err) { logger.error('Error in job handler %s.', job.handlerName, err) @@ -111,7 +106,7 @@ class JobScheduler { callback(null) } - private async onJobError (jobHandler: JobHandler, job: JobInstance, err: Error) { + private async onJobError (jobHandler: JobHandler, job: JobInstance, err: Error) { job.state = JOB_STATES.ERROR try { @@ -122,12 +117,12 @@ class JobScheduler { } } - private async onJobSuccess (jobHandler: JobHandler, job: JobInstance, jobResult: any) { + private async onJobSuccess (jobHandler: JobHandler, job: JobInstance, jobResult: T) { job.state = JOB_STATES.SUCCESS try { await job.save() - jobHandler.onSuccess(job.id, jobResult) + jobHandler.onSuccess(job.id, jobResult, this) } catch (err) { this.cannotSaveJobError(err) } diff --git a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts index d7c614fb8..c5efe8eeb 100644 --- a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts @@ -1,10 +1,14 @@ -import { JobScheduler, JobHandler } from '../job-scheduler' - +import { JobCategory } from '../../../../shared' +import { JobHandler, JobScheduler } from '../job-scheduler' import * as videoFileOptimizer from './video-file-optimizer-handler' import * as videoFileTranscoder from './video-file-transcoder-handler' -import { JobCategory } from '../../../../shared' +import { VideoInstance } from '../../../models/video/video-interface' -const jobHandlers: { [ handlerName: string ]: JobHandler } = { +type TranscodingJobPayload = { + videoUUID: string + resolution?: number +} +const jobHandlers: { [ handlerName: string ]: JobHandler } = { videoFileOptimizer, videoFileTranscoder } @@ -13,5 +17,6 @@ const jobCategory: JobCategory = 'transcoding' const transcodingJobScheduler = new JobScheduler(jobCategory, jobHandlers) export { + TranscodingJobPayload, transcodingJobScheduler } diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts index f019c28bc..47603a66c 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts @@ -1,12 +1,13 @@ import * as Bluebird from 'bluebird' +import { computeResolutionsToTranscode, logger } from '../../../helpers' import { database as db } from '../../../initializers/database' -import { logger, computeResolutionsToTranscode } from '../../../helpers' import { VideoInstance } from '../../../models' -import { addVideoToFriends } from '../../friends' +import { sendAddVideo } from '../../activitypub/send-request' import { JobScheduler } from '../job-scheduler' +import { TranscodingJobPayload } from './transcoding-job-scheduler' -async function process (data: { videoUUID: string }, jobId: number) { +async function process (data: TranscodingJobPayload, jobId: number) { const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) // No video, maybe deleted? if (!video) { @@ -24,7 +25,7 @@ function onError (err: Error, jobId: number) { return Promise.resolve() } -async function onSuccess (jobId: number, video: VideoInstance) { +async function onSuccess (jobId: number, video: VideoInstance, jobScheduler: JobScheduler) { if (video === undefined) return undefined logger.info('Job %d is a success.', jobId) @@ -34,10 +35,8 @@ async function onSuccess (jobId: number, video: VideoInstance) { // Video does not exist anymore if (!videoDatabase) return undefined - const remoteVideo = await videoDatabase.toAddRemoteJSON() - - // Now we'll add the video's meta data to our friends - await addVideoToFriends(remoteVideo, null) + // Now we'll add the video's meta data to our followers + await sendAddVideo(video, undefined) const originalFileHeight = await videoDatabase.getOriginalFileHeight() // Create transcoding jobs if there are enabled resolutions @@ -59,7 +58,7 @@ async function onSuccess (jobId: number, video: VideoInstance) { resolution } - const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput) + const p = jobScheduler.createJob(t, 'videoFileTranscoder', dataInput) tasks.push(p) } diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts index 397b95795..77e5d9f7f 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts @@ -1,8 +1,8 @@ -import { database as db } from '../../../initializers/database' -import { updateVideoToFriends } from '../../friends' +import { VideoResolution } from '../../../../shared' import { logger } from '../../../helpers' +import { database as db } from '../../../initializers/database' import { VideoInstance } from '../../../models' -import { VideoResolution } from '../../../../shared' +import { sendUpdateVideo } from '../../activitypub/send-request' async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) @@ -32,10 +32,7 @@ async function onSuccess (jobId: number, video: VideoInstance) { // Video does not exist anymore if (!videoDatabase) return undefined - const remoteVideo = videoDatabase.toUpdateRemoteJSON() - - // Now we'll add the video's meta data to our friends - await updateVideoToFriends(remoteVideo, null) + await sendUpdateVideo(video, undefined) return undefined } diff --git a/server/lib/request/abstract-request-scheduler.ts b/server/lib/request/abstract-request-scheduler.ts deleted file mode 100644 index f838c47f2..000000000 --- a/server/lib/request/abstract-request-scheduler.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { isEmpty } from 'lodash' -import * as Bluebird from 'bluebird' - -import { database as db } from '../../initializers/database' -import { logger, makeSecureRequest } from '../../helpers' -import { AbstractRequestClass, AbstractRequestToPodClass, PodInstance } from '../../models' -import { - API_VERSION, - REQUESTS_IN_PARALLEL, - REQUESTS_INTERVAL -} from '../../initializers' - -interface RequestsObjects { - [ id: string ]: { - toPod: PodInstance - endpoint: string - ids: number[] // ids - datas: U[] - } -} - -abstract class AbstractRequestScheduler { - requestInterval: number - limitPods: number - limitPerPod: number - - protected lastRequestTimestamp: number - protected timer: NodeJS.Timer - protected description: string - - constructor () { - this.lastRequestTimestamp = 0 - this.timer = null - this.requestInterval = REQUESTS_INTERVAL - } - - abstract getRequestModel (): AbstractRequestClass - abstract getRequestToPodModel (): AbstractRequestToPodClass - abstract buildRequestsObjects (requestsGrouped: T): RequestsObjects - - activate () { - logger.info('Requests scheduler activated.') - this.lastRequestTimestamp = Date.now() - - this.timer = setInterval(() => { - this.lastRequestTimestamp = Date.now() - this.makeRequests() - }, this.requestInterval) - } - - deactivate () { - logger.info('Requests scheduler deactivated.') - clearInterval(this.timer) - this.timer = null - } - - forceSend () { - logger.info('Force requests scheduler sending.') - this.makeRequests() - } - - remainingMilliSeconds () { - if (this.timer === null) return -1 - - return REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) - } - - remainingRequestsCount () { - return this.getRequestModel().countTotalRequests() - } - - flush () { - return this.getRequestModel().removeAll() - } - - // --------------------------------------------------------------------------- - - // Make a requests to friends of a certain type - protected async makeRequest (toPod: PodInstance, requestEndpoint: string, requestsToMake: any) { - const params = { - toPod: toPod, - method: 'POST' as 'POST', - path: '/api/' + API_VERSION + '/remote/' + requestEndpoint, - data: requestsToMake // Requests we need to make - } - - // Make multiple retry requests to all of pods - // The function fire some useful callbacks - try { - const { response } = await makeSecureRequest(params) - - // 400 because if the other pod is not up to date, it may not understand our request - if ([ 200, 201, 204, 400 ].indexOf(response.statusCode) === -1) { - throw new Error('Status code not 20x or 400 : ' + response.statusCode) - } - } catch (err) { - logger.error('Error sending secure request to %s pod.', toPod.host, err) - - throw err - } - } - - // Make all the requests of the scheduler - protected async makeRequests () { - let requestsGrouped: T - - try { - requestsGrouped = await this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod) - } catch (err) { - logger.error('Cannot get the list of "%s".', this.description, { error: err.stack }) - throw err - } - - // We want to group requests by destinations pod and endpoint - const requestsToMake = this.buildRequestsObjects(requestsGrouped) - - // If there are no requests, abort - if (isEmpty(requestsToMake) === true) { - logger.info('No "%s" to make.', this.description) - return { goodPods: [], badPods: [] } - } - - logger.info('Making "%s" to friends.', this.description) - - const goodPods: number[] = [] - const badPods: number[] = [] - - await Bluebird.map(Object.keys(requestsToMake), async hashKey => { - const requestToMake = requestsToMake[hashKey] - const toPod: PodInstance = requestToMake.toPod - - try { - await this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas) - logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) - goodPods.push(requestToMake.toPod.id) - - this.afterRequestHook() - - // Remove the pod id of these request ids - await this.getRequestToPodModel() - .removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id) - } catch (err) { - badPods.push(requestToMake.toPod.id) - logger.info('Cannot make request to %s.', toPod.host, err) - } - }, { concurrency: REQUESTS_IN_PARALLEL }) - - this.afterRequestsHook() - - // All the requests were made, we update the pods score - db.Pod.updatePodsScore(goodPods, badPods) - } - - protected afterRequestHook () { - // Nothing to do, let children re-implement it - } - - protected afterRequestsHook () { - // Nothing to do, let children re-implement it - } -} - -// --------------------------------------------------------------------------- - -export { - AbstractRequestScheduler, - RequestsObjects -} diff --git a/server/lib/request/index.ts b/server/lib/request/index.ts deleted file mode 100644 index 47d60e5b4..000000000 --- a/server/lib/request/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './abstract-request-scheduler' -export * from './request-scheduler' -export * from './request-video-event-scheduler' -export * from './request-video-qadu-scheduler' diff --git a/server/lib/request/request-scheduler.ts b/server/lib/request/request-scheduler.ts deleted file mode 100644 index c3f7f6429..000000000 --- a/server/lib/request/request-scheduler.ts +++ /dev/null @@ -1,96 +0,0 @@ -import * as Sequelize from 'sequelize' - -import { database as db } from '../../initializers/database' -import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler' -import { logger } from '../../helpers' -import { REQUESTS_LIMIT_PODS, REQUESTS_LIMIT_PER_POD } from '../../initializers' -import { RequestsGrouped } from '../../models' -import { RequestEndpoint, RemoteVideoRequest } from '../../../shared' - -export type RequestSchedulerOptions = { - type: string - endpoint: RequestEndpoint - data: Object - toIds: number[] - transaction: Sequelize.Transaction -} - -class RequestScheduler extends AbstractRequestScheduler { - constructor () { - super() - - // We limit the size of the requests - this.limitPods = REQUESTS_LIMIT_PODS - this.limitPerPod = REQUESTS_LIMIT_PER_POD - - this.description = 'requests' - } - - getRequestModel () { - return db.Request - } - - getRequestToPodModel () { - return db.RequestToPod - } - - buildRequestsObjects (requestsGrouped: RequestsGrouped) { - const requestsToMakeGrouped: RequestsObjects = {} - - for (const toPodId of Object.keys(requestsGrouped)) { - for (const data of requestsGrouped[toPodId]) { - const request = data.request - const pod = data.pod - const hashKey = toPodId + request.endpoint - - if (!requestsToMakeGrouped[hashKey]) { - requestsToMakeGrouped[hashKey] = { - toPod: pod, - endpoint: request.endpoint, - ids: [], // request ids, to delete them from the DB in the future - datas: [] // requests data, - } - } - - requestsToMakeGrouped[hashKey].ids.push(request.id) - requestsToMakeGrouped[hashKey].datas.push(request.request) - } - } - - return requestsToMakeGrouped - } - - async createRequest ({ type, endpoint, data, toIds, transaction }: RequestSchedulerOptions) { - // If there are no destination pods abort - if (toIds.length === 0) return undefined - - const createQuery = { - endpoint, - request: { - type: type, - data: data - } - } - - const dbRequestOptions: Sequelize.CreateOptions = { - transaction - } - - const request = await db.Request.create(createQuery, dbRequestOptions) - await request.setPods(toIds, dbRequestOptions) - } - - // --------------------------------------------------------------------------- - - afterRequestsHook () { - // Flush requests with no pod - this.getRequestModel().removeWithEmptyTo() - .catch(err => logger.error('Error when removing requests with no pods.', err)) - } -} - -// --------------------------------------------------------------------------- - -export { - RequestScheduler -} diff --git a/server/lib/request/request-video-event-scheduler.ts b/server/lib/request/request-video-event-scheduler.ts deleted file mode 100644 index 5f21287f0..000000000 --- a/server/lib/request/request-video-event-scheduler.ts +++ /dev/null @@ -1,129 +0,0 @@ -import * as Sequelize from 'sequelize' - -import { database as db } from '../../initializers/database' -import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler' -import { - REQUESTS_VIDEO_EVENT_LIMIT_PODS, - REQUESTS_VIDEO_EVENT_LIMIT_PER_POD, - REQUEST_VIDEO_EVENT_ENDPOINT -} from '../../initializers' -import { RequestsVideoEventGrouped } from '../../models' -import { RequestVideoEventType, RemoteVideoEventRequest, RemoteVideoEventType } from '../../../shared' - -export type RequestVideoEventSchedulerOptions = { - type: RequestVideoEventType - videoId: number - count?: number - transaction?: Sequelize.Transaction -} - -class RequestVideoEventScheduler extends AbstractRequestScheduler { - constructor () { - super() - - // We limit the size of the requests - this.limitPods = REQUESTS_VIDEO_EVENT_LIMIT_PODS - this.limitPerPod = REQUESTS_VIDEO_EVENT_LIMIT_PER_POD - - this.description = 'video event requests' - } - - getRequestModel () { - return db.RequestVideoEvent - } - - getRequestToPodModel () { - return db.RequestVideoEvent - } - - buildRequestsObjects (eventRequests: RequestsVideoEventGrouped) { - const requestsToMakeGrouped: RequestsObjects = {} - - /* Example: - { - pod1: { - video1: { views: 4, likes: 5 }, - video2: { likes: 5 } - } - } - */ - const eventsPerVideoPerPod: { - [ podId: string ]: { - [ videoUUID: string ]: { - views?: number - likes?: number - dislikes?: number - } - } - } = {} - - // We group video events per video and per pod - // We add the counts of the same event types - for (const toPodId of Object.keys(eventRequests)) { - for (const eventToProcess of eventRequests[toPodId]) { - if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {} - - if (!requestsToMakeGrouped[toPodId]) { - requestsToMakeGrouped[toPodId] = { - toPod: eventToProcess.pod, - endpoint: REQUEST_VIDEO_EVENT_ENDPOINT, - ids: [], // request ids, to delete them from the DB in the future - datas: [] // requests data - } - } - requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id) - - const eventsPerVideo = eventsPerVideoPerPod[toPodId] - const uuid = eventToProcess.video.uuid - if (!eventsPerVideo[uuid]) eventsPerVideo[uuid] = {} - - const events = eventsPerVideo[uuid] - if (!events[eventToProcess.type]) events[eventToProcess.type] = 0 - - events[eventToProcess.type] += eventToProcess.count - } - } - - // Now we build our requests array per pod - for (const toPodId of Object.keys(eventsPerVideoPerPod)) { - const eventsForPod = eventsPerVideoPerPod[toPodId] - - for (const uuid of Object.keys(eventsForPod)) { - const eventsForVideo = eventsForPod[uuid] - - for (const eventType of Object.keys(eventsForVideo)) { - requestsToMakeGrouped[toPodId].datas.push({ - data: { - uuid, - eventType: eventType as RemoteVideoEventType, - count: +eventsForVideo[eventType] - } - }) - } - } - } - - return requestsToMakeGrouped - } - - createRequest ({ type, videoId, count, transaction }: RequestVideoEventSchedulerOptions) { - if (count === undefined) count = 1 - - const dbRequestOptions: Sequelize.CreateOptions = {} - if (transaction) dbRequestOptions.transaction = transaction - - const createQuery = { - type, - count, - videoId - } - - return db.RequestVideoEvent.create(createQuery, dbRequestOptions) - } -} - -// --------------------------------------------------------------------------- - -export { - RequestVideoEventScheduler -} diff --git a/server/lib/request/request-video-qadu-scheduler.ts b/server/lib/request/request-video-qadu-scheduler.ts deleted file mode 100644 index 24ee59d29..000000000 --- a/server/lib/request/request-video-qadu-scheduler.ts +++ /dev/null @@ -1,148 +0,0 @@ -import * as Sequelize from 'sequelize' - -import { database as db } from '../../initializers/database' -import { AbstractRequestScheduler, RequestsObjects } from './abstract-request-scheduler' -import { logger } from '../../helpers' -import { - REQUESTS_VIDEO_QADU_LIMIT_PODS, - REQUESTS_VIDEO_QADU_LIMIT_PER_POD, - REQUEST_VIDEO_QADU_ENDPOINT, - REQUEST_VIDEO_QADU_TYPES -} from '../../initializers' -import { RequestsVideoQaduGrouped, PodInstance } from '../../models' -import { RemoteQaduVideoRequest, RequestVideoQaduType } from '../../../shared' - -// We create a custom interface because we need "videos" attribute for our computations -interface RequestsObjectsCustom extends RequestsObjects { - [ id: string ]: { - toPod: PodInstance - endpoint: string - ids: number[] // ids - datas: U[] - - videos: { - [ uuid: string ]: { - uuid: string - likes?: number - dislikes?: number - views?: number - } - } - } -} - -export type RequestVideoQaduSchedulerOptions = { - type: RequestVideoQaduType - videoId: number - transaction?: Sequelize.Transaction -} - -class RequestVideoQaduScheduler extends AbstractRequestScheduler { - constructor () { - super() - - // We limit the size of the requests - this.limitPods = REQUESTS_VIDEO_QADU_LIMIT_PODS - this.limitPerPod = REQUESTS_VIDEO_QADU_LIMIT_PER_POD - - this.description = 'video QADU requests' - } - - getRequestModel () { - return db.RequestVideoQadu - } - - getRequestToPodModel () { - return db.RequestVideoQadu - } - - buildRequestsObjects (requests: RequestsVideoQaduGrouped) { - const requestsToMakeGrouped: RequestsObjectsCustom = {} - - for (const toPodId of Object.keys(requests)) { - for (const data of requests[toPodId]) { - const request = data.request - const video = data.video - const pod = data.pod - const hashKey = toPodId - - if (!requestsToMakeGrouped[hashKey]) { - requestsToMakeGrouped[hashKey] = { - toPod: pod, - endpoint: REQUEST_VIDEO_QADU_ENDPOINT, - ids: [], // request ids, to delete them from the DB in the future - datas: [], // requests data - videos: {} - } - } - - // Maybe another attribute was filled for this video - let videoData = requestsToMakeGrouped[hashKey].videos[video.id] - if (!videoData) videoData = { uuid: null } - - switch (request.type) { - case REQUEST_VIDEO_QADU_TYPES.LIKES: - videoData.likes = video.likes - break - - case REQUEST_VIDEO_QADU_TYPES.DISLIKES: - videoData.dislikes = video.dislikes - break - - case REQUEST_VIDEO_QADU_TYPES.VIEWS: - videoData.views = video.views - break - - default: - logger.error('Unknown request video QADU type %s.', request.type) - return undefined - } - - // Do not forget the uuid so the remote pod can identify the video - videoData.uuid = video.uuid - requestsToMakeGrouped[hashKey].ids.push(request.id) - - // Maybe there are multiple quick and dirty update for the same video - // We use this hash map to dedupe them - requestsToMakeGrouped[hashKey].videos[video.id] = videoData - } - } - - // Now we deduped similar quick and dirty updates, we can build our requests data - for (const hashKey of Object.keys(requestsToMakeGrouped)) { - for (const videoUUID of Object.keys(requestsToMakeGrouped[hashKey].videos)) { - const videoData = requestsToMakeGrouped[hashKey].videos[videoUUID] - - requestsToMakeGrouped[hashKey].datas.push({ - data: videoData - }) - } - - // We don't need it anymore, it was just to build our data array - delete requestsToMakeGrouped[hashKey].videos - } - - return requestsToMakeGrouped - } - - async createRequest ({ type, videoId, transaction }: RequestVideoQaduSchedulerOptions) { - const dbRequestOptions: Sequelize.BulkCreateOptions = {} - if (transaction) dbRequestOptions.transaction = transaction - - // Send the update to all our friends - const podIds = await db.Pod.listAllIds(transaction) - const queries = [] - for (const podId of podIds) { - queries.push({ type, videoId, podId }) - } - - await db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions) - return undefined - } -} - -// --------------------------------------------------------------------------- - -export { - RequestVideoQaduScheduler -} diff --git a/server/lib/user.ts b/server/lib/user.ts index 57c653e55..1094c2401 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -1,6 +1,9 @@ +import * as Sequelize from 'sequelize' +import { getActivityPubUrl } from '../helpers/activitypub' +import { createPrivateAndPublicKeys } from '../helpers/peertube-crypto' import { database as db } from '../initializers' +import { CONFIG } from '../initializers/constants' import { UserInstance } from '../models' -import { addVideoAccountToFriends } from './friends' import { createVideoChannel } from './video-channel' async function createUserAccountAndChannel (user: UserInstance, validateUser = true) { @@ -11,32 +14,46 @@ async function createUserAccountAndChannel (user: UserInstance, validateUser = t } const userCreated = await user.save(userOptions) - const accountInstance = db.Account.build({ - name: userCreated.username, - podId: null, // It is our pod - userId: userCreated.id - }) - - const accountCreated = await accountInstance.save({ transaction: t }) - - const remoteVideoAccount = accountCreated.toAddRemoteJSON() - - // Now we'll add the video channel's meta data to our friends - const account = await addVideoAccountToFriends(remoteVideoAccount, t) + const accountCreated = await createLocalAccount(user.username, user.id, null, t) const videoChannelInfo = { name: `Default ${userCreated.username} channel` } const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) - return { account, videoChannel } + return { account: accountCreated, videoChannel } }) return res } +async function createLocalAccount (name: string, userId: number, applicationId: number, t: Sequelize.Transaction) { + const { publicKey, privateKey } = await createPrivateAndPublicKeys() + const url = getActivityPubUrl('account', name) + + const accountInstance = db.Account.build({ + name, + url, + publicKey, + privateKey, + followersCount: 0, + followingCount: 0, + inboxUrl: url + '/inbox', + outboxUrl: url + '/outbox', + sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox', + followersUrl: url + '/followers', + followingUrl: url + '/following', + userId, + applicationId, + podId: null // It is our pod + }) + + return accountInstance.save({ transaction: t }) +} + // --------------------------------------------------------------------------- export { - createUserAccountAndChannel + createUserAccountAndChannel, + createLocalAccount } diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index f81383ce8..459d9d4a8 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts @@ -1,10 +1,10 @@ import * as Sequelize from 'sequelize' -import { addVideoChannelToFriends } from './friends' import { database as db } from '../initializers' import { logger } from '../helpers' import { AccountInstance } from '../models' import { VideoChannelCreate } from '../../shared/models' +import { sendCreateVideoChannel } from './activitypub/send-request' async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account: AccountInstance, t: Sequelize.Transaction) { const videoChannelData = { @@ -22,10 +22,7 @@ async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account // Do not forget to add Account information to the created video channel videoChannelCreated.Account = account - const remoteVideoChannel = videoChannelCreated.toAddRemoteJSON() - - // Now we'll add the video channel's meta data to our friends - await addVideoChannelToFriends(remoteVideoChannel, t) + sendCreateVideoChannel(videoChannelCreated, t) return videoChannelCreated } diff --git a/server/middlewares/validators/activitypub/index.ts b/server/middlewares/validators/activitypub/index.ts index f1f26043e..84d1107fc 100644 --- a/server/middlewares/validators/activitypub/index.ts +++ b/server/middlewares/validators/activitypub/index.ts @@ -1,3 +1,2 @@ -export * from './pods' +export * from './activity' export * from './signature' -export * from './videos' diff --git a/server/middlewares/validators/activitypub/pods.ts b/server/middlewares/validators/activitypub/pods.ts deleted file mode 100644 index f917b61ee..000000000 --- a/server/middlewares/validators/activitypub/pods.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { body } from 'express-validator/check' -import * as express from 'express' - -import { database as db } from '../../../initializers' -import { isHostValid, logger } from '../../../helpers' -import { checkErrors } from '../utils' - -const remotePodsAddValidator = [ - body('host').custom(isHostValid).withMessage('Should have a host'), - body('email').isEmail().withMessage('Should have an email'), - body('publicKey').not().isEmpty().withMessage('Should have a public key'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking podsAdd parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - db.Pod.loadByHost(req.body.host) - .then(pod => { - // Pod with this host already exists - if (pod) { - return res.sendStatus(409) - } - - return next() - }) - .catch(err => { - logger.error('Cannot load pod by host.', err) - res.sendStatus(500) - }) - }) - } -] - -// --------------------------------------------------------------------------- - -export { - remotePodsAddValidator -} diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 46c00d679..0b7573d4f 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -2,7 +2,6 @@ export * from './account' export * from './oembed' export * from './activitypub' export * from './pagination' -export * from './pods' export * from './sort' export * from './users' export * from './videos' diff --git a/server/middlewares/validators/pods.ts b/server/middlewares/validators/pods.ts deleted file mode 100644 index 8465fea53..000000000 --- a/server/middlewares/validators/pods.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { body, param } from 'express-validator/check' -import * as express from 'express' - -import { database as db } from '../../initializers/database' -import { checkErrors } from './utils' -import { logger, isEachUniqueHostValid, isTestInstance } from '../../helpers' -import { CONFIG } from '../../initializers' -import { hasFriends } from '../../lib' - -const makeFriendsValidator = [ - body('hosts').custom(isEachUniqueHostValid).withMessage('Should have an array of unique hosts'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - // Force https if the administrator wants to make friends - if (isTestInstance() === false && CONFIG.WEBSERVER.SCHEME === 'http') { - return res.status(400) - .json({ - error: 'Cannot make friends with a non HTTPS web server.' - }) - .end() - } - - logger.debug('Checking makeFriends parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - hasFriends() - .then(heHasFriends => { - if (heHasFriends === true) { - // We need to quit our friends before make new ones - return res.sendStatus(409) - } - - return next() - }) - .catch(err => { - logger.error('Cannot know if we have friends.', err) - res.sendStatus(500) - }) - }) - } -] - -const podRemoveValidator = [ - param('id').isNumeric().not().isEmpty().withMessage('Should have a valid id'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking podRemoveValidator parameters', { parameters: req.params }) - - checkErrors(req, res, () => { - db.Pod.load(req.params.id) - .then(pod => { - if (!pod) { - logger.error('Cannot find pod %d.', req.params.id) - return res.sendStatus(404) - } - - res.locals.pod = pod - return next() - }) - .catch(err => { - logger.error('Cannot load pod %d.', req.params.id, err) - res.sendStatus(500) - }) - }) - } -] - -// --------------------------------------------------------------------------- - -export { - makeFriendsValidator, - podRemoveValidator -} diff --git a/server/models/account/account-interface.ts b/server/models/account/account-interface.ts index 2ef3e2246..a662eb992 100644 --- a/server/models/account/account-interface.ts +++ b/server/models/account/account-interface.ts @@ -13,8 +13,8 @@ export namespace AccountMethods { export type LoadAccountByPodAndUUID = (uuid: string, podId: number, transaction: Sequelize.Transaction) => Bluebird export type LoadLocalAccountByName = (name: string) => Bluebird export type ListOwned = () => Bluebird - export type ListFollowerUrlsForApi = (name: string, start: number, count: number) => Promise< ResultList > - export type ListFollowingUrlsForApi = (name: string, start: number, count: number) => Promise< ResultList > + export type ListFollowerUrlsForApi = (name: string, start: number, count?: number) => Promise< ResultList > + export type ListFollowingUrlsForApi = (name: string, start: number, count?: number) => Promise< ResultList > export type ToActivityPubObject = (this: AccountInstance) => ActivityPubActor export type IsOwned = (this: AccountInstance) => boolean diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 00c0aefd4..a79e13880 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -268,14 +268,15 @@ function afterDestroy (account: AccountInstance) { uuid: account.uuid } - return removeVideoAccountToFriends(removeVideoAccountToFriendsParams) + // FIXME: remove account in followers + // return removeVideoAccountToFriends(removeVideoAccountToFriendsParams) } return undefined } toActivityPubObject = function (this: AccountInstance) { - const type = this.podId ? 'Application' : 'Person' + const type = this.podId ? 'Application' as 'Application' : 'Person' as 'Person' const json = { type, @@ -346,11 +347,11 @@ listOwned = function () { return Account.findAll(query) } -listFollowerUrlsForApi = function (name: string, start: number, count: number) { +listFollowerUrlsForApi = function (name: string, start: number, count?: number) { return createListFollowForApiQuery('followers', name, start, count) } -listFollowingUrlsForApi = function (name: string, start: number, count: number) { +listFollowingUrlsForApi = function (name: string, start: number, count?: number) { return createListFollowForApiQuery('following', name, start, count) } @@ -405,7 +406,7 @@ loadAccountByPodAndUUID = function (uuid: string, podId: number, transaction: Se // ------------------------------ UTILS ------------------------------ -async function createListFollowForApiQuery (type: 'followers' | 'following', name: string, start: number, count: number) { +async function createListFollowForApiQuery (type: 'followers' | 'following', name: string, start: number, count?: number) { let firstJoin: string let secondJoin: string @@ -421,11 +422,13 @@ async function createListFollowForApiQuery (type: 'followers' | 'following', nam const tasks: Promise[] = [] for (const selection of selections) { - const query = 'SELECT ' + selection + ' FROM "Account" ' + + let query = 'SELECT ' + selection + ' FROM "Account" ' + 'INNER JOIN "AccountFollower" ON "AccountFollower"."' + firstJoin + '" = "Account"."id" ' + 'INNER JOIN "Account" AS "Followers" ON "Followers"."id" = "AccountFollower"."' + secondJoin + '" ' + 'WHERE "Account"."name" = \'$name\' ' + - 'LIMIT ' + start + ', ' + count + 'LIMIT ' + start + + if (count !== undefined) query += ', ' + count const options = { bind: { name }, diff --git a/server/models/job/job-interface.ts b/server/models/job/job-interface.ts index 163930a4f..411a05029 100644 --- a/server/models/job/job-interface.ts +++ b/server/models/job/job-interface.ts @@ -14,7 +14,7 @@ export interface JobClass { export interface JobAttributes { state: JobState handlerName: string - handlerInputData: object + handlerInputData: any } export interface JobInstance extends JobClass, JobAttributes, Sequelize.Instance { diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 93a611fa0..183ff3436 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -1,7 +1,6 @@ import * as Sequelize from 'sequelize' import { isVideoChannelNameValid, isVideoChannelDescriptionValid } from '../../helpers' -import { removeVideoChannelToFriends } from '../../lib' import { addMethodsToModel, getSort } from '../utils' import { @@ -143,12 +142,13 @@ toFormattedJSON = function (this: VideoChannelInstance) { toActivityPubObject = function (this: VideoChannelInstance) { const json = { + type: 'VideoChannel' as 'VideoChannel', + id: this.url, uuid: this.uuid, + content: this.description, name: this.name, - description: this.description, - createdAt: this.createdAt, - updatedAt: this.updatedAt, - ownerUUID: this.Account.uuid + published: this.createdAt, + updated: this.updatedAt } return json @@ -180,7 +180,7 @@ function afterDestroy (videoChannel: VideoChannelInstance) { uuid: videoChannel.uuid } - return removeVideoChannelToFriends(removeVideoChannelToFriendsParams) + // FIXME: send remove event to followers } return undefined @@ -277,7 +277,7 @@ loadByUUIDOrUrl = function (uuid: string, url: string, t?: Sequelize.Transaction { uuid }, { url } ] - }, + } } if (t !== undefined) query.transaction = t diff --git a/server/models/video/video.ts b/server/models/video/video.ts index b5d333347..10ae5097c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1,58 +1,52 @@ -import * as safeBuffer from 'safe-buffer' -const Buffer = safeBuffer.Buffer -import * as magnetUtil from 'magnet-uri' import { map, maxBy, truncate } from 'lodash' +import * as magnetUtil from 'magnet-uri' import * as parseTorrent from 'parse-torrent' import { join } from 'path' +import * as safeBuffer from 'safe-buffer' import * as Sequelize from 'sequelize' - -import { TagInstance } from './tag-interface' +import { VideoPrivacy, VideoResolution } from '../../../shared' +import { VideoTorrentObject } from '../../../shared/models/activitypub/objects/video-torrent-object' import { - logger, - isVideoNameValid, + createTorrentPromise, + generateImageFromVideoFile, + getActivityPubUrl, + getVideoFileHeight, isVideoCategoryValid, - isVideoLicenceValid, - isVideoLanguageValid, - isVideoNSFWValid, isVideoDescriptionValid, isVideoDurationValid, + isVideoLanguageValid, + isVideoLicenceValid, + isVideoNameValid, + isVideoNSFWValid, isVideoPrivacyValid, - readFileBufferPromise, - unlinkPromise, + logger, renamePromise, - writeFilePromise, - createTorrentPromise, statPromise, - generateImageFromVideoFile, transcode, - getVideoFileHeight, - getActivityPubUrl + unlinkPromise, + writeFilePromise } from '../../helpers' import { + API_VERSION, CONFIG, + CONSTRAINTS_FIELDS, + PREVIEWS_SIZE, REMOTE_SCHEME, STATIC_PATHS, + THUMBNAILS_SIZE, VIDEO_CATEGORIES, - VIDEO_LICENCES, VIDEO_LANGUAGES, - THUMBNAILS_SIZE, - PREVIEWS_SIZE, - CONSTRAINTS_FIELDS, - API_VERSION, + VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../initializers' -import { removeVideoToFriends } from '../../lib' -import { VideoResolution, VideoPrivacy } from '../../../shared' -import { VideoFileInstance, VideoFileModel } from './video-file-interface' import { addMethodsToModel, getSort } from '../utils' -import { - VideoInstance, - VideoAttributes, - VideoMethods -} from './video-interface' -import { VideoTorrentObject } from '../../../shared/models/activitypub/objects/video-torrent-object' +import { TagInstance } from './tag-interface' +import { VideoFileInstance, VideoFileModel } from './video-file-interface' +import { VideoAttributes, VideoInstance, VideoMethods } from './video-interface' + +const Buffer = safeBuffer.Buffer let Video: Sequelize.Model let getOriginalFile: VideoMethods.GetOriginalFile @@ -374,8 +368,8 @@ function afterDestroy (video: VideoInstance) { } tasks.push( - video.removePreview(), - removeVideoToFriends(removeVideoToFriendsParams) + video.removePreview() + // FIXME: remove video for followers ) // Remove physical files and torrents @@ -566,7 +560,7 @@ toActivityPubObject = function (this: VideoInstance) { const { baseUrlHttp, baseUrlWs } = getBaseUrls(this) const tag = this.Tags.map(t => ({ - type: 'Hashtag', + type: 'Hashtag' as 'Hashtag', name: t.name })) @@ -596,7 +590,7 @@ toActivityPubObject = function (this: VideoInstance) { } const videoObject: VideoTorrentObject = { - type: 'Video', + type: 'Video' as 'Video', id: getActivityPubUrl('video', this.uuid), name: this.name, // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration @@ -604,15 +598,15 @@ toActivityPubObject = function (this: VideoInstance) { uuid: this.uuid, tag, category: { - id: this.category, - label: this.getCategoryLabel() + identifier: this.category + '', + name: this.getCategoryLabel() }, licence: { - id: this.licence, + identifier: this.licence + '', name: this.getLicenceLabel() }, language: { - id: this.language, + identifier: this.language + '', name: this.getLanguageLabel() }, views: this.views, diff --git a/server/tests/api/video-channels.ts b/server/tests/api/video-channels.ts index 601b527ef..b851d7185 100644 --- a/server/tests/api/video-channels.ts +++ b/server/tests/api/video-channels.ts @@ -65,7 +65,7 @@ describe('Test a video channels', function () { }) it('Should have two video channels when getting author channels', async () => { - const res = await getAuthorVideoChannelsList(server.url, userInfo.author.uuid) + const res = await getAuthorVideoChannelsList(server.url, userInfo.account.uuid) expect(res.body.total).to.equal(2) expect(res.body.data).to.be.an('array') diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts index dc562c00a..a2494da25 100644 --- a/shared/models/activitypub/activity.ts +++ b/shared/models/activitypub/activity.ts @@ -4,7 +4,7 @@ import { } from './objects' import { ActivityPubSignature } from './activitypub-signature' -export type Activity = ActivityCreate | ActivityUpdate | ActivityFlag +export type Activity = ActivityCreate | ActivityAdd | ActivityUpdate | ActivityFlag // Flag -> report abuse export type ActivityType = 'Create' | 'Add' | 'Update' | 'Flag' diff --git a/shared/models/pods/pod.model.ts b/shared/models/pods/pod.model.ts index d25421936..ff9e8f2f5 100644 --- a/shared/models/pods/pod.model.ts +++ b/shared/models/pods/pod.model.ts @@ -1,7 +1,6 @@ export interface Pod { id: number, host: string, - email: string, score: number, createdAt: Date } diff --git a/shared/models/users/user.model.ts b/shared/models/users/user.model.ts index ee2147590..a8012734c 100644 --- a/shared/models/users/user.model.ts +++ b/shared/models/users/user.model.ts @@ -9,7 +9,7 @@ export interface User { role: UserRole videoQuota: number createdAt: Date, - author: { + account: { id: number uuid: string } diff --git a/yarn.lock b/yarn.lock index 52685a8cc..773ff7350 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3473,12 +3473,6 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request-replay@^1.0.2: - version "1.0.4" - resolved "https://registry.yarnpkg.com/request-replay/-/request-replay-1.0.4.tgz#b6e5953a7eb39fc8a48e8111c277d35355adfe06" - dependencies: - retry "^0.10.0" - request@2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" @@ -3564,10 +3558,6 @@ retry-as-promised@^2.3.1: bluebird "^3.4.6" debug "^2.6.9" -retry@^0.10.0: - version "0.10.1" - resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4" - rimraf@2, rimraf@^2.2.8, rimraf@^2.4.2, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.1: version "2.6.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" -- 2.41.0