From e4f97babf701481b55cc10fb3448feab5f97c867 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 9 Nov 2017 17:51:58 +0100 Subject: Begin activitypub --- server/controllers/activitypub/client.ts | 65 +++ server/controllers/activitypub/inbox.ts | 72 +++ server/controllers/activitypub/index.ts | 15 + server/controllers/activitypub/pods.ts | 69 +++ server/controllers/activitypub/videos.ts | 589 +++++++++++++++++++++ server/controllers/api/remote/index.ts | 18 - server/controllers/api/remote/pods.ts | 69 --- server/controllers/api/remote/videos.ts | 589 --------------------- server/helpers/activitypub.ts | 123 +++++ server/helpers/core-utils.ts | 26 +- .../custom-validators/activitypub/account.ts | 123 +++++ .../helpers/custom-validators/activitypub/index.ts | 4 + .../helpers/custom-validators/activitypub/misc.ts | 17 + .../custom-validators/activitypub/signature.ts | 22 + .../custom-validators/activitypub/videos.ts | 184 +++++++ server/helpers/custom-validators/index.ts | 2 +- server/helpers/custom-validators/remote/index.ts | 1 - server/helpers/custom-validators/remote/videos.ts | 184 ------- server/helpers/ffmpeg-utils.ts | 1 - server/helpers/index.ts | 2 + server/helpers/peertube-crypto.ts | 158 ++---- server/helpers/requests.ts | 78 +-- server/helpers/webfinger.ts | 44 ++ server/initializers/checker.ts | 2 +- server/initializers/constants.ts | 31 +- server/initializers/database.ts | 14 +- server/lib/activitypub/index.ts | 3 + server/lib/activitypub/process-create.ts | 104 ++++ server/lib/activitypub/process-flag.ts | 17 + server/lib/activitypub/process-update.ts | 29 + server/lib/activitypub/send-request.ts | 129 +++++ server/lib/index.ts | 1 + server/lib/jobs/handlers/index.ts | 17 - server/lib/jobs/handlers/video-file-optimizer.ts | 85 --- server/lib/jobs/handlers/video-file-transcoder.ts | 49 -- .../http-request-broadcast-handler.ts | 25 + .../http-request-job-scheduler.ts | 17 + .../http-request-unicast-handler.ts | 25 + .../lib/jobs/http-request-job-scheduler/index.ts | 1 + server/lib/jobs/index.ts | 3 +- server/lib/jobs/job-scheduler.ts | 35 +- server/lib/jobs/transcoding-job-scheduler/index.ts | 1 + .../transcoding-job-scheduler.ts | 17 + .../video-file-optimizer-handler.ts | 85 +++ .../video-file-transcoder-handler.ts | 49 ++ server/lib/user.ts | 18 +- server/lib/video-channel.ts | 10 +- server/middlewares/activitypub.ts | 57 ++ server/middlewares/index.ts | 2 +- server/middlewares/secure.ts | 55 -- server/middlewares/validators/account.ts | 53 ++ server/middlewares/validators/activitypub/index.ts | 3 + server/middlewares/validators/activitypub/pods.ts | 38 ++ .../validators/activitypub/signature.ts | 30 ++ .../middlewares/validators/activitypub/videos.ts | 61 +++ server/middlewares/validators/index.ts | 3 +- server/middlewares/validators/remote/index.ts | 3 - server/middlewares/validators/remote/pods.ts | 38 -- server/middlewares/validators/remote/signature.ts | 22 - server/middlewares/validators/remote/videos.ts | 61 --- server/models/account/account-follow-interface.ts | 23 + server/models/account/account-follow.ts | 56 ++ server/models/account/account-interface.ts | 74 +++ .../models/account/account-video-rate-interface.ts | 26 + server/models/account/account-video-rate.ts | 78 +++ server/models/account/account.ts | 444 ++++++++++++++++ server/models/account/index.ts | 4 + server/models/account/user-interface.ts | 69 +++ server/models/account/user.ts | 311 +++++++++++ server/models/index.ts | 2 +- server/models/job/job-interface.ts | 6 +- server/models/job/job.ts | 12 +- server/models/oauth/oauth-token-interface.ts | 2 +- server/models/pod/pod-interface.ts | 2 - server/models/pod/pod.ts | 12 - server/models/user/index.ts | 2 - server/models/user/user-interface.ts | 69 --- server/models/user/user-video-rate-interface.ts | 26 - server/models/user/user-video-rate.ts | 78 --- server/models/user/user.ts | 312 ----------- server/models/video/author-interface.ts | 45 -- server/models/video/author.ts | 171 ------ server/models/video/video-channel-interface.ts | 38 +- server/models/video/video-channel.ts | 100 ++-- server/models/video/video-interface.ts | 60 ++- server/models/video/video.ts | 378 ++++++------- 86 files changed, 3647 insertions(+), 2401 deletions(-) create mode 100644 server/controllers/activitypub/client.ts create mode 100644 server/controllers/activitypub/inbox.ts create mode 100644 server/controllers/activitypub/index.ts create mode 100644 server/controllers/activitypub/pods.ts create mode 100644 server/controllers/activitypub/videos.ts delete mode 100644 server/controllers/api/remote/index.ts delete mode 100644 server/controllers/api/remote/pods.ts delete mode 100644 server/controllers/api/remote/videos.ts create mode 100644 server/helpers/activitypub.ts create mode 100644 server/helpers/custom-validators/activitypub/account.ts create mode 100644 server/helpers/custom-validators/activitypub/index.ts create mode 100644 server/helpers/custom-validators/activitypub/misc.ts create mode 100644 server/helpers/custom-validators/activitypub/signature.ts create mode 100644 server/helpers/custom-validators/activitypub/videos.ts delete mode 100644 server/helpers/custom-validators/remote/index.ts delete mode 100644 server/helpers/custom-validators/remote/videos.ts create mode 100644 server/helpers/webfinger.ts create mode 100644 server/lib/activitypub/index.ts create mode 100644 server/lib/activitypub/process-create.ts create mode 100644 server/lib/activitypub/process-flag.ts create mode 100644 server/lib/activitypub/process-update.ts create mode 100644 server/lib/activitypub/send-request.ts delete mode 100644 server/lib/jobs/handlers/index.ts delete mode 100644 server/lib/jobs/handlers/video-file-optimizer.ts delete mode 100644 server/lib/jobs/handlers/video-file-transcoder.ts create mode 100644 server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts create mode 100644 server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts create mode 100644 server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts create mode 100644 server/lib/jobs/http-request-job-scheduler/index.ts create mode 100644 server/lib/jobs/transcoding-job-scheduler/index.ts create mode 100644 server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts create mode 100644 server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts create mode 100644 server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts create mode 100644 server/middlewares/activitypub.ts delete mode 100644 server/middlewares/secure.ts create mode 100644 server/middlewares/validators/account.ts create mode 100644 server/middlewares/validators/activitypub/index.ts create mode 100644 server/middlewares/validators/activitypub/pods.ts create mode 100644 server/middlewares/validators/activitypub/signature.ts create mode 100644 server/middlewares/validators/activitypub/videos.ts delete mode 100644 server/middlewares/validators/remote/index.ts delete mode 100644 server/middlewares/validators/remote/pods.ts delete mode 100644 server/middlewares/validators/remote/signature.ts delete mode 100644 server/middlewares/validators/remote/videos.ts create mode 100644 server/models/account/account-follow-interface.ts create mode 100644 server/models/account/account-follow.ts create mode 100644 server/models/account/account-interface.ts create mode 100644 server/models/account/account-video-rate-interface.ts create mode 100644 server/models/account/account-video-rate.ts create mode 100644 server/models/account/account.ts create mode 100644 server/models/account/index.ts create mode 100644 server/models/account/user-interface.ts create mode 100644 server/models/account/user.ts delete mode 100644 server/models/user/index.ts delete mode 100644 server/models/user/user-interface.ts delete mode 100644 server/models/user/user-video-rate-interface.ts delete mode 100644 server/models/user/user-video-rate.ts delete mode 100644 server/models/user/user.ts delete mode 100644 server/models/video/author-interface.ts delete mode 100644 server/models/video/author.ts (limited to 'server') diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts new file mode 100644 index 000000000..28d08b3f4 --- /dev/null +++ b/server/controllers/activitypub/client.ts @@ -0,0 +1,65 @@ +// Intercept ActivityPub client requests +import * as express from 'express' + +import { database as db } from '../../initializers' +import { executeIfActivityPub, localAccountValidator } from '../../middlewares' +import { pageToStartAndCount } from '../../helpers' +import { AccountInstance } from '../../models' +import { activityPubCollectionPagination } from '../../helpers/activitypub' +import { ACTIVITY_PUB } from '../../initializers/constants' +import { asyncMiddleware } from '../../middlewares/async' + +const activityPubClientRouter = express.Router() + +activityPubClientRouter.get('/account/:name', + executeIfActivityPub(localAccountValidator), + executeIfActivityPub(asyncMiddleware(accountController)) +) + +activityPubClientRouter.get('/account/:name/followers', + executeIfActivityPub(localAccountValidator), + executeIfActivityPub(asyncMiddleware(accountFollowersController)) +) + +activityPubClientRouter.get('/account/:name/following', + executeIfActivityPub(localAccountValidator), + executeIfActivityPub(asyncMiddleware(accountFollowingController)) +) + +// --------------------------------------------------------------------------- + +export { + activityPubClientRouter +} + +// --------------------------------------------------------------------------- + +async function accountController (req: express.Request, res: express.Response, next: express.NextFunction) { + const account: AccountInstance = res.locals.account + + return res.json(account.toActivityPubObject()).end() +} + +async function accountFollowersController (req: express.Request, res: express.Response, next: express.NextFunction) { + const account: AccountInstance = res.locals.account + + const page = req.params.page || 1 + const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) + + const result = await db.Account.listFollowerUrlsForApi(account.name, start, count) + const activityPubResult = activityPubCollectionPagination(req.url, page, result) + + return res.json(activityPubResult) +} + +async function accountFollowingController (req: express.Request, res: express.Response, next: express.NextFunction) { + const account: AccountInstance = res.locals.account + + const page = req.params.page || 1 + const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) + + const result = await db.Account.listFollowingUrlsForApi(account.name, start, count) + const activityPubResult = activityPubCollectionPagination(req.url, page, result) + + return res.json(activityPubResult) +} diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts new file mode 100644 index 000000000..79d989c2c --- /dev/null +++ b/server/controllers/activitypub/inbox.ts @@ -0,0 +1,72 @@ +import * as express from 'express' + +import { + processCreateActivity, + processUpdateActivity, + processFlagActivity +} from '../../lib' +import { + Activity, + ActivityType, + RootActivity, + ActivityPubCollection, + ActivityPubOrderedCollection +} from '../../../shared' +import { + signatureValidator, + checkSignature, + asyncMiddleware +} from '../../middlewares' +import { logger } from '../../helpers' + +const processActivity: { [ P in ActivityType ]: (activity: Activity) => Promise } = { + Create: processCreateActivity, + Update: processUpdateActivity, + Flag: processFlagActivity +} + +const inboxRouter = express.Router() + +inboxRouter.post('/', + signatureValidator, + asyncMiddleware(checkSignature), + // inboxValidator, + asyncMiddleware(inboxController) +) + +// --------------------------------------------------------------------------- + +export { + inboxRouter +} + +// --------------------------------------------------------------------------- + +async function inboxController (req: express.Request, res: express.Response, next: express.NextFunction) { + const rootActivity: RootActivity = req.body + let activities: Activity[] = [] + + if ([ 'Collection', 'CollectionPage' ].indexOf(rootActivity.type) !== -1) { + activities = (rootActivity as ActivityPubCollection).items + } else if ([ 'OrderedCollection', 'OrderedCollectionPage' ].indexOf(rootActivity.type) !== -1) { + activities = (rootActivity as ActivityPubOrderedCollection).orderedItems + } else { + activities = [ rootActivity as Activity ] + } + + await processActivities(activities) + + res.status(204).end() +} + +async function processActivities (activities: Activity[]) { + for (const activity of activities) { + const activityProcessor = processActivity[activity.type] + if (activityProcessor === undefined) { + logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) + continue + } + + await activityProcessor(activity) + } +} diff --git a/server/controllers/activitypub/index.ts b/server/controllers/activitypub/index.ts new file mode 100644 index 000000000..7a4602b37 --- /dev/null +++ b/server/controllers/activitypub/index.ts @@ -0,0 +1,15 @@ +import * as express from 'express' + +import { badRequest } from '../../helpers' +import { inboxRouter } from './inbox' + +const remoteRouter = express.Router() + +remoteRouter.use('/inbox', inboxRouter) +remoteRouter.use('/*', badRequest) + +// --------------------------------------------------------------------------- + +export { + remoteRouter +} diff --git a/server/controllers/activitypub/pods.ts b/server/controllers/activitypub/pods.ts new file mode 100644 index 000000000..326eb61ac --- /dev/null +++ b/server/controllers/activitypub/pods.ts @@ -0,0 +1,69 @@ +import * as express from 'express' + +import { database as db } from '../../../initializers/database' +import { + checkSignature, + signatureValidator, + setBodyHostPort, + remotePodsAddValidator, + asyncMiddleware +} from '../../../middlewares' +import { sendOwnedDataToPod } from '../../../lib' +import { getMyPublicCert, getFormattedObjects } from '../../../helpers' +import { CONFIG } from '../../../initializers' +import { PodInstance } from '../../../models' +import { PodSignature, Pod as FormattedPod } from '../../../../shared' + +const remotePodsRouter = express.Router() + +remotePodsRouter.post('/remove', + signatureValidator, + checkSignature, + asyncMiddleware(removePods) +) + +remotePodsRouter.post('/list', + asyncMiddleware(remotePodsList) +) + +remotePodsRouter.post('/add', + setBodyHostPort, // We need to modify the host before running the validator! + remotePodsAddValidator, + asyncMiddleware(addPods) +) + +// --------------------------------------------------------------------------- + +export { + remotePodsRouter +} + +// --------------------------------------------------------------------------- + +async function addPods (req: express.Request, res: express.Response, next: express.NextFunction) { + const information = req.body + + const pod = db.Pod.build(information) + const podCreated = await pod.save() + + await sendOwnedDataToPod(podCreated.id) + + const cert = await getMyPublicCert() + return res.json({ cert, email: CONFIG.ADMIN.EMAIL }) +} + +async function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) { + const pods = await db.Pod.list() + + return res.json(getFormattedObjects(pods, pods.length)) +} + +async function removePods (req: express.Request, res: express.Response, next: express.NextFunction) { + const signature: PodSignature = req.body.signature + const host = signature.host + + const pod = await db.Pod.loadByHost(host) + await pod.destroy() + + return res.type('json').status(204).end() +} diff --git a/server/controllers/activitypub/videos.ts b/server/controllers/activitypub/videos.ts new file mode 100644 index 000000000..cba47f0a1 --- /dev/null +++ b/server/controllers/activitypub/videos.ts @@ -0,0 +1,589 @@ +import * as express from 'express' +import * as Bluebird from 'bluebird' +import * as Sequelize from 'sequelize' + +import { database as db } from '../../../initializers/database' +import { + REQUEST_ENDPOINT_ACTIONS, + REQUEST_ENDPOINTS, + REQUEST_VIDEO_EVENT_TYPES, + REQUEST_VIDEO_QADU_TYPES +} from '../../../initializers' +import { + checkSignature, + signatureValidator, + remoteVideosValidator, + remoteQaduVideosValidator, + remoteEventsVideosValidator +} from '../../../middlewares' +import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers' +import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib' +import { PodInstance, VideoFileInstance } from '../../../models' +import { + RemoteVideoRequest, + RemoteVideoCreateData, + RemoteVideoUpdateData, + RemoteVideoRemoveData, + RemoteVideoReportAbuseData, + RemoteQaduVideoRequest, + RemoteQaduVideoData, + RemoteVideoEventRequest, + RemoteVideoEventData, + RemoteVideoChannelCreateData, + RemoteVideoChannelUpdateData, + RemoteVideoChannelRemoveData, + RemoteVideoAuthorRemoveData, + RemoteVideoAuthorCreateData +} from '../../../../shared' +import { VideoInstance } from '../../../models/video/video-interface' + +const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] + +// Functions to call when processing a remote request +// FIXME: use RemoteVideoRequestType as id type +const functionsHash: { [ id: string ]: (...args) => Promise } = {} +functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper +functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper +functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper +functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper +functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper +functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper +functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper +functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper +functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper + +const remoteVideosRouter = express.Router() + +remoteVideosRouter.post('/', + signatureValidator, + checkSignature, + remoteVideosValidator, + remoteVideos +) + +remoteVideosRouter.post('/qadu', + signatureValidator, + checkSignature, + remoteQaduVideosValidator, + remoteVideosQadu +) + +remoteVideosRouter.post('/events', + signatureValidator, + checkSignature, + remoteEventsVideosValidator, + remoteVideosEvents +) + +// --------------------------------------------------------------------------- + +export { + remoteVideosRouter +} + +// --------------------------------------------------------------------------- + +function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) { + const requests: RemoteVideoRequest[] = req.body.data + const fromPod = res.locals.secure.pod + + // We need to process in the same order to keep consistency + Bluebird.each(requests, request => { + const data = request.data + + // Get the function we need to call in order to process the request + const fun = functionsHash[request.type] + if (fun === undefined) { + logger.error('Unknown remote request type %s.', request.type) + return + } + + return fun.call(this, data, fromPod) + }) + .catch(err => logger.error('Error managing remote videos.', err)) + + // Don't block the other pod + return res.type('json').status(204).end() +} + +function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) { + const requests: RemoteQaduVideoRequest[] = req.body.data + const fromPod = res.locals.secure.pod + + Bluebird.each(requests, request => { + const videoData = request.data + + return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod) + }) + .catch(err => logger.error('Error managing remote videos.', err)) + + return res.type('json').status(204).end() +} + +function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) { + const requests: RemoteVideoEventRequest[] = req.body.data + const fromPod = res.locals.secure.pod + + Bluebird.each(requests, request => { + const eventData = request.data + + return processVideosEventsRetryWrapper(eventData, fromPod) + }) + .catch(err => logger.error('Error managing remote videos.', err)) + + return res.type('json').status(204).end() +} + +async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) { + const options = { + arguments: [ eventData, fromPod ], + errorMessage: 'Cannot process videos events with many retries.' + } + + await retryTransactionWrapper(processVideosEvents, options) +} + +async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) { + await db.sequelize.transaction(async t => { + const sequelizeOptions = { transaction: t } + const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t) + + let columnToUpdate + let qaduType + + switch (eventData.eventType) { + case REQUEST_VIDEO_EVENT_TYPES.VIEWS: + columnToUpdate = 'views' + qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS + break + + case REQUEST_VIDEO_EVENT_TYPES.LIKES: + columnToUpdate = 'likes' + qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES + break + + case REQUEST_VIDEO_EVENT_TYPES.DISLIKES: + columnToUpdate = 'dislikes' + qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES + break + + default: + throw new Error('Unknown video event type.') + } + + const query = {} + query[columnToUpdate] = eventData.count + + await videoInstance.increment(query, sequelizeOptions) + + const qadusParams = [ + { + videoId: videoInstance.id, + type: qaduType + } + ] + await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) + }) + + logger.info('Remote video event processed for video with uuid %s.', eventData.uuid) +} + +async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) { + const options = { + arguments: [ videoData, fromPod ], + errorMessage: 'Cannot update quick and dirty the remote video with many retries.' + } + + await retryTransactionWrapper(quickAndDirtyUpdateVideo, options) +} + +async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { + let videoUUID = '' + + await db.sequelize.transaction(async t => { + const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t) + const sequelizeOptions = { transaction: t } + + videoUUID = videoInstance.uuid + + if (videoData.views) { + videoInstance.set('views', videoData.views) + } + + if (videoData.likes) { + videoInstance.set('likes', videoData.likes) + } + + if (videoData.dislikes) { + videoInstance.set('dislikes', videoData.dislikes) + } + + await videoInstance.save(sequelizeOptions) + }) + + logger.info('Remote video with uuid %s quick and dirty updated', videoUUID) +} + +// Handle retries on fail +async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { + const options = { + arguments: [ videoToCreateData, fromPod ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + await retryTransactionWrapper(addRemoteVideo, options) +} + +async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { + logger.debug('Adding remote video "%s".', videoToCreateData.uuid) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) + if (videoFromDatabase) throw new Error('UUID already exists.') + + const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) + if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') + + const tags = videoToCreateData.tags + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + + const videoData = { + name: videoToCreateData.name, + uuid: videoToCreateData.uuid, + category: videoToCreateData.category, + licence: videoToCreateData.licence, + language: videoToCreateData.language, + nsfw: videoToCreateData.nsfw, + description: videoToCreateData.truncatedDescription, + channelId: videoChannel.id, + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: videoToCreateData.updatedAt, + views: videoToCreateData.views, + likes: videoToCreateData.likes, + dislikes: videoToCreateData.dislikes, + remote: true, + privacy: videoToCreateData.privacy + } + + const video = db.Video.build(videoData) + await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) + const videoCreated = await video.save(sequelizeOptions) + + const tasks = [] + for (const fileData of videoToCreateData.files) { + const videoFileInstance = db.VideoFile.build({ + extname: fileData.extname, + infoHash: fileData.infoHash, + resolution: fileData.resolution, + size: fileData.size, + videoId: videoCreated.id + }) + + tasks.push(videoFileInstance.save(sequelizeOptions)) + } + + await Promise.all(tasks) + + await videoCreated.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) +} + +// Handle retries on fail +async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { + const options = { + arguments: [ videoAttributesToUpdate, fromPod ], + errorMessage: 'Cannot update the remote video with many retries' + } + + await retryTransactionWrapper(updateRemoteVideo, options) +} + +async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) + let videoInstance: VideoInstance + let videoFieldsSave: object + + try { + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t) + videoFieldsSave = videoInstance.toJSON() + const tags = videoAttributesToUpdate.tags + + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('category', videoAttributesToUpdate.category) + videoInstance.set('licence', videoAttributesToUpdate.licence) + videoInstance.set('language', videoAttributesToUpdate.language) + videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) + videoInstance.set('description', videoAttributesToUpdate.truncatedDescription) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) + videoInstance.set('views', videoAttributesToUpdate.views) + videoInstance.set('likes', videoAttributesToUpdate.likes) + videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) + videoInstance.set('privacy', videoAttributesToUpdate.privacy) + + await videoInstance.save(sequelizeOptions) + + // Remove old video files + const videoFileDestroyTasks: Bluebird[] = [] + for (const videoFile of videoInstance.VideoFiles) { + videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) + } + await Promise.all(videoFileDestroyTasks) + + const videoFileCreateTasks: Bluebird[] = [] + for (const fileData of videoAttributesToUpdate.files) { + const videoFileInstance = db.VideoFile.build({ + extname: fileData.extname, + infoHash: fileData.infoHash, + resolution: fileData.resolution, + size: fileData.size, + videoId: videoInstance.id + }) + + videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions)) + } + + await Promise.all(videoFileCreateTasks) + + await videoInstance.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid) + } catch (err) { + if (videoInstance !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(videoInstance, videoFieldsSave) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', err) + throw err + } +} + +async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { + const options = { + arguments: [ videoToRemoveData, fromPod ], + errorMessage: 'Cannot remove the remote video channel with many retries.' + } + + await retryTransactionWrapper(removeRemoteVideo, options) +} + +async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { + logger.debug('Removing remote video "%s".', videoToRemoveData.uuid) + + await db.sequelize.transaction(async t => { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t) + await videoInstance.destroy({ transaction: t }) + }) + + logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid) +} + +async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) { + const options = { + arguments: [ authorToCreateData, fromPod ], + errorMessage: 'Cannot insert the remote video author with many retries.' + } + + await retryTransactionWrapper(addRemoteVideoAuthor, options) +} + +async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) { + logger.debug('Adding remote video author "%s".', authorToCreateData.uuid) + + await db.sequelize.transaction(async t => { + const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t) + if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.') + + const videoAuthorData = { + name: authorToCreateData.name, + uuid: authorToCreateData.uuid, + userId: null, // Not on our pod + podId: fromPod.id + } + + const author = db.Author.build(videoAuthorData) + await author.save({ transaction: t }) + }) + + logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid) +} + +async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { + const options = { + arguments: [ authorAttributesToRemove, fromPod ], + errorMessage: 'Cannot remove the remote video author with many retries.' + } + + await retryTransactionWrapper(removeRemoteVideoAuthor, options) +} + +async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { + logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid) + + await db.sequelize.transaction(async t => { + const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t) + await videoAuthor.destroy({ transaction: t }) + }) + + logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid) +} + +async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) { + const options = { + arguments: [ videoChannelToCreateData, fromPod ], + errorMessage: 'Cannot insert the remote video channel with many retries.' + } + + await retryTransactionWrapper(addRemoteVideoChannel, options) +} + +async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) { + logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid) + + await db.sequelize.transaction(async t => { + const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid) + if (videoChannelInDatabase) { + throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.') + } + + const authorUUID = videoChannelToCreateData.ownerUUID + const podId = fromPod.id + + const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t) + if (!author) throw new Error('Unknown author UUID' + authorUUID + '.') + + const videoChannelData = { + name: videoChannelToCreateData.name, + description: videoChannelToCreateData.description, + uuid: videoChannelToCreateData.uuid, + createdAt: videoChannelToCreateData.createdAt, + updatedAt: videoChannelToCreateData.updatedAt, + remote: true, + authorId: author.id + } + + const videoChannel = db.VideoChannel.build(videoChannelData) + await videoChannel.save({ transaction: t }) + }) + + logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid) +} + +async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) { + const options = { + arguments: [ videoChannelAttributesToUpdate, fromPod ], + errorMessage: 'Cannot update the remote video channel with many retries.' + } + + await retryTransactionWrapper(updateRemoteVideoChannel, options) +} + +async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) { + logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t) + videoChannelInstance.set('name', videoChannelAttributesToUpdate.name) + videoChannelInstance.set('description', videoChannelAttributesToUpdate.description) + videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt) + videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt) + + await videoChannelInstance.save(sequelizeOptions) + }) + + logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid) +} + +async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { + const options = { + arguments: [ videoChannelAttributesToRemove, fromPod ], + errorMessage: 'Cannot remove the remote video channel with many retries.' + } + + await retryTransactionWrapper(removeRemoteVideoChannel, options) +} + +async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { + logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid) + + await db.sequelize.transaction(async t => { + const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t) + await videoChannel.destroy({ transaction: t }) + }) + + logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid) +} + +async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { + const options = { + arguments: [ reportData, fromPod ], + errorMessage: 'Cannot create remote abuse video with many retries.' + } + + await retryTransactionWrapper(reportAbuseRemoteVideo, options) +} + +async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { + logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID) + + await db.sequelize.transaction(async t => { + const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t) + const videoAbuseData = { + reporterUsername: reportData.reporterUsername, + reason: reportData.reportReason, + reporterPodId: fromPod.id, + videoId: videoInstance.id + } + + await db.VideoAbuse.create(videoAbuseData) + + }) + + logger.info('Remote abuse for video uuid %s created', reportData.videoUUID) +} + +async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) { + try { + const video = await db.Video.loadLocalVideoByUUID(id, t) + + if (!video) throw new Error('Video ' + id + ' not found') + + return video + } catch (err) { + logger.error('Cannot load owned video from id.', { error: err.stack, id }) + throw err + } +} + +async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) { + try { + const video = await db.Video.loadByHostAndUUID(podHost, uuid, t) + if (!video) throw new Error('Video not found') + + return video + } catch (err) { + logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid }) + throw err + } +} diff --git a/server/controllers/api/remote/index.ts b/server/controllers/api/remote/index.ts deleted file mode 100644 index d3522772b..000000000 --- a/server/controllers/api/remote/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -import * as express from 'express' - -import { badRequest } from '../../../helpers' - -import { remotePodsRouter } from './pods' -import { remoteVideosRouter } from './videos' - -const remoteRouter = express.Router() - -remoteRouter.use('/pods', remotePodsRouter) -remoteRouter.use('/videos', remoteVideosRouter) -remoteRouter.use('/*', badRequest) - -// --------------------------------------------------------------------------- - -export { - remoteRouter -} diff --git a/server/controllers/api/remote/pods.ts b/server/controllers/api/remote/pods.ts deleted file mode 100644 index 326eb61ac..000000000 --- a/server/controllers/api/remote/pods.ts +++ /dev/null @@ -1,69 +0,0 @@ -import * as express from 'express' - -import { database as db } from '../../../initializers/database' -import { - checkSignature, - signatureValidator, - setBodyHostPort, - remotePodsAddValidator, - asyncMiddleware -} from '../../../middlewares' -import { sendOwnedDataToPod } from '../../../lib' -import { getMyPublicCert, getFormattedObjects } from '../../../helpers' -import { CONFIG } from '../../../initializers' -import { PodInstance } from '../../../models' -import { PodSignature, Pod as FormattedPod } from '../../../../shared' - -const remotePodsRouter = express.Router() - -remotePodsRouter.post('/remove', - signatureValidator, - checkSignature, - asyncMiddleware(removePods) -) - -remotePodsRouter.post('/list', - asyncMiddleware(remotePodsList) -) - -remotePodsRouter.post('/add', - setBodyHostPort, // We need to modify the host before running the validator! - remotePodsAddValidator, - asyncMiddleware(addPods) -) - -// --------------------------------------------------------------------------- - -export { - remotePodsRouter -} - -// --------------------------------------------------------------------------- - -async function addPods (req: express.Request, res: express.Response, next: express.NextFunction) { - const information = req.body - - const pod = db.Pod.build(information) - const podCreated = await pod.save() - - await sendOwnedDataToPod(podCreated.id) - - const cert = await getMyPublicCert() - return res.json({ cert, email: CONFIG.ADMIN.EMAIL }) -} - -async function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) { - const pods = await db.Pod.list() - - return res.json(getFormattedObjects(pods, pods.length)) -} - -async function removePods (req: express.Request, res: express.Response, next: express.NextFunction) { - const signature: PodSignature = req.body.signature - const host = signature.host - - const pod = await db.Pod.loadByHost(host) - await pod.destroy() - - return res.type('json').status(204).end() -} diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts deleted file mode 100644 index cba47f0a1..000000000 --- a/server/controllers/api/remote/videos.ts +++ /dev/null @@ -1,589 +0,0 @@ -import * as express from 'express' -import * as Bluebird from 'bluebird' -import * as Sequelize from 'sequelize' - -import { database as db } from '../../../initializers/database' -import { - REQUEST_ENDPOINT_ACTIONS, - REQUEST_ENDPOINTS, - REQUEST_VIDEO_EVENT_TYPES, - REQUEST_VIDEO_QADU_TYPES -} from '../../../initializers' -import { - checkSignature, - signatureValidator, - remoteVideosValidator, - remoteQaduVideosValidator, - remoteEventsVideosValidator -} from '../../../middlewares' -import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers' -import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib' -import { PodInstance, VideoFileInstance } from '../../../models' -import { - RemoteVideoRequest, - RemoteVideoCreateData, - RemoteVideoUpdateData, - RemoteVideoRemoveData, - RemoteVideoReportAbuseData, - RemoteQaduVideoRequest, - RemoteQaduVideoData, - RemoteVideoEventRequest, - RemoteVideoEventData, - RemoteVideoChannelCreateData, - RemoteVideoChannelUpdateData, - RemoteVideoChannelRemoveData, - RemoteVideoAuthorRemoveData, - RemoteVideoAuthorCreateData -} from '../../../../shared' -import { VideoInstance } from '../../../models/video/video-interface' - -const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] - -// Functions to call when processing a remote request -// FIXME: use RemoteVideoRequestType as id type -const functionsHash: { [ id: string ]: (...args) => Promise } = {} -functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper -functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper - -const remoteVideosRouter = express.Router() - -remoteVideosRouter.post('/', - signatureValidator, - checkSignature, - remoteVideosValidator, - remoteVideos -) - -remoteVideosRouter.post('/qadu', - signatureValidator, - checkSignature, - remoteQaduVideosValidator, - remoteVideosQadu -) - -remoteVideosRouter.post('/events', - signatureValidator, - checkSignature, - remoteEventsVideosValidator, - remoteVideosEvents -) - -// --------------------------------------------------------------------------- - -export { - remoteVideosRouter -} - -// --------------------------------------------------------------------------- - -function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) { - const requests: RemoteVideoRequest[] = req.body.data - const fromPod = res.locals.secure.pod - - // We need to process in the same order to keep consistency - Bluebird.each(requests, request => { - const data = request.data - - // Get the function we need to call in order to process the request - const fun = functionsHash[request.type] - if (fun === undefined) { - logger.error('Unknown remote request type %s.', request.type) - return - } - - return fun.call(this, data, fromPod) - }) - .catch(err => logger.error('Error managing remote videos.', err)) - - // Don't block the other pod - return res.type('json').status(204).end() -} - -function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) { - const requests: RemoteQaduVideoRequest[] = req.body.data - const fromPod = res.locals.secure.pod - - Bluebird.each(requests, request => { - const videoData = request.data - - return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod) - }) - .catch(err => logger.error('Error managing remote videos.', err)) - - return res.type('json').status(204).end() -} - -function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) { - const requests: RemoteVideoEventRequest[] = req.body.data - const fromPod = res.locals.secure.pod - - Bluebird.each(requests, request => { - const eventData = request.data - - return processVideosEventsRetryWrapper(eventData, fromPod) - }) - .catch(err => logger.error('Error managing remote videos.', err)) - - return res.type('json').status(204).end() -} - -async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) { - const options = { - arguments: [ eventData, fromPod ], - errorMessage: 'Cannot process videos events with many retries.' - } - - await retryTransactionWrapper(processVideosEvents, options) -} - -async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) { - await db.sequelize.transaction(async t => { - const sequelizeOptions = { transaction: t } - const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t) - - let columnToUpdate - let qaduType - - switch (eventData.eventType) { - case REQUEST_VIDEO_EVENT_TYPES.VIEWS: - columnToUpdate = 'views' - qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS - break - - case REQUEST_VIDEO_EVENT_TYPES.LIKES: - columnToUpdate = 'likes' - qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES - break - - case REQUEST_VIDEO_EVENT_TYPES.DISLIKES: - columnToUpdate = 'dislikes' - qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES - break - - default: - throw new Error('Unknown video event type.') - } - - const query = {} - query[columnToUpdate] = eventData.count - - await videoInstance.increment(query, sequelizeOptions) - - const qadusParams = [ - { - videoId: videoInstance.id, - type: qaduType - } - ] - await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) - }) - - logger.info('Remote video event processed for video with uuid %s.', eventData.uuid) -} - -async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) { - const options = { - arguments: [ videoData, fromPod ], - errorMessage: 'Cannot update quick and dirty the remote video with many retries.' - } - - await retryTransactionWrapper(quickAndDirtyUpdateVideo, options) -} - -async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { - let videoUUID = '' - - await db.sequelize.transaction(async t => { - const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t) - const sequelizeOptions = { transaction: t } - - videoUUID = videoInstance.uuid - - if (videoData.views) { - videoInstance.set('views', videoData.views) - } - - if (videoData.likes) { - videoInstance.set('likes', videoData.likes) - } - - if (videoData.dislikes) { - videoInstance.set('dislikes', videoData.dislikes) - } - - await videoInstance.save(sequelizeOptions) - }) - - logger.info('Remote video with uuid %s quick and dirty updated', videoUUID) -} - -// Handle retries on fail -async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { - const options = { - arguments: [ videoToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video with many retries.' - } - - await retryTransactionWrapper(addRemoteVideo, options) -} - -async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { - logger.debug('Adding remote video "%s".', videoToCreateData.uuid) - - await db.sequelize.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) - if (videoFromDatabase) throw new Error('UUID already exists.') - - const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) - if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') - - const tags = videoToCreateData.tags - const tagInstances = await db.Tag.findOrCreateTags(tags, t) - - const videoData = { - name: videoToCreateData.name, - uuid: videoToCreateData.uuid, - category: videoToCreateData.category, - licence: videoToCreateData.licence, - language: videoToCreateData.language, - nsfw: videoToCreateData.nsfw, - description: videoToCreateData.truncatedDescription, - channelId: videoChannel.id, - duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt, - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: videoToCreateData.updatedAt, - views: videoToCreateData.views, - likes: videoToCreateData.likes, - dislikes: videoToCreateData.dislikes, - remote: true, - privacy: videoToCreateData.privacy - } - - const video = db.Video.build(videoData) - await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) - const videoCreated = await video.save(sequelizeOptions) - - const tasks = [] - for (const fileData of videoToCreateData.files) { - const videoFileInstance = db.VideoFile.build({ - extname: fileData.extname, - infoHash: fileData.infoHash, - resolution: fileData.resolution, - size: fileData.size, - videoId: videoCreated.id - }) - - tasks.push(videoFileInstance.save(sequelizeOptions)) - } - - await Promise.all(tasks) - - await videoCreated.setTags(tagInstances, sequelizeOptions) - }) - - logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) -} - -// Handle retries on fail -async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { - const options = { - arguments: [ videoAttributesToUpdate, fromPod ], - errorMessage: 'Cannot update the remote video with many retries' - } - - await retryTransactionWrapper(updateRemoteVideo, options) -} - -async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) - let videoInstance: VideoInstance - let videoFieldsSave: object - - try { - await db.sequelize.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t) - videoFieldsSave = videoInstance.toJSON() - const tags = videoAttributesToUpdate.tags - - const tagInstances = await db.Tag.findOrCreateTags(tags, t) - - videoInstance.set('name', videoAttributesToUpdate.name) - videoInstance.set('category', videoAttributesToUpdate.category) - videoInstance.set('licence', videoAttributesToUpdate.licence) - videoInstance.set('language', videoAttributesToUpdate.language) - videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) - videoInstance.set('description', videoAttributesToUpdate.truncatedDescription) - videoInstance.set('duration', videoAttributesToUpdate.duration) - videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) - videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) - videoInstance.set('views', videoAttributesToUpdate.views) - videoInstance.set('likes', videoAttributesToUpdate.likes) - videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) - videoInstance.set('privacy', videoAttributesToUpdate.privacy) - - await videoInstance.save(sequelizeOptions) - - // Remove old video files - const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of videoInstance.VideoFiles) { - videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) - } - await Promise.all(videoFileDestroyTasks) - - const videoFileCreateTasks: Bluebird[] = [] - for (const fileData of videoAttributesToUpdate.files) { - const videoFileInstance = db.VideoFile.build({ - extname: fileData.extname, - infoHash: fileData.infoHash, - resolution: fileData.resolution, - size: fileData.size, - videoId: videoInstance.id - }) - - videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions)) - } - - await Promise.all(videoFileCreateTasks) - - await videoInstance.setTags(tagInstances, sequelizeOptions) - }) - - logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid) - } catch (err) { - if (videoInstance !== undefined && videoFieldsSave !== undefined) { - resetSequelizeInstance(videoInstance, videoFieldsSave) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', err) - throw err - } -} - -async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { - const options = { - arguments: [ videoToRemoveData, fromPod ], - errorMessage: 'Cannot remove the remote video channel with many retries.' - } - - await retryTransactionWrapper(removeRemoteVideo, options) -} - -async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { - logger.debug('Removing remote video "%s".', videoToRemoveData.uuid) - - await db.sequelize.transaction(async t => { - // We need the instance because we have to remove some other stuffs (thumbnail etc) - const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t) - await videoInstance.destroy({ transaction: t }) - }) - - logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid) -} - -async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) { - const options = { - arguments: [ authorToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video author with many retries.' - } - - await retryTransactionWrapper(addRemoteVideoAuthor, options) -} - -async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) { - logger.debug('Adding remote video author "%s".', authorToCreateData.uuid) - - await db.sequelize.transaction(async t => { - const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t) - if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.') - - const videoAuthorData = { - name: authorToCreateData.name, - uuid: authorToCreateData.uuid, - userId: null, // Not on our pod - podId: fromPod.id - } - - const author = db.Author.build(videoAuthorData) - await author.save({ transaction: t }) - }) - - logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid) -} - -async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { - const options = { - arguments: [ authorAttributesToRemove, fromPod ], - errorMessage: 'Cannot remove the remote video author with many retries.' - } - - await retryTransactionWrapper(removeRemoteVideoAuthor, options) -} - -async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { - logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid) - - await db.sequelize.transaction(async t => { - const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t) - await videoAuthor.destroy({ transaction: t }) - }) - - logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid) -} - -async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) { - const options = { - arguments: [ videoChannelToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video channel with many retries.' - } - - await retryTransactionWrapper(addRemoteVideoChannel, options) -} - -async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) { - logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid) - - await db.sequelize.transaction(async t => { - const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid) - if (videoChannelInDatabase) { - throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.') - } - - const authorUUID = videoChannelToCreateData.ownerUUID - const podId = fromPod.id - - const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t) - if (!author) throw new Error('Unknown author UUID' + authorUUID + '.') - - const videoChannelData = { - name: videoChannelToCreateData.name, - description: videoChannelToCreateData.description, - uuid: videoChannelToCreateData.uuid, - createdAt: videoChannelToCreateData.createdAt, - updatedAt: videoChannelToCreateData.updatedAt, - remote: true, - authorId: author.id - } - - const videoChannel = db.VideoChannel.build(videoChannelData) - await videoChannel.save({ transaction: t }) - }) - - logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid) -} - -async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) { - const options = { - arguments: [ videoChannelAttributesToUpdate, fromPod ], - errorMessage: 'Cannot update the remote video channel with many retries.' - } - - await retryTransactionWrapper(updateRemoteVideoChannel, options) -} - -async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) { - logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid) - - await db.sequelize.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t) - videoChannelInstance.set('name', videoChannelAttributesToUpdate.name) - videoChannelInstance.set('description', videoChannelAttributesToUpdate.description) - videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt) - videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt) - - await videoChannelInstance.save(sequelizeOptions) - }) - - logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid) -} - -async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { - const options = { - arguments: [ videoChannelAttributesToRemove, fromPod ], - errorMessage: 'Cannot remove the remote video channel with many retries.' - } - - await retryTransactionWrapper(removeRemoteVideoChannel, options) -} - -async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { - logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid) - - await db.sequelize.transaction(async t => { - const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t) - await videoChannel.destroy({ transaction: t }) - }) - - logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid) -} - -async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { - const options = { - arguments: [ reportData, fromPod ], - errorMessage: 'Cannot create remote abuse video with many retries.' - } - - await retryTransactionWrapper(reportAbuseRemoteVideo, options) -} - -async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { - logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID) - - await db.sequelize.transaction(async t => { - const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t) - const videoAbuseData = { - reporterUsername: reportData.reporterUsername, - reason: reportData.reportReason, - reporterPodId: fromPod.id, - videoId: videoInstance.id - } - - await db.VideoAbuse.create(videoAbuseData) - - }) - - logger.info('Remote abuse for video uuid %s created', reportData.videoUUID) -} - -async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) { - try { - const video = await db.Video.loadLocalVideoByUUID(id, t) - - if (!video) throw new Error('Video ' + id + ' not found') - - return video - } catch (err) { - logger.error('Cannot load owned video from id.', { error: err.stack, id }) - throw err - } -} - -async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) { - try { - const video = await db.Video.loadByHostAndUUID(podHost, uuid, t) - if (!video) throw new Error('Video not found') - - return video - } catch (err) { - logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid }) - throw err - } -} diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts new file mode 100644 index 000000000..ecb509b66 --- /dev/null +++ b/server/helpers/activitypub.ts @@ -0,0 +1,123 @@ +import * as url from 'url' + +import { database as db } from '../initializers' +import { logger } from './logger' +import { doRequest } from './requests' +import { isRemoteAccountValid } from './custom-validators' +import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' +import { ResultList } from '../../shared/models/result-list.model' + +async function fetchRemoteAccountAndCreatePod (accountUrl: string) { + const options = { + uri: accountUrl, + method: 'GET' + } + + let requestResult + try { + requestResult = await doRequest(options) + } catch (err) { + logger.warning('Cannot fetch remote account %s.', accountUrl, err) + return undefined + } + + const accountJSON: ActivityPubActor = requestResult.body + if (isRemoteAccountValid(accountJSON) === false) return undefined + + const followersCount = await fetchAccountCount(accountJSON.followers) + const followingCount = await fetchAccountCount(accountJSON.following) + + const account = db.Account.build({ + uuid: accountJSON.uuid, + name: accountJSON.preferredUsername, + url: accountJSON.url, + publicKey: accountJSON.publicKey.publicKeyPem, + privateKey: null, + followersCount: followersCount, + followingCount: followingCount, + inboxUrl: accountJSON.inbox, + outboxUrl: accountJSON.outbox, + sharedInboxUrl: accountJSON.endpoints.sharedInbox, + followersUrl: accountJSON.followers, + followingUrl: accountJSON.following + }) + + const accountHost = url.parse(account.url).host + const podOptions = { + where: { + host: accountHost + }, + defaults: { + host: accountHost + } + } + const pod = await db.Pod.findOrCreate(podOptions) + + return { account, pod } +} + +function activityPubContextify (data: object) { + return Object.assign(data,{ + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + { + 'Hashtag': 'as:Hashtag', + 'uuid': 'http://schema.org/identifier', + 'category': 'http://schema.org/category', + 'licence': 'http://schema.org/license', + 'nsfw': 'as:sensitive', + 'language': 'http://schema.org/inLanguage', + 'views': 'http://schema.org/Number', + 'size': 'http://schema.org/Number' + } + ] + }) +} + +function activityPubCollectionPagination (url: string, page: number, result: ResultList) { + const baseUrl = url.split('?').shift + + const obj = { + id: baseUrl, + type: 'Collection', + totalItems: result.total, + first: { + id: baseUrl + '?page=' + page, + type: 'CollectionPage', + totalItems: result.total, + next: baseUrl + '?page=' + (page + 1), + partOf: baseUrl, + items: result.data + } + } + + return activityPubContextify(obj) +} + +// --------------------------------------------------------------------------- + +export { + fetchRemoteAccountAndCreatePod, + activityPubContextify, + activityPubCollectionPagination +} + +// --------------------------------------------------------------------------- + +async function fetchAccountCount (url: string) { + const options = { + uri: url, + method: 'GET' + } + + let requestResult + try { + requestResult = await doRequest(options) + } catch (err) { + logger.warning('Cannot fetch remote account count %s.', url, err) + return undefined + } + + return requestResult.totalItems ? requestResult.totalItems : 0 +} diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 3dae78144..d8748e1d7 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -19,8 +19,10 @@ import * as mkdirp from 'mkdirp' import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' import * as rimraf from 'rimraf' -import * as openssl from 'openssl-wrapper' -import * as Promise from 'bluebird' +import * as pem from 'pem' +import * as jsonld from 'jsonld' +import * as jsig from 'jsonld-signatures' +jsig.use('jsonld', jsonld) function isTestInstance () { return process.env.NODE_ENV === 'test' @@ -54,6 +56,12 @@ function escapeHTML (stringParam) { return String(stringParam).replace(/[&<>"'`=\/]/g, s => entityMap[s]) } +function pageToStartAndCount (page: number, itemsPerPage: number) { + const start = (page - 1) * itemsPerPage + + return { start, count: itemsPerPage } +} + function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { return function promisified (): Promise { return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { @@ -104,13 +112,16 @@ const readdirPromise = promisify1(readdir) const mkdirpPromise = promisify1(mkdirp) const pseudoRandomBytesPromise = promisify1(pseudoRandomBytes) const accessPromise = promisify1WithVoid(access) -const opensslExecPromise = promisify2WithVoid(openssl.exec) +const createPrivateKey = promisify1(pem.createPrivateKey) +const getPublicKey = promisify1(pem.getPublicKey) const bcryptComparePromise = promisify2(bcrypt.compare) const bcryptGenSaltPromise = promisify1(bcrypt.genSalt) const bcryptHashPromise = promisify2(bcrypt.hash) const createTorrentPromise = promisify2(createTorrent) const rimrafPromise = promisify1WithVoid(rimraf) const statPromise = promisify1(stat) +const jsonldSignPromise = promisify2(jsig.sign) +const jsonldVerifyPromise = promisify2(jsig.verify) // --------------------------------------------------------------------------- @@ -118,9 +129,11 @@ export { isTestInstance, root, escapeHTML, + pageToStartAndCount, promisify0, promisify1, + readdirPromise, readFilePromise, readFileBufferPromise, @@ -130,11 +143,14 @@ export { mkdirpPromise, pseudoRandomBytesPromise, accessPromise, - opensslExecPromise, + createPrivateKey, + getPublicKey, bcryptComparePromise, bcryptGenSaltPromise, bcryptHashPromise, createTorrentPromise, rimrafPromise, - statPromise + statPromise, + jsonldSignPromise, + jsonldVerifyPromise } diff --git a/server/helpers/custom-validators/activitypub/account.ts b/server/helpers/custom-validators/activitypub/account.ts new file mode 100644 index 000000000..8a7d1b7fe --- /dev/null +++ b/server/helpers/custom-validators/activitypub/account.ts @@ -0,0 +1,123 @@ +import * as validator from 'validator' + +import { exists, isUUIDValid } from '../misc' +import { isActivityPubUrlValid } from './misc' +import { isUserUsernameValid } from '../users' + +function isAccountEndpointsObjectValid (endpointObject: any) { + return isAccountSharedInboxValid(endpointObject.sharedInbox) +} + +function isAccountSharedInboxValid (sharedInbox: string) { + return isActivityPubUrlValid(sharedInbox) +} + +function isAccountPublicKeyObjectValid (publicKeyObject: any) { + return isAccountPublicKeyIdValid(publicKeyObject.id) && + isAccountPublicKeyOwnerValid(publicKeyObject.owner) && + isAccountPublicKeyValid(publicKeyObject.publicKeyPem) +} + +function isAccountPublicKeyIdValid (id: string) { + return isActivityPubUrlValid(id) +} + +function isAccountTypeValid (type: string) { + return type === 'Person' || type === 'Application' +} + +function isAccountPublicKeyOwnerValid (owner: string) { + return isActivityPubUrlValid(owner) +} + +function isAccountPublicKeyValid (publicKey: string) { + return exists(publicKey) && + typeof publicKey === 'string' && + publicKey.startsWith('-----BEGIN PUBLIC KEY-----') && + publicKey.endsWith('-----END PUBLIC KEY-----') +} + +function isAccountIdValid (id: string) { + return isActivityPubUrlValid(id) +} + +function isAccountFollowingValid (id: string) { + return isActivityPubUrlValid(id) +} + +function isAccountFollowersValid (id: string) { + return isActivityPubUrlValid(id) +} + +function isAccountInboxValid (inbox: string) { + return isActivityPubUrlValid(inbox) +} + +function isAccountOutboxValid (outbox: string) { + return isActivityPubUrlValid(outbox) +} + +function isAccountNameValid (name: string) { + return isUserUsernameValid(name) +} + +function isAccountPreferredUsernameValid (preferredUsername: string) { + return isAccountNameValid(preferredUsername) +} + +function isAccountUrlValid (url: string) { + return isActivityPubUrlValid(url) +} + +function isAccountPrivateKeyValid (privateKey: string) { + return exists(privateKey) && + typeof privateKey === 'string' && + privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') && + privateKey.endsWith('-----END RSA PRIVATE KEY-----') +} + +function isRemoteAccountValid (remoteAccount: any) { + return isAccountIdValid(remoteAccount.id) && + isUUIDValid(remoteAccount.uuid) && + isAccountTypeValid(remoteAccount.type) && + isAccountFollowingValid(remoteAccount.following) && + isAccountFollowersValid(remoteAccount.followers) && + isAccountInboxValid(remoteAccount.inbox) && + isAccountOutboxValid(remoteAccount.outbox) && + isAccountPreferredUsernameValid(remoteAccount.preferredUsername) && + isAccountUrlValid(remoteAccount.url) && + isAccountPublicKeyObjectValid(remoteAccount.publicKey) && + isAccountEndpointsObjectValid(remoteAccount.endpoint) +} + +function isAccountFollowingCountValid (value: string) { + return exists(value) && validator.isInt('' + value, { min: 0 }) +} + +function isAccountFollowersCountValid (value: string) { + return exists(value) && validator.isInt('' + value, { min: 0 }) +} + +// --------------------------------------------------------------------------- + +export { + isAccountEndpointsObjectValid, + isAccountSharedInboxValid, + isAccountPublicKeyObjectValid, + isAccountPublicKeyIdValid, + isAccountTypeValid, + isAccountPublicKeyOwnerValid, + isAccountPublicKeyValid, + isAccountIdValid, + isAccountFollowingValid, + isAccountFollowersValid, + isAccountInboxValid, + isAccountOutboxValid, + isAccountPreferredUsernameValid, + isAccountUrlValid, + isAccountPrivateKeyValid, + isRemoteAccountValid, + isAccountFollowingCountValid, + isAccountFollowersCountValid, + isAccountNameValid +} diff --git a/server/helpers/custom-validators/activitypub/index.ts b/server/helpers/custom-validators/activitypub/index.ts new file mode 100644 index 000000000..800f0ddf3 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/index.ts @@ -0,0 +1,4 @@ +export * from './account' +export * from './signature' +export * from './misc' +export * from './videos' diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts new file mode 100644 index 000000000..806d33483 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -0,0 +1,17 @@ +import { exists } from '../misc' + +function isActivityPubUrlValid (url: string) { + const isURLOptions = { + require_host: true, + require_tld: true, + require_protocol: true, + require_valid_protocol: true, + protocols: [ 'http', 'https' ] + } + + return exists(url) && validator.isURL(url, isURLOptions) +} + +export { + isActivityPubUrlValid +} diff --git a/server/helpers/custom-validators/activitypub/signature.ts b/server/helpers/custom-validators/activitypub/signature.ts new file mode 100644 index 000000000..683ed2b1c --- /dev/null +++ b/server/helpers/custom-validators/activitypub/signature.ts @@ -0,0 +1,22 @@ +import { exists } from '../misc' +import { isActivityPubUrlValid } from './misc' + +function isSignatureTypeValid (signatureType: string) { + return exists(signatureType) && signatureType === 'GraphSignature2012' +} + +function isSignatureCreatorValid (signatureCreator: string) { + return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator) +} + +function isSignatureValueValid (signatureValue: string) { + return exists(signatureValue) && signatureValue.length > 0 +} + +// --------------------------------------------------------------------------- + +export { + isSignatureTypeValid, + isSignatureCreatorValid, + isSignatureValueValid +} diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts new file mode 100644 index 000000000..e0ffba679 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -0,0 +1,184 @@ +import 'express-validator' +import { has, values } from 'lodash' + +import { + REQUEST_ENDPOINTS, + REQUEST_ENDPOINT_ACTIONS, + REQUEST_VIDEO_EVENT_TYPES +} from '../../../initializers' +import { isArray, isDateValid, isUUIDValid } from '../misc' +import { + isVideoThumbnailDataValid, + isVideoAbuseReasonValid, + isVideoAbuseReporterUsernameValid, + isVideoViewsValid, + isVideoLikesValid, + isVideoDislikesValid, + isVideoEventCountValid, + isRemoteVideoCategoryValid, + isRemoteVideoLicenceValid, + isRemoteVideoLanguageValid, + isVideoNSFWValid, + isVideoTruncatedDescriptionValid, + isVideoDurationValid, + isVideoFileInfoHashValid, + isVideoNameValid, + isVideoTagsValid, + isVideoFileExtnameValid, + isVideoFileResolutionValid +} from '../videos' +import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../video-channels' +import { isVideoAuthorNameValid } from '../video-authors' + +const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] + +const checkers: { [ id: string ]: (obj: any) => boolean } = {} +checkers[ENDPOINT_ACTIONS.ADD_VIDEO] = checkAddVideo +checkers[ENDPOINT_ACTIONS.UPDATE_VIDEO] = checkUpdateVideo +checkers[ENDPOINT_ACTIONS.REMOVE_VIDEO] = checkRemoveVideo +checkers[ENDPOINT_ACTIONS.REPORT_ABUSE] = checkReportVideo +checkers[ENDPOINT_ACTIONS.ADD_CHANNEL] = checkAddVideoChannel +checkers[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = checkUpdateVideoChannel +checkers[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = checkRemoveVideoChannel +checkers[ENDPOINT_ACTIONS.ADD_AUTHOR] = checkAddAuthor +checkers[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = checkRemoveAuthor + +function removeBadRequestVideos (requests: any[]) { + for (let i = requests.length - 1; i >= 0 ; i--) { + const request = requests[i] + const video = request.data + + if ( + !video || + checkers[request.type] === undefined || + checkers[request.type](video) === false + ) { + requests.splice(i, 1) + } + } +} + +function removeBadRequestVideosQadu (requests: any[]) { + for (let i = requests.length - 1; i >= 0 ; i--) { + const request = requests[i] + const video = request.data + + if ( + !video || + ( + isUUIDValid(video.uuid) && + (has(video, 'views') === false || isVideoViewsValid(video.views)) && + (has(video, 'likes') === false || isVideoLikesValid(video.likes)) && + (has(video, 'dislikes') === false || isVideoDislikesValid(video.dislikes)) + ) === false + ) { + requests.splice(i, 1) + } + } +} + +function removeBadRequestVideosEvents (requests: any[]) { + for (let i = requests.length - 1; i >= 0 ; i--) { + const request = requests[i] + const eventData = request.data + + if ( + !eventData || + ( + isUUIDValid(eventData.uuid) && + values(REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && + isVideoEventCountValid(eventData.count) + ) === false + ) { + requests.splice(i, 1) + } + } +} + +// --------------------------------------------------------------------------- + +export { + removeBadRequestVideos, + removeBadRequestVideosQadu, + removeBadRequestVideosEvents +} + +// --------------------------------------------------------------------------- + +function isCommonVideoAttributesValid (video: any) { + return isDateValid(video.createdAt) && + isDateValid(video.updatedAt) && + isRemoteVideoCategoryValid(video.category) && + isRemoteVideoLicenceValid(video.licence) && + isRemoteVideoLanguageValid(video.language) && + isVideoNSFWValid(video.nsfw) && + isVideoTruncatedDescriptionValid(video.truncatedDescription) && + isVideoDurationValid(video.duration) && + isVideoNameValid(video.name) && + isVideoTagsValid(video.tags) && + isUUIDValid(video.uuid) && + isVideoViewsValid(video.views) && + isVideoLikesValid(video.likes) && + isVideoDislikesValid(video.dislikes) && + isArray(video.files) && + video.files.every(videoFile => { + if (!videoFile) return false + + return ( + isVideoFileInfoHashValid(videoFile.infoHash) && + isVideoFileExtnameValid(videoFile.extname) && + isVideoFileResolutionValid(videoFile.resolution) + ) + }) +} + +function checkAddVideo (video: any) { + return isCommonVideoAttributesValid(video) && + isUUIDValid(video.channelUUID) && + isVideoThumbnailDataValid(video.thumbnailData) +} + +function checkUpdateVideo (video: any) { + return isCommonVideoAttributesValid(video) +} + +function checkRemoveVideo (video: any) { + return isUUIDValid(video.uuid) +} + +function checkReportVideo (abuse: any) { + return isUUIDValid(abuse.videoUUID) && + isVideoAbuseReasonValid(abuse.reportReason) && + isVideoAbuseReporterUsernameValid(abuse.reporterUsername) +} + +function checkAddVideoChannel (videoChannel: any) { + return isUUIDValid(videoChannel.uuid) && + isVideoChannelNameValid(videoChannel.name) && + isVideoChannelDescriptionValid(videoChannel.description) && + isDateValid(videoChannel.createdAt) && + isDateValid(videoChannel.updatedAt) && + isUUIDValid(videoChannel.ownerUUID) +} + +function checkUpdateVideoChannel (videoChannel: any) { + return isUUIDValid(videoChannel.uuid) && + isVideoChannelNameValid(videoChannel.name) && + isVideoChannelDescriptionValid(videoChannel.description) && + isDateValid(videoChannel.createdAt) && + isDateValid(videoChannel.updatedAt) && + isUUIDValid(videoChannel.ownerUUID) +} + +function checkRemoveVideoChannel (videoChannel: any) { + return isUUIDValid(videoChannel.uuid) +} + +function checkAddAuthor (author: any) { + return isUUIDValid(author.uuid) && + isVideoAuthorNameValid(author.name) +} + +function checkRemoveAuthor (author: any) { + return isUUIDValid(author.uuid) +} diff --git a/server/helpers/custom-validators/index.ts b/server/helpers/custom-validators/index.ts index c79982660..869b08870 100644 --- a/server/helpers/custom-validators/index.ts +++ b/server/helpers/custom-validators/index.ts @@ -1,4 +1,4 @@ -export * from './remote' +export * from './activitypub' export * from './misc' export * from './pods' export * from './pods' diff --git a/server/helpers/custom-validators/remote/index.ts b/server/helpers/custom-validators/remote/index.ts deleted file mode 100644 index e29a9b767..000000000 --- a/server/helpers/custom-validators/remote/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './videos' diff --git a/server/helpers/custom-validators/remote/videos.ts b/server/helpers/custom-validators/remote/videos.ts deleted file mode 100644 index e0ffba679..000000000 --- a/server/helpers/custom-validators/remote/videos.ts +++ /dev/null @@ -1,184 +0,0 @@ -import 'express-validator' -import { has, values } from 'lodash' - -import { - REQUEST_ENDPOINTS, - REQUEST_ENDPOINT_ACTIONS, - REQUEST_VIDEO_EVENT_TYPES -} from '../../../initializers' -import { isArray, isDateValid, isUUIDValid } from '../misc' -import { - isVideoThumbnailDataValid, - isVideoAbuseReasonValid, - isVideoAbuseReporterUsernameValid, - isVideoViewsValid, - isVideoLikesValid, - isVideoDislikesValid, - isVideoEventCountValid, - isRemoteVideoCategoryValid, - isRemoteVideoLicenceValid, - isRemoteVideoLanguageValid, - isVideoNSFWValid, - isVideoTruncatedDescriptionValid, - isVideoDurationValid, - isVideoFileInfoHashValid, - isVideoNameValid, - isVideoTagsValid, - isVideoFileExtnameValid, - isVideoFileResolutionValid -} from '../videos' -import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../video-channels' -import { isVideoAuthorNameValid } from '../video-authors' - -const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] - -const checkers: { [ id: string ]: (obj: any) => boolean } = {} -checkers[ENDPOINT_ACTIONS.ADD_VIDEO] = checkAddVideo -checkers[ENDPOINT_ACTIONS.UPDATE_VIDEO] = checkUpdateVideo -checkers[ENDPOINT_ACTIONS.REMOVE_VIDEO] = checkRemoveVideo -checkers[ENDPOINT_ACTIONS.REPORT_ABUSE] = checkReportVideo -checkers[ENDPOINT_ACTIONS.ADD_CHANNEL] = checkAddVideoChannel -checkers[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = checkUpdateVideoChannel -checkers[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = checkRemoveVideoChannel -checkers[ENDPOINT_ACTIONS.ADD_AUTHOR] = checkAddAuthor -checkers[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = checkRemoveAuthor - -function removeBadRequestVideos (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const video = request.data - - if ( - !video || - checkers[request.type] === undefined || - checkers[request.type](video) === false - ) { - requests.splice(i, 1) - } - } -} - -function removeBadRequestVideosQadu (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const video = request.data - - if ( - !video || - ( - isUUIDValid(video.uuid) && - (has(video, 'views') === false || isVideoViewsValid(video.views)) && - (has(video, 'likes') === false || isVideoLikesValid(video.likes)) && - (has(video, 'dislikes') === false || isVideoDislikesValid(video.dislikes)) - ) === false - ) { - requests.splice(i, 1) - } - } -} - -function removeBadRequestVideosEvents (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const eventData = request.data - - if ( - !eventData || - ( - isUUIDValid(eventData.uuid) && - values(REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && - isVideoEventCountValid(eventData.count) - ) === false - ) { - requests.splice(i, 1) - } - } -} - -// --------------------------------------------------------------------------- - -export { - removeBadRequestVideos, - removeBadRequestVideosQadu, - removeBadRequestVideosEvents -} - -// --------------------------------------------------------------------------- - -function isCommonVideoAttributesValid (video: any) { - return isDateValid(video.createdAt) && - isDateValid(video.updatedAt) && - isRemoteVideoCategoryValid(video.category) && - isRemoteVideoLicenceValid(video.licence) && - isRemoteVideoLanguageValid(video.language) && - isVideoNSFWValid(video.nsfw) && - isVideoTruncatedDescriptionValid(video.truncatedDescription) && - isVideoDurationValid(video.duration) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isUUIDValid(video.uuid) && - isVideoViewsValid(video.views) && - isVideoLikesValid(video.likes) && - isVideoDislikesValid(video.dislikes) && - isArray(video.files) && - video.files.every(videoFile => { - if (!videoFile) return false - - return ( - isVideoFileInfoHashValid(videoFile.infoHash) && - isVideoFileExtnameValid(videoFile.extname) && - isVideoFileResolutionValid(videoFile.resolution) - ) - }) -} - -function checkAddVideo (video: any) { - return isCommonVideoAttributesValid(video) && - isUUIDValid(video.channelUUID) && - isVideoThumbnailDataValid(video.thumbnailData) -} - -function checkUpdateVideo (video: any) { - return isCommonVideoAttributesValid(video) -} - -function checkRemoveVideo (video: any) { - return isUUIDValid(video.uuid) -} - -function checkReportVideo (abuse: any) { - return isUUIDValid(abuse.videoUUID) && - isVideoAbuseReasonValid(abuse.reportReason) && - isVideoAbuseReporterUsernameValid(abuse.reporterUsername) -} - -function checkAddVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) && - isVideoChannelNameValid(videoChannel.name) && - isVideoChannelDescriptionValid(videoChannel.description) && - isDateValid(videoChannel.createdAt) && - isDateValid(videoChannel.updatedAt) && - isUUIDValid(videoChannel.ownerUUID) -} - -function checkUpdateVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) && - isVideoChannelNameValid(videoChannel.name) && - isVideoChannelDescriptionValid(videoChannel.description) && - isDateValid(videoChannel.createdAt) && - isDateValid(videoChannel.updatedAt) && - isUUIDValid(videoChannel.ownerUUID) -} - -function checkRemoveVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) -} - -function checkAddAuthor (author: any) { - return isUUIDValid(author.uuid) && - isVideoAuthorNameValid(author.name) -} - -function checkRemoveAuthor (author: any) { - return isUUIDValid(author.uuid) -} diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index f18b6bd9a..c07dddefe 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -1,4 +1,3 @@ -import * as Promise from 'bluebird' import * as ffmpeg from 'fluent-ffmpeg' import { CONFIG } from '../initializers' diff --git a/server/helpers/index.ts b/server/helpers/index.ts index 846bd796f..2c7ac3954 100644 --- a/server/helpers/index.ts +++ b/server/helpers/index.ts @@ -1,3 +1,4 @@ +export * from './activitypub' export * from './core-utils' export * from './logger' export * from './custom-validators' @@ -6,3 +7,4 @@ export * from './database-utils' export * from './peertube-crypto' export * from './requests' export * from './utils' +export * from './webfinger' diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 10a226af4..6d50e446f 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts @@ -1,77 +1,68 @@ -import * as crypto from 'crypto' -import { join } from 'path' +import * as jsig from 'jsonld-signatures' import { - SIGNATURE_ALGORITHM, - SIGNATURE_ENCODING, - PRIVATE_CERT_NAME, - CONFIG, - BCRYPT_SALT_SIZE, - PUBLIC_CERT_NAME + PRIVATE_RSA_KEY_SIZE, + BCRYPT_SALT_SIZE } from '../initializers' import { - readFilePromise, bcryptComparePromise, bcryptGenSaltPromise, bcryptHashPromise, - accessPromise, - opensslExecPromise + createPrivateKey, + getPublicKey, + jsonldSignPromise, + jsonldVerifyPromise } from './core-utils' import { logger } from './logger' +import { AccountInstance } from '../models/account/account-interface' -function checkSignature (publicKey: string, data: string, hexSignature: string) { - const verify = crypto.createVerify(SIGNATURE_ALGORITHM) - - let dataString - if (typeof data === 'string') { - dataString = data - } else { - try { - dataString = JSON.stringify(data) - } catch (err) { - logger.error('Cannot check signature.', err) - return false - } - } +async function createPrivateAndPublicKeys () { + logger.info('Generating a RSA key...') - verify.update(dataString, 'utf8') + const { key } = await createPrivateKey(PRIVATE_RSA_KEY_SIZE) + const { publicKey } = await getPublicKey(key) - const isValid = verify.verify(publicKey, hexSignature, SIGNATURE_ENCODING) - return isValid + return { privateKey: key, publicKey } } -async function sign (data: string | Object) { - const sign = crypto.createSign(SIGNATURE_ALGORITHM) - - let dataString: string - if (typeof data === 'string') { - dataString = data - } else { - try { - dataString = JSON.stringify(data) - } catch (err) { - logger.error('Cannot sign data.', err) - return '' - } +function isSignatureVerified (fromAccount: AccountInstance, signedDocument: object) { + const publicKeyObject = { + '@context': jsig.SECURITY_CONTEXT_URL, + '@id': fromAccount.url, + '@type': 'CryptographicKey', + owner: fromAccount.url, + publicKeyPem: fromAccount.publicKey } - sign.update(dataString, 'utf8') + const publicKeyOwnerObject = { + '@context': jsig.SECURITY_CONTEXT_URL, + '@id': fromAccount.url, + publicKey: [ publicKeyObject ] + } - const myKey = await getMyPrivateCert() - return sign.sign(myKey, SIGNATURE_ENCODING) -} + const options = { + publicKey: publicKeyObject, + publicKeyOwner: publicKeyOwnerObject + } -function comparePassword (plainPassword: string, hashPassword: string) { - return bcryptComparePromise(plainPassword, hashPassword) + return jsonldVerifyPromise(signedDocument, options) + .catch(err => { + logger.error('Cannot check signature.', err) + return false + }) } -async function createCertsIfNotExist () { - const exist = await certsExist() - if (exist === true) { - return +function signObject (byAccount: AccountInstance, data: any) { + const options = { + privateKeyPem: byAccount.privateKey, + creator: byAccount.url } - return createCerts() + return jsonldSignPromise(data, options) +} + +function comparePassword (plainPassword: string, hashPassword: string) { + return bcryptComparePromise(plainPassword, hashPassword) } async function cryptPassword (password: string) { @@ -80,69 +71,12 @@ async function cryptPassword (password: string) { return bcryptHashPromise(password, salt) } -function getMyPrivateCert () { - const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) - return readFilePromise(certPath, 'utf8') -} - -function getMyPublicCert () { - const certPath = join(CONFIG.STORAGE.CERT_DIR, PUBLIC_CERT_NAME) - return readFilePromise(certPath, 'utf8') -} - // --------------------------------------------------------------------------- export { - checkSignature, + isSignatureVerified, comparePassword, - createCertsIfNotExist, + createPrivateAndPublicKeys, cryptPassword, - getMyPrivateCert, - getMyPublicCert, - sign -} - -// --------------------------------------------------------------------------- - -async function certsExist () { - const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) - - // If there is an error the certificates do not exist - try { - await accessPromise(certPath) - - return true - } catch { - return false - } -} - -async function createCerts () { - const exist = await certsExist() - if (exist === true) { - const errorMessage = 'Certs already exist.' - logger.warning(errorMessage) - throw new Error(errorMessage) - } - - logger.info('Generating a RSA key...') - - const privateCertPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) - const genRsaOptions = { - 'out': privateCertPath, - '2048': false - } - - await opensslExecPromise('genrsa', genRsaOptions) - logger.info('RSA key generated.') - logger.info('Managing public key...') - - const publicCertPath = join(CONFIG.STORAGE.CERT_DIR, 'peertube.pub') - const rsaOptions = { - 'in': privateCertPath, - 'pubout': true, - 'out': publicCertPath - } - - await opensslExecPromise('rsa', rsaOptions) + signObject } diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index af1f401de..8c4c983f7 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -9,7 +9,13 @@ import { } from '../initializers' import { PodInstance } from '../models' import { PodSignature } from '../../shared' -import { sign } from './peertube-crypto' +import { signObject } from './peertube-crypto' + +function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { + return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { + request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) + }) +} type MakeRetryRequestParams = { url: string, @@ -31,61 +37,57 @@ function makeRetryRequest (params: MakeRetryRequestParams) { } type MakeSecureRequestParams = { - method: 'GET' | 'POST' toPod: PodInstance path: string data?: Object } function makeSecureRequest (params: MakeSecureRequestParams) { - return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { - const requestParams: { - url: string, - json: { - signature: PodSignature, - data: any - } - } = { - url: REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path, - json: { - signature: null, - data: null - } + const requestParams: { + method: 'POST', + uri: string, + json: { + signature: PodSignature, + data: any } - - if (params.method !== 'POST') { - return rej(new Error('Cannot make a secure request with a non POST method.')) + } = { + method: 'POST', + uri: REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path, + json: { + signature: null, + data: null } + } - const host = CONFIG.WEBSERVER.HOST + const host = CONFIG.WEBSERVER.HOST - let dataToSign - if (params.data) { - dataToSign = params.data - } else { - // We do not have data to sign so we just take our host - // It is not ideal but the connection should be in HTTPS - dataToSign = host - } + let dataToSign + if (params.data) { + dataToSign = params.data + } else { + // We do not have data to sign so we just take our host + // It is not ideal but the connection should be in HTTPS + dataToSign = host + } - sign(dataToSign).then(signature => { - requestParams.json.signature = { - host, // Which host we pretend to be - signature - } + sign(dataToSign).then(signature => { + requestParams.json.signature = { + host, // Which host we pretend to be + signature + } - // If there are data information - if (params.data) { - requestParams.json.data = params.data - } + // If there are data information + if (params.data) { + requestParams.json.data = params.data + } - request.post(requestParams, (err, response, body) => err ? rej(err) : res({ response, body })) - }) + return doRequest(requestParams) }) } // --------------------------------------------------------------------------- export { + doRequest, makeRetryRequest, makeSecureRequest } diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts new file mode 100644 index 000000000..9586fa562 --- /dev/null +++ b/server/helpers/webfinger.ts @@ -0,0 +1,44 @@ +import * as WebFinger from 'webfinger.js' + +import { isTestInstance } from './core-utils' +import { isActivityPubUrlValid } from './custom-validators' +import { WebFingerData } from '../../shared' +import { fetchRemoteAccountAndCreatePod } from './activitypub' + +const webfinger = new WebFinger({ + webfist_fallback: false, + tls_only: isTestInstance(), + uri_fallback: false, + request_timeout: 3000 +}) + +async function getAccountFromWebfinger (url: string) { + const webfingerData: WebFingerData = await webfingerLookup(url) + + if (Array.isArray(webfingerData.links) === false) return undefined + + const selfLink = webfingerData.links.find(l => l.rel === 'self') + if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) return undefined + + const { account } = await fetchRemoteAccountAndCreatePod(selfLink.href) + + return account +} + +// --------------------------------------------------------------------------- + +export { + getAccountFromWebfinger +} + +// --------------------------------------------------------------------------- + +function webfingerLookup (url: string) { + return new Promise((res, rej) => { + webfinger.lookup('nick@silverbucket.net', (err, p) => { + if (err) return rej(err) + + return p + }) + }) +} diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 9eaef1695..b69188f7e 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -2,7 +2,7 @@ import * as config from 'config' import { promisify0 } from '../helpers/core-utils' import { OAuthClientModel } from '../models/oauth/oauth-client-interface' -import { UserModel } from '../models/user/user-interface' +import { UserModel } from '../models/account/user-interface' // Some checks on configuration files function checkConfig () { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index d349abaf0..cb838cf16 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -10,7 +10,8 @@ import { RequestVideoEventType, RequestVideoQaduType, RemoteVideoRequestType, - JobState + JobState, + JobCategory } from '../../shared/models' import { VideoPrivacy } from '../../shared/models/videos/video-privacy.enum' @@ -60,7 +61,6 @@ const CONFIG = { PASSWORD: config.get('database.password') }, STORAGE: { - CERT_DIR: join(root(), config.get('storage.certs')), LOG_DIR: join(root(), config.get('storage.logs')), VIDEOS_DIR: join(root(), config.get('storage.videos')), THUMBNAILS_DIR: join(root(), config.get('storage.thumbnails')), @@ -211,6 +211,10 @@ const FRIEND_SCORE = { MAX: 1000 } +const ACTIVITY_PUB = { + COLLECTION_ITEMS_PER_PAGE: 10 +} + // --------------------------------------------------------------------------- // Number of points we add/remove from a friend after a successful/bad request @@ -288,17 +292,23 @@ const JOB_STATES: { [ id: string ]: JobState } = { ERROR: 'error', SUCCESS: 'success' } +const JOB_CATEGORIES: { [ id: string ]: JobCategory } = { + TRANSCODING: 'transcoding', + HTTP_REQUEST: 'http-request' +} // How many maximum jobs we fetch from the database per cycle -const JOBS_FETCH_LIMIT_PER_CYCLE = 10 +const JOBS_FETCH_LIMIT_PER_CYCLE = { + transcoding: 10, + httpRequest: 20 +} // 1 minutes let JOBS_FETCHING_INTERVAL = 60000 // --------------------------------------------------------------------------- -const PRIVATE_CERT_NAME = 'peertube.key.pem' -const PUBLIC_CERT_NAME = 'peertube.pub' -const SIGNATURE_ALGORITHM = 'RSA-SHA256' -const SIGNATURE_ENCODING = 'hex' +// const SIGNATURE_ALGORITHM = 'RSA-SHA256' +// const SIGNATURE_ENCODING = 'hex' +const PRIVATE_RSA_KEY_SIZE = 2048 // Password encryption const BCRYPT_SALT_SIZE = 10 @@ -368,14 +378,13 @@ export { JOB_STATES, JOBS_FETCH_LIMIT_PER_CYCLE, JOBS_FETCHING_INTERVAL, + JOB_CATEGORIES, LAST_MIGRATION_VERSION, OAUTH_LIFETIME, OPENGRAPH_AND_OEMBED_COMMENT, PAGINATION_COUNT_DEFAULT, PODS_SCORE, PREVIEWS_SIZE, - PRIVATE_CERT_NAME, - PUBLIC_CERT_NAME, REMOTE_SCHEME, REQUEST_ENDPOINT_ACTIONS, REQUEST_ENDPOINTS, @@ -393,11 +402,11 @@ export { REQUESTS_VIDEO_QADU_LIMIT_PODS, RETRY_REQUESTS, SEARCHABLE_COLUMNS, - SIGNATURE_ALGORITHM, - SIGNATURE_ENCODING, + PRIVATE_RSA_KEY_SIZE, SORTABLE_COLUMNS, STATIC_MAX_AGE, STATIC_PATHS, + ACTIVITY_PUB, THUMBNAILS_SIZE, VIDEO_CATEGORIES, VIDEO_LANGUAGES, diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 141566c3a..52e766394 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -15,8 +15,9 @@ import { BlacklistedVideoModel } from './../models/video/video-blacklist-interfa import { VideoFileModel } from './../models/video/video-file-interface' import { VideoAbuseModel } from './../models/video/video-abuse-interface' import { VideoChannelModel } from './../models/video/video-channel-interface' -import { UserModel } from './../models/user/user-interface' -import { UserVideoRateModel } from './../models/user/user-video-rate-interface' +import { UserModel } from '../models/account/user-interface' +import { AccountVideoRateModel } from '../models/account/account-video-rate-interface' +import { AccountFollowModel } from '../models/account/account-follow-interface' import { TagModel } from './../models/video/tag-interface' import { RequestModel } from './../models/request/request-interface' import { RequestVideoQaduModel } from './../models/request/request-video-qadu-interface' @@ -26,7 +27,7 @@ import { PodModel } from './../models/pod/pod-interface' import { OAuthTokenModel } from './../models/oauth/oauth-token-interface' import { OAuthClientModel } from './../models/oauth/oauth-client-interface' import { JobModel } from './../models/job/job-interface' -import { AuthorModel } from './../models/video/author-interface' +import { AccountModel } from './../models/account/account-interface' import { ApplicationModel } from './../models/application/application-interface' const dbname = CONFIG.DATABASE.DBNAME @@ -38,7 +39,7 @@ const database: { init?: (silent: boolean) => Promise, Application?: ApplicationModel, - Author?: AuthorModel, + Account?: AccountModel, Job?: JobModel, OAuthClient?: OAuthClientModel, OAuthToken?: OAuthTokenModel, @@ -48,7 +49,8 @@ const database: { RequestVideoQadu?: RequestVideoQaduModel, Request?: RequestModel, Tag?: TagModel, - UserVideoRate?: UserVideoRateModel, + AccountVideoRate?: AccountVideoRateModel, + AccountFollow?: AccountFollowModel, User?: UserModel, VideoAbuse?: VideoAbuseModel, VideoChannel?: VideoChannelModel, @@ -126,7 +128,7 @@ async function getModelFiles (modelDirectory: string) { return true }) - const tasks: Bluebird[] = [] + const tasks: Promise[] = [] // For each directory we read it and append model in the modelFilePaths array for (const directory of directories) { diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts new file mode 100644 index 000000000..740800606 --- /dev/null +++ b/server/lib/activitypub/index.ts @@ -0,0 +1,3 @@ +export * from './process-create' +export * from './process-flag' +export * from './process-update' diff --git a/server/lib/activitypub/process-create.ts b/server/lib/activitypub/process-create.ts new file mode 100644 index 000000000..114ff1848 --- /dev/null +++ b/server/lib/activitypub/process-create.ts @@ -0,0 +1,104 @@ +import { + ActivityCreate, + VideoTorrentObject, + VideoChannelObject +} from '../../../shared' +import { database as db } from '../../initializers' +import { logger, retryTransactionWrapper } from '../../helpers' + +function processCreateActivity (activity: ActivityCreate) { + const activityObject = activity.object + const activityType = activityObject.type + + if (activityType === 'Video') { + return processCreateVideo(activityObject as VideoTorrentObject) + } else if (activityType === 'VideoChannel') { + return processCreateVideoChannel(activityObject as VideoChannelObject) + } + + logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) + return Promise.resolve() +} + +// --------------------------------------------------------------------------- + +export { + processCreateActivity +} + +// --------------------------------------------------------------------------- + +function processCreateVideo (video: VideoTorrentObject) { + const options = { + arguments: [ video ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + return retryTransactionWrapper(addRemoteVideo, options) +} + +async function addRemoteVideo (videoToCreateData: VideoTorrentObject) { + logger.debug('Adding remote video %s.', videoToCreateData.url) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) + if (videoFromDatabase) throw new Error('UUID already exists.') + + const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) + if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') + + const tags = videoToCreateData.tags + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + + const videoData = { + name: videoToCreateData.name, + uuid: videoToCreateData.uuid, + category: videoToCreateData.category, + licence: videoToCreateData.licence, + language: videoToCreateData.language, + nsfw: videoToCreateData.nsfw, + description: videoToCreateData.truncatedDescription, + channelId: videoChannel.id, + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: videoToCreateData.updatedAt, + views: videoToCreateData.views, + likes: videoToCreateData.likes, + dislikes: videoToCreateData.dislikes, + remote: true, + privacy: videoToCreateData.privacy + } + + const video = db.Video.build(videoData) + await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) + const videoCreated = await video.save(sequelizeOptions) + + const tasks = [] + for (const fileData of videoToCreateData.files) { + const videoFileInstance = db.VideoFile.build({ + extname: fileData.extname, + infoHash: fileData.infoHash, + resolution: fileData.resolution, + size: fileData.size, + videoId: videoCreated.id + }) + + tasks.push(videoFileInstance.save(sequelizeOptions)) + } + + await Promise.all(tasks) + + await videoCreated.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) +} + +function processCreateVideoChannel (videoChannel: VideoChannelObject) { + +} diff --git a/server/lib/activitypub/process-flag.ts b/server/lib/activitypub/process-flag.ts new file mode 100644 index 000000000..6fa862ee9 --- /dev/null +++ b/server/lib/activitypub/process-flag.ts @@ -0,0 +1,17 @@ +import { + ActivityCreate, + VideoTorrentObject, + VideoChannelObject +} from '../../../shared' + +function processFlagActivity (activity: ActivityCreate) { + // empty +} + +// --------------------------------------------------------------------------- + +export { + processFlagActivity +} + +// --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/process-update.ts b/server/lib/activitypub/process-update.ts new file mode 100644 index 000000000..187c7be7c --- /dev/null +++ b/server/lib/activitypub/process-update.ts @@ -0,0 +1,29 @@ +import { + ActivityCreate, + VideoTorrentObject, + VideoChannelObject +} from '../../../shared' + +function processUpdateActivity (activity: ActivityCreate) { + if (activity.object.type === 'Video') { + return processUpdateVideo(activity.object) + } else if (activity.object.type === 'VideoChannel') { + return processUpdateVideoChannel(activity.object) + } +} + +// --------------------------------------------------------------------------- + +export { + processUpdateActivity +} + +// --------------------------------------------------------------------------- + +function processUpdateVideo (video: VideoTorrentObject) { + +} + +function processUpdateVideoChannel (videoChannel: VideoChannelObject) { + +} diff --git a/server/lib/activitypub/send-request.ts b/server/lib/activitypub/send-request.ts new file mode 100644 index 000000000..6a31c226d --- /dev/null +++ b/server/lib/activitypub/send-request.ts @@ -0,0 +1,129 @@ +import * as Sequelize from 'sequelize' + +import { + AccountInstance, + VideoInstance, + VideoChannelInstance +} from '../../models' +import { httpRequestJobScheduler } from '../jobs' +import { signObject, activityPubContextify } from '../../helpers' +import { Activity } from '../../../shared' + +function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { + const videoChannelObject = videoChannel.toActivityPubObject() + const data = createActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) + + return broadcastToFollowers(data, t) +} + +function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { + const videoChannelObject = videoChannel.toActivityPubObject() + const data = updateActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) + + return broadcastToFollowers(data, t) +} + +function sendDeleteVideoChannel (videoChannel: VideoChannelInstance, t: Sequelize.Transaction) { + const videoChannelObject = videoChannel.toActivityPubObject() + const data = deleteActivityData(videoChannel.url, videoChannel.Account, videoChannelObject) + + return broadcastToFollowers(data, t) +} + +function sendAddVideo (video: VideoInstance, t: Sequelize.Transaction) { + const videoObject = video.toActivityPubObject() + const data = addActivityData(video.url, video.VideoChannel.Account, video.VideoChannel.url, videoObject) + + return broadcastToFollowers(data, t) +} + +function sendUpdateVideo (video: VideoInstance, t: Sequelize.Transaction) { + const videoObject = video.toActivityPubObject() + const data = updateActivityData(video.url, video.VideoChannel.Account, videoObject) + + return broadcastToFollowers(data, t) +} + +function sendDeleteVideo (video: VideoInstance, t: Sequelize.Transaction) { + const videoObject = video.toActivityPubObject() + const data = deleteActivityData(video.url, video.VideoChannel.Account, videoObject) + + return broadcastToFollowers(data, t) +} + +// --------------------------------------------------------------------------- + +export { + +} + +// --------------------------------------------------------------------------- + +function broadcastToFollowers (data: any, t: Sequelize.Transaction) { + return httpRequestJobScheduler.createJob(t, 'http-request', 'httpRequestBroadcastHandler', data) +} + +function buildSignedActivity (byAccount: AccountInstance, data: Object) { + const activity = activityPubContextify(data) + + return signObject(byAccount, activity) as Promise +} + +async function getPublicActivityTo (account: AccountInstance) { + const inboxUrls = await account.getFollowerSharedInboxUrls() + + return inboxUrls.concat('https://www.w3.org/ns/activitystreams#Public') +} + +async function createActivityData (url: string, byAccount: AccountInstance, object: any) { + const to = await getPublicActivityTo(byAccount) + const base = { + type: 'Create', + id: url, + actor: byAccount.url, + to, + object + } + + return buildSignedActivity(byAccount, base) +} + +async function updateActivityData (url: string, byAccount: AccountInstance, object: any) { + const to = await getPublicActivityTo(byAccount) + const base = { + type: 'Update', + id: url, + actor: byAccount.url, + to, + object + } + + return buildSignedActivity(byAccount, base) +} + +async function deleteActivityData (url: string, byAccount: AccountInstance, object: any) { + const to = await getPublicActivityTo(byAccount) + const base = { + type: 'Update', + id: url, + actor: byAccount.url, + to, + object + } + + return buildSignedActivity(byAccount, base) +} + +async function addActivityData (url: string, byAccount: AccountInstance, target: string, object: any) { + const to = await getPublicActivityTo(byAccount) + const base = { + type: 'Add', + id: url, + actor: byAccount.url, + to, + object, + target + } + + return buildSignedActivity(byAccount, base) +} diff --git a/server/lib/index.ts b/server/lib/index.ts index d1534b085..bfb415ad2 100644 --- a/server/lib/index.ts +++ b/server/lib/index.ts @@ -1,3 +1,4 @@ +export * from './activitypub' export * from './cache' export * from './jobs' export * from './request' diff --git a/server/lib/jobs/handlers/index.ts b/server/lib/jobs/handlers/index.ts deleted file mode 100644 index cef1f89a9..000000000 --- a/server/lib/jobs/handlers/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as videoFileOptimizer from './video-file-optimizer' -import * as videoFileTranscoder from './video-file-transcoder' - -export interface JobHandler { - process (data: object, jobId: number): T - onError (err: Error, jobId: number) - onSuccess (jobId: number, jobResult: T) -} - -const jobHandlers: { [ handlerName: string ]: JobHandler } = { - videoFileOptimizer, - videoFileTranscoder -} - -export { - jobHandlers -} diff --git a/server/lib/jobs/handlers/video-file-optimizer.ts b/server/lib/jobs/handlers/video-file-optimizer.ts deleted file mode 100644 index ccded4721..000000000 --- a/server/lib/jobs/handlers/video-file-optimizer.ts +++ /dev/null @@ -1,85 +0,0 @@ -import * as Bluebird from 'bluebird' - -import { database as db } from '../../../initializers/database' -import { logger, computeResolutionsToTranscode } from '../../../helpers' -import { VideoInstance } from '../../../models' -import { addVideoToFriends } from '../../friends' -import { JobScheduler } from '../job-scheduler' - -async function process (data: { videoUUID: string }, jobId: number) { - const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) - return undefined - } - - await video.optimizeOriginalVideofile() - - return video -} - -function onError (err: Error, jobId: number) { - logger.error('Error when optimized video file in job %d.', jobId, err) - return Promise.resolve() -} - -async function onSuccess (jobId: number, video: VideoInstance) { - if (video === undefined) return undefined - - logger.info('Job %d is a success.', jobId) - - // Maybe the video changed in database, refresh it - const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) - // Video does not exist anymore - if (!videoDatabase) return undefined - - const remoteVideo = await videoDatabase.toAddRemoteJSON() - - // Now we'll add the video's meta data to our friends - await addVideoToFriends(remoteVideo, null) - - const originalFileHeight = await videoDatabase.getOriginalFileHeight() - // Create transcoding jobs if there are enabled resolutions - - const resolutionsEnabled = computeResolutionsToTranscode(originalFileHeight) - logger.info( - 'Resolutions computed for video %s and origin file height of %d.', videoDatabase.uuid, originalFileHeight, - { resolutions: resolutionsEnabled } - ) - - if (resolutionsEnabled.length !== 0) { - try { - await db.sequelize.transaction(async t => { - const tasks: Bluebird[] = [] - - for (const resolution of resolutionsEnabled) { - const dataInput = { - videoUUID: videoDatabase.uuid, - resolution - } - - const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput) - tasks.push(p) - } - - await Promise.all(tasks) - }) - - logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) - } catch (err) { - logger.warn('Cannot transcode the video.', err) - } - } else { - logger.info('No transcoding jobs created for video %s (no resolutions enabled).') - return undefined - } -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/lib/jobs/handlers/video-file-transcoder.ts b/server/lib/jobs/handlers/video-file-transcoder.ts deleted file mode 100644 index 853645510..000000000 --- a/server/lib/jobs/handlers/video-file-transcoder.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { database as db } from '../../../initializers/database' -import { updateVideoToFriends } from '../../friends' -import { logger } from '../../../helpers' -import { VideoInstance } from '../../../models' -import { VideoResolution } from '../../../../shared' - -async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { - const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) - return undefined - } - - await video.transcodeOriginalVideofile(data.resolution) - - return video -} - -function onError (err: Error, jobId: number) { - logger.error('Error when transcoding video file in job %d.', jobId, err) - return Promise.resolve() -} - -async function onSuccess (jobId: number, video: VideoInstance) { - if (video === undefined) return undefined - - logger.info('Job %d is a success.', jobId) - - // Maybe the video changed in database, refresh it - const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) - // Video does not exist anymore - if (!videoDatabase) return undefined - - const remoteVideo = videoDatabase.toUpdateRemoteJSON() - - // Now we'll add the video's meta data to our friends - await updateVideoToFriends(remoteVideo, null) - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts new file mode 100644 index 000000000..6b6946d02 --- /dev/null +++ b/server/lib/jobs/http-request-job-scheduler/http-request-broadcast-handler.ts @@ -0,0 +1,25 @@ +import * as Bluebird from 'bluebird' + +import { database as db } from '../../../initializers/database' +import { logger } from '../../../helpers' + +async function process (data: { videoUUID: string }, jobId: number) { + +} + +function onError (err: Error, jobId: number) { + logger.error('Error when optimized video file in job %d.', jobId, err) + return Promise.resolve() +} + +async function onSuccess (jobId: number) { + +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts new file mode 100644 index 000000000..42cb9139c --- /dev/null +++ b/server/lib/jobs/http-request-job-scheduler/http-request-job-scheduler.ts @@ -0,0 +1,17 @@ +import { JobScheduler, JobHandler } from '../job-scheduler' + +import * as httpRequestBroadcastHandler from './http-request-broadcast-handler' +import * as httpRequestUnicastHandler from './http-request-unicast-handler' +import { JobCategory } from '../../../../shared' + +const jobHandlers: { [ handlerName: string ]: JobHandler } = { + httpRequestBroadcastHandler, + httpRequestUnicastHandler +} +const jobCategory: JobCategory = 'http-request' + +const httpRequestJobScheduler = new JobScheduler(jobCategory, jobHandlers) + +export { + httpRequestJobScheduler +} diff --git a/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts new file mode 100644 index 000000000..6b6946d02 --- /dev/null +++ b/server/lib/jobs/http-request-job-scheduler/http-request-unicast-handler.ts @@ -0,0 +1,25 @@ +import * as Bluebird from 'bluebird' + +import { database as db } from '../../../initializers/database' +import { logger } from '../../../helpers' + +async function process (data: { videoUUID: string }, jobId: number) { + +} + +function onError (err: Error, jobId: number) { + logger.error('Error when optimized video file in job %d.', jobId, err) + return Promise.resolve() +} + +async function onSuccess (jobId: number) { + +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/http-request-job-scheduler/index.ts b/server/lib/jobs/http-request-job-scheduler/index.ts new file mode 100644 index 000000000..4d2573296 --- /dev/null +++ b/server/lib/jobs/http-request-job-scheduler/index.ts @@ -0,0 +1 @@ +export * from './http-request-job-scheduler' diff --git a/server/lib/jobs/index.ts b/server/lib/jobs/index.ts index b18a3d845..a92743707 100644 --- a/server/lib/jobs/index.ts +++ b/server/lib/jobs/index.ts @@ -1 +1,2 @@ -export * from './job-scheduler' +export * from './http-request-job-scheduler' +export * from './transcoding-job-scheduler' diff --git a/server/lib/jobs/job-scheduler.ts b/server/lib/jobs/job-scheduler.ts index 61d483268..89a4bca88 100644 --- a/server/lib/jobs/job-scheduler.ts +++ b/server/lib/jobs/job-scheduler.ts @@ -1,39 +1,41 @@ import { AsyncQueue, forever, queue } from 'async' import * as Sequelize from 'sequelize' -import { database as db } from '../../initializers/database' import { + database as db, JOBS_FETCHING_INTERVAL, JOBS_FETCH_LIMIT_PER_CYCLE, JOB_STATES } from '../../initializers' import { logger } from '../../helpers' import { JobInstance } from '../../models' -import { JobHandler, jobHandlers } from './handlers' +import { JobCategory } from '../../../shared' +export interface JobHandler { + process (data: object, jobId: number): T + onError (err: Error, jobId: number) + onSuccess (jobId: number, jobResult: T) +} type JobQueueCallback = (err: Error) => void -class JobScheduler { - - private static instance: JobScheduler +class JobScheduler { - private constructor () { } - - static get Instance () { - return this.instance || (this.instance = new this()) - } + constructor ( + private jobCategory: JobCategory, + private jobHandlers: { [ id: string ]: JobHandler } + ) {} async activate () { - const limit = JOBS_FETCH_LIMIT_PER_CYCLE + const limit = JOBS_FETCH_LIMIT_PER_CYCLE[this.jobCategory] - logger.info('Jobs scheduler activated.') + logger.info('Jobs scheduler %s activated.', this.jobCategory) const jobsQueue = queue(this.processJob.bind(this)) // Finish processing jobs from a previous start const state = JOB_STATES.PROCESSING try { - const jobs = await db.Job.listWithLimit(limit, state) + const jobs = await db.Job.listWithLimitByCategory(limit, state, this.jobCategory) this.enqueueJobs(jobsQueue, jobs) } catch (err) { @@ -49,7 +51,7 @@ class JobScheduler { const state = JOB_STATES.PENDING try { - const jobs = await db.Job.listWithLimit(limit, state) + const jobs = await db.Job.listWithLimitByCategory(limit, state, this.jobCategory) this.enqueueJobs(jobsQueue, jobs) } catch (err) { @@ -64,9 +66,10 @@ class JobScheduler { ) } - createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: object) { + createJob (transaction: Sequelize.Transaction, category: JobCategory, handlerName: string, handlerInputData: object) { const createQuery = { state: JOB_STATES.PENDING, + category, handlerName, handlerInputData } @@ -80,7 +83,7 @@ class JobScheduler { } private async processJob (job: JobInstance, callback: (err: Error) => void) { - const jobHandler = jobHandlers[job.handlerName] + const jobHandler = this.jobHandlers[job.handlerName] if (jobHandler === undefined) { logger.error('Unknown job handler for job %s.', job.handlerName) return callback(null) diff --git a/server/lib/jobs/transcoding-job-scheduler/index.ts b/server/lib/jobs/transcoding-job-scheduler/index.ts new file mode 100644 index 000000000..73152a1be --- /dev/null +++ b/server/lib/jobs/transcoding-job-scheduler/index.ts @@ -0,0 +1 @@ +export * from './transcoding-job-scheduler' diff --git a/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts new file mode 100644 index 000000000..d7c614fb8 --- /dev/null +++ b/server/lib/jobs/transcoding-job-scheduler/transcoding-job-scheduler.ts @@ -0,0 +1,17 @@ +import { JobScheduler, JobHandler } from '../job-scheduler' + +import * as videoFileOptimizer from './video-file-optimizer-handler' +import * as videoFileTranscoder from './video-file-transcoder-handler' +import { JobCategory } from '../../../../shared' + +const jobHandlers: { [ handlerName: string ]: JobHandler } = { + videoFileOptimizer, + videoFileTranscoder +} +const jobCategory: JobCategory = 'transcoding' + +const transcodingJobScheduler = new JobScheduler(jobCategory, jobHandlers) + +export { + transcodingJobScheduler +} diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts new file mode 100644 index 000000000..ccded4721 --- /dev/null +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts @@ -0,0 +1,85 @@ +import * as Bluebird from 'bluebird' + +import { database as db } from '../../../initializers/database' +import { logger, computeResolutionsToTranscode } from '../../../helpers' +import { VideoInstance } from '../../../models' +import { addVideoToFriends } from '../../friends' +import { JobScheduler } from '../job-scheduler' + +async function process (data: { videoUUID: string }, jobId: number) { + const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) + // No video, maybe deleted? + if (!video) { + logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) + return undefined + } + + await video.optimizeOriginalVideofile() + + return video +} + +function onError (err: Error, jobId: number) { + logger.error('Error when optimized video file in job %d.', jobId, err) + return Promise.resolve() +} + +async function onSuccess (jobId: number, video: VideoInstance) { + if (video === undefined) return undefined + + logger.info('Job %d is a success.', jobId) + + // Maybe the video changed in database, refresh it + const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) + // Video does not exist anymore + if (!videoDatabase) return undefined + + const remoteVideo = await videoDatabase.toAddRemoteJSON() + + // Now we'll add the video's meta data to our friends + await addVideoToFriends(remoteVideo, null) + + const originalFileHeight = await videoDatabase.getOriginalFileHeight() + // Create transcoding jobs if there are enabled resolutions + + const resolutionsEnabled = computeResolutionsToTranscode(originalFileHeight) + logger.info( + 'Resolutions computed for video %s and origin file height of %d.', videoDatabase.uuid, originalFileHeight, + { resolutions: resolutionsEnabled } + ) + + if (resolutionsEnabled.length !== 0) { + try { + await db.sequelize.transaction(async t => { + const tasks: Bluebird[] = [] + + for (const resolution of resolutionsEnabled) { + const dataInput = { + videoUUID: videoDatabase.uuid, + resolution + } + + const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput) + tasks.push(p) + } + + await Promise.all(tasks) + }) + + logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) + } catch (err) { + logger.warn('Cannot transcode the video.', err) + } + } else { + logger.info('No transcoding jobs created for video %s (no resolutions enabled).') + return undefined + } +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts new file mode 100644 index 000000000..853645510 --- /dev/null +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts @@ -0,0 +1,49 @@ +import { database as db } from '../../../initializers/database' +import { updateVideoToFriends } from '../../friends' +import { logger } from '../../../helpers' +import { VideoInstance } from '../../../models' +import { VideoResolution } from '../../../../shared' + +async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { + const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) + // No video, maybe deleted? + if (!video) { + logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) + return undefined + } + + await video.transcodeOriginalVideofile(data.resolution) + + return video +} + +function onError (err: Error, jobId: number) { + logger.error('Error when transcoding video file in job %d.', jobId, err) + return Promise.resolve() +} + +async function onSuccess (jobId: number, video: VideoInstance) { + if (video === undefined) return undefined + + logger.info('Job %d is a success.', jobId) + + // Maybe the video changed in database, refresh it + const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) + // Video does not exist anymore + if (!videoDatabase) return undefined + + const remoteVideo = videoDatabase.toUpdateRemoteJSON() + + // Now we'll add the video's meta data to our friends + await updateVideoToFriends(remoteVideo, null) + + return undefined +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/user.ts b/server/lib/user.ts index a92f4777b..57c653e55 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -1,9 +1,9 @@ import { database as db } from '../initializers' import { UserInstance } from '../models' -import { addVideoAuthorToFriends } from './friends' +import { addVideoAccountToFriends } from './friends' import { createVideoChannel } from './video-channel' -async function createUserAuthorAndChannel (user: UserInstance, validateUser = true) { +async function createUserAccountAndChannel (user: UserInstance, validateUser = true) { const res = await db.sequelize.transaction(async t => { const userOptions = { transaction: t, @@ -11,25 +11,25 @@ async function createUserAuthorAndChannel (user: UserInstance, validateUser = tr } const userCreated = await user.save(userOptions) - const authorInstance = db.Author.build({ + const accountInstance = db.Account.build({ name: userCreated.username, podId: null, // It is our pod userId: userCreated.id }) - const authorCreated = await authorInstance.save({ transaction: t }) + const accountCreated = await accountInstance.save({ transaction: t }) - const remoteVideoAuthor = authorCreated.toAddRemoteJSON() + const remoteVideoAccount = accountCreated.toAddRemoteJSON() // Now we'll add the video channel's meta data to our friends - const author = await addVideoAuthorToFriends(remoteVideoAuthor, t) + const account = await addVideoAccountToFriends(remoteVideoAccount, t) const videoChannelInfo = { name: `Default ${userCreated.username} channel` } - const videoChannel = await createVideoChannel(videoChannelInfo, authorCreated, t) + const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t) - return { author, videoChannel } + return { account, videoChannel } }) return res @@ -38,5 +38,5 @@ async function createUserAuthorAndChannel (user: UserInstance, validateUser = tr // --------------------------------------------------------------------------- export { - createUserAuthorAndChannel + createUserAccountAndChannel } diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index 678ffe643..a6dd4d061 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts @@ -3,15 +3,15 @@ import * as Sequelize from 'sequelize' import { addVideoChannelToFriends } from './friends' import { database as db } from '../initializers' import { logger } from '../helpers' -import { AuthorInstance } from '../models' +import { AccountInstance } from '../models' import { VideoChannelCreate } from '../../shared/models' -async function createVideoChannel (videoChannelInfo: VideoChannelCreate, author: AuthorInstance, t: Sequelize.Transaction) { +async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account: AccountInstance, t: Sequelize.Transaction) { const videoChannelData = { name: videoChannelInfo.name, description: videoChannelInfo.description, remote: false, - authorId: author.id + authorId: account.id } const videoChannel = db.VideoChannel.build(videoChannelData) @@ -19,8 +19,8 @@ async function createVideoChannel (videoChannelInfo: VideoChannelCreate, author: const videoChannelCreated = await videoChannel.save(options) - // Do not forget to add Author information to the created video channel - videoChannelCreated.Author = author + // Do not forget to add Account information to the created video channel + videoChannelCreated.Account = account const remoteVideoChannel = videoChannelCreated.toAddRemoteJSON() diff --git a/server/middlewares/activitypub.ts b/server/middlewares/activitypub.ts new file mode 100644 index 000000000..6cf8eea6f --- /dev/null +++ b/server/middlewares/activitypub.ts @@ -0,0 +1,57 @@ +import { Request, Response, NextFunction } from 'express' + +import { database as db } from '../initializers' +import { + logger, + getAccountFromWebfinger, + isSignatureVerified +} from '../helpers' +import { ActivityPubSignature } from '../../shared' + +async function checkSignature (req: Request, res: Response, next: NextFunction) { + const signatureObject: ActivityPubSignature = req.body.signature + + logger.debug('Checking signature of account %s...', signatureObject.creator) + + let account = await db.Account.loadByUrl(signatureObject.creator) + + // We don't have this account in our database, fetch it on remote + if (!account) { + account = await getAccountFromWebfinger(signatureObject.creator) + + if (!account) { + return res.sendStatus(403) + } + + // Save our new account in database + await account.save() + } + + const verified = await isSignatureVerified(account, req.body) + if (verified === false) return res.sendStatus(403) + + res.locals.signature.account = account + + return next() +} + +function executeIfActivityPub (fun: any | any[]) { + return (req: Request, res: Response, next: NextFunction) => { + if (req.header('Accept') !== 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"') { + return next() + } + + if (Array.isArray(fun) === true) { + fun[0](req, res, next) // FIXME: doesn't work + } + + return fun(req, res, next) + } +} + +// --------------------------------------------------------------------------- + +export { + checkSignature, + executeIfActivityPub +} diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index cec3e0b2a..40480450b 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts @@ -1,9 +1,9 @@ export * from './validators' +export * from './activitypub' export * from './async' export * from './oauth' export * from './pagination' export * from './pods' export * from './search' -export * from './secure' export * from './sort' export * from './user-right' diff --git a/server/middlewares/secure.ts b/server/middlewares/secure.ts deleted file mode 100644 index 5dd809f15..000000000 --- a/server/middlewares/secure.ts +++ /dev/null @@ -1,55 +0,0 @@ -import 'express-validator' -import * as express from 'express' - -import { database as db } from '../initializers' -import { - logger, - checkSignature as peertubeCryptoCheckSignature -} from '../helpers' -import { PodSignature } from '../../shared' - -async function checkSignature (req: express.Request, res: express.Response, next: express.NextFunction) { - const signatureObject: PodSignature = req.body.signature - const host = signatureObject.host - - try { - const pod = await db.Pod.loadByHost(host) - if (pod === null) { - logger.error('Unknown pod %s.', host) - return res.sendStatus(403) - } - - logger.debug('Checking signature from %s.', host) - - let signatureShouldBe - // If there is data in the body the sender used it for its signature - // If there is no data we just use its host as signature - if (req.body.data) { - signatureShouldBe = req.body.data - } else { - signatureShouldBe = host - } - - const signatureOk = peertubeCryptoCheckSignature(pod.publicKey, signatureShouldBe, signatureObject.signature) - - if (signatureOk === true) { - res.locals.secure = { - pod - } - - return next() - } - - logger.error('Signature is not okay in body for %s.', signatureObject.host) - return res.sendStatus(403) - } catch (err) { - logger.error('Cannot get signed host in body.', { error: err.stack, signature: signatureObject.signature }) - return res.sendStatus(500) - } -} - -// --------------------------------------------------------------------------- - -export { - checkSignature -} diff --git a/server/middlewares/validators/account.ts b/server/middlewares/validators/account.ts new file mode 100644 index 000000000..5abe942d6 --- /dev/null +++ b/server/middlewares/validators/account.ts @@ -0,0 +1,53 @@ +import { param } from 'express-validator/check' +import * as express from 'express' + +import { database as db } from '../../initializers/database' +import { checkErrors } from './utils' +import { + logger, + isUserUsernameValid, + isUserPasswordValid, + isUserVideoQuotaValid, + isUserDisplayNSFWValid, + isUserRoleValid, + isAccountNameValid +} from '../../helpers' +import { AccountInstance } from '../../models' + +const localAccountValidator = [ + param('name').custom(isAccountNameValid).withMessage('Should have a valid account name'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking localAccountValidator parameters', { parameters: req.params }) + + checkErrors(req, res, () => { + checkLocalAccountExists(req.params.name, res, next) + }) + } +] + +// --------------------------------------------------------------------------- + +export { + localAccountValidator +} + +// --------------------------------------------------------------------------- + +function checkLocalAccountExists (name: string, res: express.Response, callback: (err: Error, account: AccountInstance) => void) { + db.Account.loadLocalAccountByName(name) + .then(account => { + if (!account) { + return res.status(404) + .send({ error: 'Account not found' }) + .end() + } + + res.locals.account = account + return callback(null, account) + }) + .catch(err => { + logger.error('Error in account request validator.', err) + return res.sendStatus(500) + }) +} diff --git a/server/middlewares/validators/activitypub/index.ts b/server/middlewares/validators/activitypub/index.ts new file mode 100644 index 000000000..f1f26043e --- /dev/null +++ b/server/middlewares/validators/activitypub/index.ts @@ -0,0 +1,3 @@ +export * from './pods' +export * from './signature' +export * from './videos' diff --git a/server/middlewares/validators/activitypub/pods.ts b/server/middlewares/validators/activitypub/pods.ts new file mode 100644 index 000000000..f917b61ee --- /dev/null +++ b/server/middlewares/validators/activitypub/pods.ts @@ -0,0 +1,38 @@ +import { body } from 'express-validator/check' +import * as express from 'express' + +import { database as db } from '../../../initializers' +import { isHostValid, logger } from '../../../helpers' +import { checkErrors } from '../utils' + +const remotePodsAddValidator = [ + body('host').custom(isHostValid).withMessage('Should have a host'), + body('email').isEmail().withMessage('Should have an email'), + body('publicKey').not().isEmpty().withMessage('Should have a public key'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking podsAdd parameters', { parameters: req.body }) + + checkErrors(req, res, () => { + db.Pod.loadByHost(req.body.host) + .then(pod => { + // Pod with this host already exists + if (pod) { + return res.sendStatus(409) + } + + return next() + }) + .catch(err => { + logger.error('Cannot load pod by host.', err) + res.sendStatus(500) + }) + }) + } +] + +// --------------------------------------------------------------------------- + +export { + remotePodsAddValidator +} diff --git a/server/middlewares/validators/activitypub/signature.ts b/server/middlewares/validators/activitypub/signature.ts new file mode 100644 index 000000000..0ce15c1f6 --- /dev/null +++ b/server/middlewares/validators/activitypub/signature.ts @@ -0,0 +1,30 @@ +import { body } from 'express-validator/check' +import * as express from 'express' + +import { + logger, + isDateValid, + isSignatureTypeValid, + isSignatureCreatorValid, + isSignatureValueValid +} from '../../../helpers' +import { checkErrors } from '../utils' + +const signatureValidator = [ + body('signature.type').custom(isSignatureTypeValid).withMessage('Should have a valid signature type'), + body('signature.created').custom(isDateValid).withMessage('Should have a valid signature created date'), + body('signature.creator').custom(isSignatureCreatorValid).withMessage('Should have a valid signature creator'), + body('signature.signatureValue').custom(isSignatureValueValid).withMessage('Should have a valid signature value'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking activitypub signature parameter', { parameters: { signature: req.body.signature } }) + + checkErrors(req, res, next) + } +] + +// --------------------------------------------------------------------------- + +export { + signatureValidator +} diff --git a/server/middlewares/validators/activitypub/videos.ts b/server/middlewares/validators/activitypub/videos.ts new file mode 100644 index 000000000..497320cc1 --- /dev/null +++ b/server/middlewares/validators/activitypub/videos.ts @@ -0,0 +1,61 @@ +import { body } from 'express-validator/check' +import * as express from 'express' + +import { + logger, + isArray, + removeBadRequestVideos, + removeBadRequestVideosQadu, + removeBadRequestVideosEvents +} from '../../../helpers' +import { checkErrors } from '../utils' + +const remoteVideosValidator = [ + body('data').custom(isArray), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking remoteVideos parameters', { parameters: req.body }) + + checkErrors(req, res, () => { + removeBadRequestVideos(req.body.data) + + return next() + }) + } +] + +const remoteQaduVideosValidator = [ + body('data').custom(isArray), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body }) + + checkErrors(req, res, () => { + removeBadRequestVideosQadu(req.body.data) + + return next() + }) + } +] + +const remoteEventsVideosValidator = [ + body('data').custom(isArray), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body }) + + checkErrors(req, res, () => { + removeBadRequestVideosEvents(req.body.data) + + return next() + }) + } +] + +// --------------------------------------------------------------------------- + +export { + remoteVideosValidator, + remoteQaduVideosValidator, + remoteEventsVideosValidator +} diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 247f6039e..46c00d679 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -1,5 +1,6 @@ +export * from './account' export * from './oembed' -export * from './remote' +export * from './activitypub' export * from './pagination' export * from './pods' export * from './sort' diff --git a/server/middlewares/validators/remote/index.ts b/server/middlewares/validators/remote/index.ts deleted file mode 100644 index f1f26043e..000000000 --- a/server/middlewares/validators/remote/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './pods' -export * from './signature' -export * from './videos' diff --git a/server/middlewares/validators/remote/pods.ts b/server/middlewares/validators/remote/pods.ts deleted file mode 100644 index f917b61ee..000000000 --- a/server/middlewares/validators/remote/pods.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { body } from 'express-validator/check' -import * as express from 'express' - -import { database as db } from '../../../initializers' -import { isHostValid, logger } from '../../../helpers' -import { checkErrors } from '../utils' - -const remotePodsAddValidator = [ - body('host').custom(isHostValid).withMessage('Should have a host'), - body('email').isEmail().withMessage('Should have an email'), - body('publicKey').not().isEmpty().withMessage('Should have a public key'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking podsAdd parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - db.Pod.loadByHost(req.body.host) - .then(pod => { - // Pod with this host already exists - if (pod) { - return res.sendStatus(409) - } - - return next() - }) - .catch(err => { - logger.error('Cannot load pod by host.', err) - res.sendStatus(500) - }) - }) - } -] - -// --------------------------------------------------------------------------- - -export { - remotePodsAddValidator -} diff --git a/server/middlewares/validators/remote/signature.ts b/server/middlewares/validators/remote/signature.ts deleted file mode 100644 index d3937b515..000000000 --- a/server/middlewares/validators/remote/signature.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { body } from 'express-validator/check' -import * as express from 'express' - -import { logger, isHostValid } from '../../../helpers' -import { checkErrors } from '../utils' - -const signatureValidator = [ - body('signature.host').custom(isHostValid).withMessage('Should have a signature host'), - body('signature.signature').not().isEmpty().withMessage('Should have a signature'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } }) - - checkErrors(req, res, next) - } -] - -// --------------------------------------------------------------------------- - -export { - signatureValidator -} diff --git a/server/middlewares/validators/remote/videos.ts b/server/middlewares/validators/remote/videos.ts deleted file mode 100644 index 497320cc1..000000000 --- a/server/middlewares/validators/remote/videos.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { body } from 'express-validator/check' -import * as express from 'express' - -import { - logger, - isArray, - removeBadRequestVideos, - removeBadRequestVideosQadu, - removeBadRequestVideosEvents -} from '../../../helpers' -import { checkErrors } from '../utils' - -const remoteVideosValidator = [ - body('data').custom(isArray), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking remoteVideos parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - removeBadRequestVideos(req.body.data) - - return next() - }) - } -] - -const remoteQaduVideosValidator = [ - body('data').custom(isArray), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - removeBadRequestVideosQadu(req.body.data) - - return next() - }) - } -] - -const remoteEventsVideosValidator = [ - body('data').custom(isArray), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - removeBadRequestVideosEvents(req.body.data) - - return next() - }) - } -] - -// --------------------------------------------------------------------------- - -export { - remoteVideosValidator, - remoteQaduVideosValidator, - remoteEventsVideosValidator -} diff --git a/server/models/account/account-follow-interface.ts b/server/models/account/account-follow-interface.ts new file mode 100644 index 000000000..3be383649 --- /dev/null +++ b/server/models/account/account-follow-interface.ts @@ -0,0 +1,23 @@ +import * as Sequelize from 'sequelize' +import * as Promise from 'bluebird' + +import { VideoRateType } from '../../../shared/models/videos/video-rate.type' + +export namespace AccountFollowMethods { +} + +export interface AccountFollowClass { +} + +export interface AccountFollowAttributes { + accountId: number + targetAccountId: number +} + +export interface AccountFollowInstance extends AccountFollowClass, AccountFollowAttributes, Sequelize.Instance { + id: number + createdAt: Date + updatedAt: Date +} + +export interface AccountFollowModel extends AccountFollowClass, Sequelize.Model {} diff --git a/server/models/account/account-follow.ts b/server/models/account/account-follow.ts new file mode 100644 index 000000000..9bf03b253 --- /dev/null +++ b/server/models/account/account-follow.ts @@ -0,0 +1,56 @@ +import * as Sequelize from 'sequelize' + +import { addMethodsToModel } from '../utils' +import { + AccountFollowInstance, + AccountFollowAttributes, + + AccountFollowMethods +} from './account-follow-interface' + +let AccountFollow: Sequelize.Model + +export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { + AccountFollow = sequelize.define('AccountFollow', + { }, + { + indexes: [ + { + fields: [ 'accountId' ], + unique: true + }, + { + fields: [ 'targetAccountId' ], + unique: true + } + ] + } + ) + + const classMethods = [ + associate + ] + addMethodsToModel(AccountFollow, classMethods) + + return AccountFollow +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + AccountFollow.belongsTo(models.Account, { + foreignKey: { + name: 'accountId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + + AccountFollow.belongsTo(models.Account, { + foreignKey: { + name: 'targetAccountId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} diff --git a/server/models/account/account-interface.ts b/server/models/account/account-interface.ts new file mode 100644 index 000000000..2ef3e2246 --- /dev/null +++ b/server/models/account/account-interface.ts @@ -0,0 +1,74 @@ +import * as Sequelize from 'sequelize' +import * as Bluebird from 'bluebird' + +import { PodInstance } from '../pod/pod-interface' +import { VideoChannelInstance } from '../video/video-channel-interface' +import { ActivityPubActor } from '../../../shared' +import { ResultList } from '../../../shared/models/result-list.model' + +export namespace AccountMethods { + export type Load = (id: number) => Bluebird + export type LoadByUUID = (uuid: string) => Bluebird + export type LoadByUrl = (url: string) => Bluebird + export type LoadAccountByPodAndUUID = (uuid: string, podId: number, transaction: Sequelize.Transaction) => Bluebird + export type LoadLocalAccountByName = (name: string) => Bluebird + export type ListOwned = () => Bluebird + export type ListFollowerUrlsForApi = (name: string, start: number, count: number) => Promise< ResultList > + export type ListFollowingUrlsForApi = (name: string, start: number, count: number) => Promise< ResultList > + + export type ToActivityPubObject = (this: AccountInstance) => ActivityPubActor + export type IsOwned = (this: AccountInstance) => boolean + export type GetFollowerSharedInboxUrls = (this: AccountInstance) => Bluebird + export type GetFollowingUrl = (this: AccountInstance) => string + export type GetFollowersUrl = (this: AccountInstance) => string + export type GetPublicKeyUrl = (this: AccountInstance) => string +} + +export interface AccountClass { + loadAccountByPodAndUUID: AccountMethods.LoadAccountByPodAndUUID + load: AccountMethods.Load + loadByUUID: AccountMethods.LoadByUUID + loadByUrl: AccountMethods.LoadByUrl + loadLocalAccountByName: AccountMethods.LoadLocalAccountByName + listOwned: AccountMethods.ListOwned + listFollowerUrlsForApi: AccountMethods.ListFollowerUrlsForApi + listFollowingUrlsForApi: AccountMethods.ListFollowingUrlsForApi +} + +export interface AccountAttributes { + name: string + url: string + publicKey: string + privateKey: string + followersCount: number + followingCount: number + inboxUrl: string + outboxUrl: string + sharedInboxUrl: string + followersUrl: string + followingUrl: string + + uuid?: string + + podId?: number + userId?: number + applicationId?: number +} + +export interface AccountInstance extends AccountClass, AccountAttributes, Sequelize.Instance { + isOwned: AccountMethods.IsOwned + toActivityPubObject: AccountMethods.ToActivityPubObject + getFollowerSharedInboxUrls: AccountMethods.GetFollowerSharedInboxUrls + getFollowingUrl: AccountMethods.GetFollowingUrl + getFollowersUrl: AccountMethods.GetFollowersUrl + getPublicKeyUrl: AccountMethods.GetPublicKeyUrl + + id: number + createdAt: Date + updatedAt: Date + + Pod: PodInstance + VideoChannels: VideoChannelInstance[] +} + +export interface AccountModel extends AccountClass, Sequelize.Model {} diff --git a/server/models/account/account-video-rate-interface.ts b/server/models/account/account-video-rate-interface.ts new file mode 100644 index 000000000..82cbe38cc --- /dev/null +++ b/server/models/account/account-video-rate-interface.ts @@ -0,0 +1,26 @@ +import * as Sequelize from 'sequelize' +import * as Promise from 'bluebird' + +import { VideoRateType } from '../../../shared/models/videos/video-rate.type' + +export namespace AccountVideoRateMethods { + export type Load = (accountId: number, videoId: number, transaction: Sequelize.Transaction) => Promise +} + +export interface AccountVideoRateClass { + load: AccountVideoRateMethods.Load +} + +export interface AccountVideoRateAttributes { + type: VideoRateType + accountId: number + videoId: number +} + +export interface AccountVideoRateInstance extends AccountVideoRateClass, AccountVideoRateAttributes, Sequelize.Instance { + id: number + createdAt: Date + updatedAt: Date +} + +export interface AccountVideoRateModel extends AccountVideoRateClass, Sequelize.Model {} diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts new file mode 100644 index 000000000..7f7c97606 --- /dev/null +++ b/server/models/account/account-video-rate.ts @@ -0,0 +1,78 @@ +/* + Account rates per video. +*/ +import { values } from 'lodash' +import * as Sequelize from 'sequelize' + +import { VIDEO_RATE_TYPES } from '../../initializers' + +import { addMethodsToModel } from '../utils' +import { + AccountVideoRateInstance, + AccountVideoRateAttributes, + + AccountVideoRateMethods +} from './account-video-rate-interface' + +let AccountVideoRate: Sequelize.Model +let load: AccountVideoRateMethods.Load + +export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { + AccountVideoRate = sequelize.define('AccountVideoRate', + { + type: { + type: DataTypes.ENUM(values(VIDEO_RATE_TYPES)), + allowNull: false + } + }, + { + indexes: [ + { + fields: [ 'videoId', 'accountId', 'type' ], + unique: true + } + ] + } + ) + + const classMethods = [ + associate, + + load + ] + addMethodsToModel(AccountVideoRate, classMethods) + + return AccountVideoRate +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + AccountVideoRate.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + + AccountVideoRate.belongsTo(models.Account, { + foreignKey: { + name: 'accountId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} + +load = function (accountId: number, videoId: number, transaction: Sequelize.Transaction) { + const options: Sequelize.FindOptions = { + where: { + accountId, + videoId + } + } + if (transaction) options.transaction = transaction + + return AccountVideoRate.findOne(options) +} diff --git a/server/models/account/account.ts b/server/models/account/account.ts new file mode 100644 index 000000000..00c0aefd4 --- /dev/null +++ b/server/models/account/account.ts @@ -0,0 +1,444 @@ +import * as Sequelize from 'sequelize' + +import { + isUserUsernameValid, + isAccountPublicKeyValid, + isAccountUrlValid, + isAccountPrivateKeyValid, + isAccountFollowersCountValid, + isAccountFollowingCountValid, + isAccountInboxValid, + isAccountOutboxValid, + isAccountSharedInboxValid, + isAccountFollowersValid, + isAccountFollowingValid, + activityPubContextify +} from '../../helpers' + +import { addMethodsToModel } from '../utils' +import { + AccountInstance, + AccountAttributes, + + AccountMethods +} from './account-interface' + +let Account: Sequelize.Model +let loadAccountByPodAndUUID: AccountMethods.LoadAccountByPodAndUUID +let load: AccountMethods.Load +let loadByUUID: AccountMethods.LoadByUUID +let loadByUrl: AccountMethods.LoadByUrl +let loadLocalAccountByName: AccountMethods.LoadLocalAccountByName +let listOwned: AccountMethods.ListOwned +let listFollowerUrlsForApi: AccountMethods.ListFollowerUrlsForApi +let listFollowingUrlsForApi: AccountMethods.ListFollowingUrlsForApi +let isOwned: AccountMethods.IsOwned +let toActivityPubObject: AccountMethods.ToActivityPubObject +let getFollowerSharedInboxUrls: AccountMethods.GetFollowerSharedInboxUrls +let getFollowingUrl: AccountMethods.GetFollowingUrl +let getFollowersUrl: AccountMethods.GetFollowersUrl +let getPublicKeyUrl: AccountMethods.GetPublicKeyUrl + +export default function defineAccount (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { + Account = sequelize.define('Account', + { + uuid: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + allowNull: false, + validate: { + isUUID: 4 + } + }, + name: { + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: value => { + const res = isUserUsernameValid(value) + if (res === false) throw new Error('Username is not valid.') + } + } + }, + url: { + type: DataTypes.STRING, + allowNull: false, + validate: { + urlValid: value => { + const res = isAccountUrlValid(value) + if (res === false) throw new Error('URL is not valid.') + } + } + }, + publicKey: { + type: DataTypes.STRING, + allowNull: false, + validate: { + publicKeyValid: value => { + const res = isAccountPublicKeyValid(value) + if (res === false) throw new Error('Public key is not valid.') + } + } + }, + privateKey: { + type: DataTypes.STRING, + allowNull: false, + validate: { + privateKeyValid: value => { + const res = isAccountPrivateKeyValid(value) + if (res === false) throw new Error('Private key is not valid.') + } + } + }, + followersCount: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + followersCountValid: value => { + const res = isAccountFollowersCountValid(value) + if (res === false) throw new Error('Followers count is not valid.') + } + } + }, + followingCount: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + followersCountValid: value => { + const res = isAccountFollowingCountValid(value) + if (res === false) throw new Error('Following count is not valid.') + } + } + }, + inboxUrl: { + type: DataTypes.STRING, + allowNull: false, + validate: { + inboxUrlValid: value => { + const res = isAccountInboxValid(value) + if (res === false) throw new Error('Inbox URL is not valid.') + } + } + }, + outboxUrl: { + type: DataTypes.STRING, + allowNull: false, + validate: { + outboxUrlValid: value => { + const res = isAccountOutboxValid(value) + if (res === false) throw new Error('Outbox URL is not valid.') + } + } + }, + sharedInboxUrl: { + type: DataTypes.STRING, + allowNull: false, + validate: { + sharedInboxUrlValid: value => { + const res = isAccountSharedInboxValid(value) + if (res === false) throw new Error('Shared inbox URL is not valid.') + } + } + }, + followersUrl: { + type: DataTypes.STRING, + allowNull: false, + validate: { + followersUrlValid: value => { + const res = isAccountFollowersValid(value) + if (res === false) throw new Error('Followers URL is not valid.') + } + } + }, + followingUrl: { + type: DataTypes.STRING, + allowNull: false, + validate: { + followingUrlValid: value => { + const res = isAccountFollowingValid(value) + if (res === false) throw new Error('Following URL is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'name' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'userId' ], + unique: true + }, + { + fields: [ 'applicationId' ], + unique: true + }, + { + fields: [ 'name', 'podId' ], + unique: true + } + ], + hooks: { afterDestroy } + } + ) + + const classMethods = [ + associate, + loadAccountByPodAndUUID, + load, + loadByUUID, + loadLocalAccountByName, + listOwned, + listFollowerUrlsForApi, + listFollowingUrlsForApi + ] + const instanceMethods = [ + isOwned, + toActivityPubObject, + getFollowerSharedInboxUrls, + getFollowingUrl, + getFollowersUrl, + getPublicKeyUrl + ] + addMethodsToModel(Account, classMethods, instanceMethods) + + return Account +} + +// --------------------------------------------------------------------------- + +function associate (models) { + Account.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: true + }, + onDelete: 'cascade' + }) + + Account.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: true + }, + onDelete: 'cascade' + }) + + Account.belongsTo(models.Application, { + foreignKey: { + name: 'userId', + allowNull: true + }, + onDelete: 'cascade' + }) + + Account.hasMany(models.VideoChannel, { + foreignKey: { + name: 'accountId', + allowNull: false + }, + onDelete: 'cascade', + hooks: true + }) + + Account.hasMany(models.AccountFollower, { + foreignKey: { + name: 'accountId', + allowNull: false + }, + onDelete: 'cascade' + }) + + Account.hasMany(models.AccountFollower, { + foreignKey: { + name: 'targetAccountId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function afterDestroy (account: AccountInstance) { + if (account.isOwned()) { + const removeVideoAccountToFriendsParams = { + uuid: account.uuid + } + + return removeVideoAccountToFriends(removeVideoAccountToFriendsParams) + } + + return undefined +} + +toActivityPubObject = function (this: AccountInstance) { + const type = this.podId ? 'Application' : 'Person' + + const json = { + type, + id: this.url, + following: this.getFollowingUrl(), + followers: this.getFollowersUrl(), + inbox: this.inboxUrl, + outbox: this.outboxUrl, + preferredUsername: this.name, + url: this.url, + name: this.name, + endpoints: { + sharedInbox: this.sharedInboxUrl + }, + uuid: this.uuid, + publicKey: { + id: this.getPublicKeyUrl(), + owner: this.url, + publicKeyPem: this.publicKey + } + } + + return activityPubContextify(json) +} + +isOwned = function (this: AccountInstance) { + return this.podId === null +} + +getFollowerSharedInboxUrls = function (this: AccountInstance) { + const query: Sequelize.FindOptions = { + attributes: [ 'sharedInboxUrl' ], + include: [ + { + model: Account['sequelize'].models.AccountFollower, + where: { + targetAccountId: this.id + } + } + ] + } + + return Account.findAll(query) + .then(accounts => accounts.map(a => a.sharedInboxUrl)) +} + +getFollowingUrl = function (this: AccountInstance) { + return this.url + '/followers' +} + +getFollowersUrl = function (this: AccountInstance) { + return this.url + '/followers' +} + +getPublicKeyUrl = function (this: AccountInstance) { + return this.url + '#main-key' +} + +// ------------------------------ STATICS ------------------------------ + +listOwned = function () { + const query: Sequelize.FindOptions = { + where: { + podId: null + } + } + + return Account.findAll(query) +} + +listFollowerUrlsForApi = function (name: string, start: number, count: number) { + return createListFollowForApiQuery('followers', name, start, count) +} + +listFollowingUrlsForApi = function (name: string, start: number, count: number) { + return createListFollowForApiQuery('following', name, start, count) +} + +load = function (id: number) { + return Account.findById(id) +} + +loadByUUID = function (uuid: string) { + const query: Sequelize.FindOptions = { + where: { + uuid + } + } + + return Account.findOne(query) +} + +loadLocalAccountByName = function (name: string) { + const query: Sequelize.FindOptions = { + where: { + name, + userId: { + [Sequelize.Op.ne]: null + } + } + } + + return Account.findOne(query) +} + +loadByUrl = function (url: string) { + const query: Sequelize.FindOptions = { + where: { + url + } + } + + return Account.findOne(query) +} + +loadAccountByPodAndUUID = function (uuid: string, podId: number, transaction: Sequelize.Transaction) { + const query: Sequelize.FindOptions = { + where: { + podId, + uuid + }, + transaction + } + + return Account.find(query) +} + +// ------------------------------ UTILS ------------------------------ + +async function createListFollowForApiQuery (type: 'followers' | 'following', name: string, start: number, count: number) { + let firstJoin: string + let secondJoin: string + + if (type === 'followers') { + firstJoin = 'targetAccountId' + secondJoin = 'accountId' + } else { + firstJoin = 'accountId' + secondJoin = 'targetAccountId' + } + + const selections = [ '"Followers"."url" AS "url"', 'COUNT(*) AS "total"' ] + const tasks: Promise[] = [] + + for (const selection of selections) { + const query = 'SELECT ' + selection + ' FROM "Account" ' + + 'INNER JOIN "AccountFollower" ON "AccountFollower"."' + firstJoin + '" = "Account"."id" ' + + 'INNER JOIN "Account" AS "Followers" ON "Followers"."id" = "AccountFollower"."' + secondJoin + '" ' + + 'WHERE "Account"."name" = \'$name\' ' + + 'LIMIT ' + start + ', ' + count + + const options = { + bind: { name }, + type: Sequelize.QueryTypes.SELECT + } + tasks.push(Account['sequelize'].query(query, options)) + } + + const [ followers, [ { total } ]] = await Promise.all(tasks) + const urls: string[] = followers.map(f => f.url) + + return { + data: urls, + total: parseInt(total, 10) + } +} diff --git a/server/models/account/index.ts b/server/models/account/index.ts new file mode 100644 index 000000000..179f66974 --- /dev/null +++ b/server/models/account/index.ts @@ -0,0 +1,4 @@ +export * from './account-interface' +export * from './account-follow-interface' +export * from './account-video-rate-interface' +export * from './user-interface' diff --git a/server/models/account/user-interface.ts b/server/models/account/user-interface.ts new file mode 100644 index 000000000..1a04fb750 --- /dev/null +++ b/server/models/account/user-interface.ts @@ -0,0 +1,69 @@ +import * as Sequelize from 'sequelize' +import * as Bluebird from 'bluebird' + +// Don't use barrel, import just what we need +import { AccountInstance } from './account-interface' +import { User as FormattedUser } from '../../../shared/models/users/user.model' +import { ResultList } from '../../../shared/models/result-list.model' +import { UserRight } from '../../../shared/models/users/user-right.enum' +import { UserRole } from '../../../shared/models/users/user-role' + +export namespace UserMethods { + export type HasRight = (this: UserInstance, right: UserRight) => boolean + export type IsPasswordMatch = (this: UserInstance, password: string) => Promise + + export type ToFormattedJSON = (this: UserInstance) => FormattedUser + export type IsAbleToUploadVideo = (this: UserInstance, videoFile: Express.Multer.File) => Promise + + export type CountTotal = () => Bluebird + + export type GetByUsername = (username: string) => Bluebird + + export type ListForApi = (start: number, count: number, sort: string) => Bluebird< ResultList > + + export type LoadById = (id: number) => Bluebird + + export type LoadByUsername = (username: string) => Bluebird + export type LoadByUsernameAndPopulateChannels = (username: string) => Bluebird + + export type LoadByUsernameOrEmail = (username: string, email: string) => Bluebird +} + +export interface UserClass { + isPasswordMatch: UserMethods.IsPasswordMatch, + toFormattedJSON: UserMethods.ToFormattedJSON, + hasRight: UserMethods.HasRight, + isAbleToUploadVideo: UserMethods.IsAbleToUploadVideo, + + countTotal: UserMethods.CountTotal, + getByUsername: UserMethods.GetByUsername, + listForApi: UserMethods.ListForApi, + loadById: UserMethods.LoadById, + loadByUsername: UserMethods.LoadByUsername, + loadByUsernameAndPopulateChannels: UserMethods.LoadByUsernameAndPopulateChannels, + loadByUsernameOrEmail: UserMethods.LoadByUsernameOrEmail +} + +export interface UserAttributes { + id?: number + password: string + username: string + email: string + displayNSFW?: boolean + role: UserRole + videoQuota: number + + Account?: AccountInstance +} + +export interface UserInstance extends UserClass, UserAttributes, Sequelize.Instance { + id: number + createdAt: Date + updatedAt: Date + + isPasswordMatch: UserMethods.IsPasswordMatch + toFormattedJSON: UserMethods.ToFormattedJSON + hasRight: UserMethods.HasRight +} + +export interface UserModel extends UserClass, Sequelize.Model {} diff --git a/server/models/account/user.ts b/server/models/account/user.ts new file mode 100644 index 000000000..1401762c5 --- /dev/null +++ b/server/models/account/user.ts @@ -0,0 +1,311 @@ +import * as Sequelize from 'sequelize' + +import { getSort, addMethodsToModel } from '../utils' +import { + cryptPassword, + comparePassword, + isUserPasswordValid, + isUserUsernameValid, + isUserDisplayNSFWValid, + isUserVideoQuotaValid, + isUserRoleValid +} from '../../helpers' +import { UserRight, USER_ROLE_LABELS, hasUserRight } from '../../../shared' + +import { + UserInstance, + UserAttributes, + + UserMethods +} from './user-interface' + +let User: Sequelize.Model +let isPasswordMatch: UserMethods.IsPasswordMatch +let hasRight: UserMethods.HasRight +let toFormattedJSON: UserMethods.ToFormattedJSON +let countTotal: UserMethods.CountTotal +let getByUsername: UserMethods.GetByUsername +let listForApi: UserMethods.ListForApi +let loadById: UserMethods.LoadById +let loadByUsername: UserMethods.LoadByUsername +let loadByUsernameAndPopulateChannels: UserMethods.LoadByUsernameAndPopulateChannels +let loadByUsernameOrEmail: UserMethods.LoadByUsernameOrEmail +let isAbleToUploadVideo: UserMethods.IsAbleToUploadVideo + +export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { + User = sequelize.define('User', + { + password: { + type: DataTypes.STRING, + allowNull: false, + validate: { + passwordValid: value => { + const res = isUserPasswordValid(value) + if (res === false) throw new Error('Password not valid.') + } + } + }, + username: { + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: value => { + const res = isUserUsernameValid(value) + if (res === false) throw new Error('Username not valid.') + } + } + }, + email: { + type: DataTypes.STRING(400), + allowNull: false, + validate: { + isEmail: true + } + }, + displayNSFW: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + validate: { + nsfwValid: value => { + const res = isUserDisplayNSFWValid(value) + if (res === false) throw new Error('Display NSFW is not valid.') + } + } + }, + role: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + roleValid: value => { + const res = isUserRoleValid(value) + if (res === false) throw new Error('Role is not valid.') + } + } + }, + videoQuota: { + type: DataTypes.BIGINT, + allowNull: false, + validate: { + videoQuotaValid: value => { + const res = isUserVideoQuotaValid(value) + if (res === false) throw new Error('Video quota is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'username' ], + unique: true + }, + { + fields: [ 'email' ], + unique: true + } + ], + hooks: { + beforeCreate: beforeCreateOrUpdate, + beforeUpdate: beforeCreateOrUpdate + } + } + ) + + const classMethods = [ + associate, + + countTotal, + getByUsername, + listForApi, + loadById, + loadByUsername, + loadByUsernameAndPopulateChannels, + loadByUsernameOrEmail + ] + const instanceMethods = [ + hasRight, + isPasswordMatch, + toFormattedJSON, + isAbleToUploadVideo + ] + addMethodsToModel(User, classMethods, instanceMethods) + + return User +} + +function beforeCreateOrUpdate (user: UserInstance) { + if (user.changed('password')) { + return cryptPassword(user.password) + .then(hash => { + user.password = hash + return undefined + }) + } +} + +// ------------------------------ METHODS ------------------------------ + +hasRight = function (this: UserInstance, right: UserRight) { + return hasUserRight(this.role, right) +} + +isPasswordMatch = function (this: UserInstance, password: string) { + return comparePassword(password, this.password) +} + +toFormattedJSON = function (this: UserInstance) { + const json = { + id: this.id, + username: this.username, + email: this.email, + displayNSFW: this.displayNSFW, + role: this.role, + roleLabel: USER_ROLE_LABELS[this.role], + videoQuota: this.videoQuota, + createdAt: this.createdAt, + author: { + id: this.Account.id, + uuid: this.Account.uuid + } + } + + if (Array.isArray(this.Account.VideoChannels) === true) { + const videoChannels = this.Account.VideoChannels + .map(c => c.toFormattedJSON()) + .sort((v1, v2) => { + if (v1.createdAt < v2.createdAt) return -1 + if (v1.createdAt === v2.createdAt) return 0 + + return 1 + }) + + json['videoChannels'] = videoChannels + } + + return json +} + +isAbleToUploadVideo = function (this: UserInstance, videoFile: Express.Multer.File) { + if (this.videoQuota === -1) return Promise.resolve(true) + + return getOriginalVideoFileTotalFromUser(this).then(totalBytes => { + return (videoFile.size + totalBytes) < this.videoQuota + }) +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + User.hasOne(models.Account, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + + User.hasMany(models.OAuthToken, { + foreignKey: 'userId', + onDelete: 'cascade' + }) +} + +countTotal = function () { + return this.count() +} + +getByUsername = function (username: string) { + const query = { + where: { + username: username + }, + include: [ { model: User['sequelize'].models.Account, required: true } ] + } + + return User.findOne(query) +} + +listForApi = function (start: number, count: number, sort: string) { + const query = { + offset: start, + limit: count, + order: [ getSort(sort) ], + include: [ { model: User['sequelize'].models.Account, required: true } ] + } + + return User.findAndCountAll(query).then(({ rows, count }) => { + return { + data: rows, + total: count + } + }) +} + +loadById = function (id: number) { + const options = { + include: [ { model: User['sequelize'].models.Account, required: true } ] + } + + return User.findById(id, options) +} + +loadByUsername = function (username: string) { + const query = { + where: { + username + }, + include: [ { model: User['sequelize'].models.Account, required: true } ] + } + + return User.findOne(query) +} + +loadByUsernameAndPopulateChannels = function (username: string) { + const query = { + where: { + username + }, + include: [ + { + model: User['sequelize'].models.Account, + required: true, + include: [ User['sequelize'].models.VideoChannel ] + } + ] + } + + return User.findOne(query) +} + +loadByUsernameOrEmail = function (username: string, email: string) { + const query = { + include: [ { model: User['sequelize'].models.Account, required: true } ], + where: { + [Sequelize.Op.or]: [ { username }, { email } ] + } + } + + // FIXME: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/18387 + return (User as any).findOne(query) +} + +// --------------------------------------------------------------------------- + +function getOriginalVideoFileTotalFromUser (user: UserInstance) { + // Don't use sequelize because we need to use a sub query + const query = 'SELECT SUM("size") AS "total" FROM ' + + '(SELECT MAX("VideoFiles"."size") AS "size" FROM "VideoFiles" ' + + 'INNER JOIN "Videos" ON "VideoFiles"."videoId" = "Videos"."id" ' + + 'INNER JOIN "VideoChannels" ON "VideoChannels"."id" = "Videos"."channelId" ' + + 'INNER JOIN "Accounts" ON "VideoChannels"."authorId" = "Accounts"."id" ' + + 'INNER JOIN "Users" ON "Accounts"."userId" = "Users"."id" ' + + 'WHERE "Users"."id" = $userId GROUP BY "Videos"."id") t' + + const options = { + bind: { userId: user.id }, + type: Sequelize.QueryTypes.SELECT + } + return User['sequelize'].query(query, options).then(([ { total } ]) => { + if (total === null) return 0 + + return parseInt(total, 10) + }) +} diff --git a/server/models/index.ts b/server/models/index.ts index b392a8a77..29479e067 100644 --- a/server/models/index.ts +++ b/server/models/index.ts @@ -3,5 +3,5 @@ export * from './job' export * from './oauth' export * from './pod' export * from './request' -export * from './user' +export * from './account' export * from './video' diff --git a/server/models/job/job-interface.ts b/server/models/job/job-interface.ts index ba5622977..163930a4f 100644 --- a/server/models/job/job-interface.ts +++ b/server/models/job/job-interface.ts @@ -1,14 +1,14 @@ import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' -import { JobState } from '../../../shared/models/job.model' +import { JobCategory, JobState } from '../../../shared/models/job.model' export namespace JobMethods { - export type ListWithLimit = (limit: number, state: JobState) => Promise + export type ListWithLimitByCategory = (limit: number, state: JobState, category: JobCategory) => Promise } export interface JobClass { - listWithLimit: JobMethods.ListWithLimit + listWithLimitByCategory: JobMethods.ListWithLimitByCategory } export interface JobAttributes { diff --git a/server/models/job/job.ts b/server/models/job/job.ts index 968f9d71d..ce1203e5a 100644 --- a/server/models/job/job.ts +++ b/server/models/job/job.ts @@ -1,7 +1,7 @@ import { values } from 'lodash' import * as Sequelize from 'sequelize' -import { JOB_STATES } from '../../initializers' +import { JOB_STATES, JOB_CATEGORIES } from '../../initializers' import { addMethodsToModel } from '../utils' import { @@ -13,7 +13,7 @@ import { import { JobState } from '../../../shared/models/job.model' let Job: Sequelize.Model -let listWithLimit: JobMethods.ListWithLimit +let listWithLimitByCategory: JobMethods.ListWithLimitByCategory export default function defineJob (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { Job = sequelize.define('Job', @@ -22,6 +22,10 @@ export default function defineJob (sequelize: Sequelize.Sequelize, DataTypes: Se type: DataTypes.ENUM(values(JOB_STATES)), allowNull: false }, + category: { + type: DataTypes.ENUM(values(JOB_CATEGORIES)), + allowNull: false + }, handlerName: { type: DataTypes.STRING, allowNull: false @@ -40,7 +44,7 @@ export default function defineJob (sequelize: Sequelize.Sequelize, DataTypes: Se } ) - const classMethods = [ listWithLimit ] + const classMethods = [ listWithLimitByCategory ] addMethodsToModel(Job, classMethods) return Job @@ -48,7 +52,7 @@ export default function defineJob (sequelize: Sequelize.Sequelize, DataTypes: Se // --------------------------------------------------------------------------- -listWithLimit = function (limit: number, state: JobState) { +listWithLimitByCategory = function (limit: number, state: JobState) { const query = { order: [ [ 'id', 'ASC' ] diff --git a/server/models/oauth/oauth-token-interface.ts b/server/models/oauth/oauth-token-interface.ts index 0c947bde8..ef97893c4 100644 --- a/server/models/oauth/oauth-token-interface.ts +++ b/server/models/oauth/oauth-token-interface.ts @@ -1,7 +1,7 @@ import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' -import { UserModel } from '../user/user-interface' +import { UserModel } from '../account/user-interface' export type OAuthTokenInfo = { refreshToken: string diff --git a/server/models/pod/pod-interface.ts b/server/models/pod/pod-interface.ts index 7e095d424..6c5aab3fa 100644 --- a/server/models/pod/pod-interface.ts +++ b/server/models/pod/pod-interface.ts @@ -48,9 +48,7 @@ export interface PodClass { export interface PodAttributes { id?: number host?: string - publicKey?: string score?: number | Sequelize.literal // Sequelize literal for 'score +' + value - email?: string } export interface PodInstance extends PodClass, PodAttributes, Sequelize.Instance { diff --git a/server/models/pod/pod.ts b/server/models/pod/pod.ts index 6b33336b8..7c8b49bf8 100644 --- a/server/models/pod/pod.ts +++ b/server/models/pod/pod.ts @@ -39,10 +39,6 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da } } }, - publicKey: { - type: DataTypes.STRING(5000), - allowNull: false - }, score: { type: DataTypes.INTEGER, defaultValue: FRIEND_SCORE.BASE, @@ -51,13 +47,6 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da isInt: true, max: FRIEND_SCORE.MAX } - }, - email: { - type: DataTypes.STRING(400), - allowNull: false, - validate: { - isEmail: true - } } }, { @@ -100,7 +89,6 @@ toFormattedJSON = function (this: PodInstance) { const json = { id: this.id, host: this.host, - email: this.email, score: this.score as number, createdAt: this.createdAt } diff --git a/server/models/user/index.ts b/server/models/user/index.ts deleted file mode 100644 index ed3689518..000000000 --- a/server/models/user/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './user-video-rate-interface' -export * from './user-interface' diff --git a/server/models/user/user-interface.ts b/server/models/user/user-interface.ts deleted file mode 100644 index 49c75aa3b..000000000 --- a/server/models/user/user-interface.ts +++ /dev/null @@ -1,69 +0,0 @@ -import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' - -// Don't use barrel, import just what we need -import { User as FormattedUser } from '../../../shared/models/users/user.model' -import { ResultList } from '../../../shared/models/result-list.model' -import { AuthorInstance } from '../video/author-interface' -import { UserRight } from '../../../shared/models/users/user-right.enum' -import { UserRole } from '../../../shared/models/users/user-role' - -export namespace UserMethods { - export type HasRight = (this: UserInstance, right: UserRight) => boolean - export type IsPasswordMatch = (this: UserInstance, password: string) => Promise - - export type ToFormattedJSON = (this: UserInstance) => FormattedUser - export type IsAbleToUploadVideo = (this: UserInstance, videoFile: Express.Multer.File) => Promise - - export type CountTotal = () => Promise - - export type GetByUsername = (username: string) => Promise - - export type ListForApi = (start: number, count: number, sort: string) => Promise< ResultList > - - export type LoadById = (id: number) => Promise - - export type LoadByUsername = (username: string) => Promise - export type LoadByUsernameAndPopulateChannels = (username: string) => Promise - - export type LoadByUsernameOrEmail = (username: string, email: string) => Promise -} - -export interface UserClass { - isPasswordMatch: UserMethods.IsPasswordMatch, - toFormattedJSON: UserMethods.ToFormattedJSON, - hasRight: UserMethods.HasRight, - isAbleToUploadVideo: UserMethods.IsAbleToUploadVideo, - - countTotal: UserMethods.CountTotal, - getByUsername: UserMethods.GetByUsername, - listForApi: UserMethods.ListForApi, - loadById: UserMethods.LoadById, - loadByUsername: UserMethods.LoadByUsername, - loadByUsernameAndPopulateChannels: UserMethods.LoadByUsernameAndPopulateChannels, - loadByUsernameOrEmail: UserMethods.LoadByUsernameOrEmail -} - -export interface UserAttributes { - id?: number - password: string - username: string - email: string - displayNSFW?: boolean - role: UserRole - videoQuota: number - - Author?: AuthorInstance -} - -export interface UserInstance extends UserClass, UserAttributes, Sequelize.Instance { - id: number - createdAt: Date - updatedAt: Date - - isPasswordMatch: UserMethods.IsPasswordMatch - toFormattedJSON: UserMethods.ToFormattedJSON - hasRight: UserMethods.HasRight -} - -export interface UserModel extends UserClass, Sequelize.Model {} diff --git a/server/models/user/user-video-rate-interface.ts b/server/models/user/user-video-rate-interface.ts deleted file mode 100644 index ea0fdc4d9..000000000 --- a/server/models/user/user-video-rate-interface.ts +++ /dev/null @@ -1,26 +0,0 @@ -import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' - -import { VideoRateType } from '../../../shared/models/videos/video-rate.type' - -export namespace UserVideoRateMethods { - export type Load = (userId: number, videoId: number, transaction: Sequelize.Transaction) => Promise -} - -export interface UserVideoRateClass { - load: UserVideoRateMethods.Load -} - -export interface UserVideoRateAttributes { - type: VideoRateType - userId: number - videoId: number -} - -export interface UserVideoRateInstance extends UserVideoRateClass, UserVideoRateAttributes, Sequelize.Instance { - id: number - createdAt: Date - updatedAt: Date -} - -export interface UserVideoRateModel extends UserVideoRateClass, Sequelize.Model {} diff --git a/server/models/user/user-video-rate.ts b/server/models/user/user-video-rate.ts deleted file mode 100644 index 7d6dd7281..000000000 --- a/server/models/user/user-video-rate.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - User rates per video. -*/ -import { values } from 'lodash' -import * as Sequelize from 'sequelize' - -import { VIDEO_RATE_TYPES } from '../../initializers' - -import { addMethodsToModel } from '../utils' -import { - UserVideoRateInstance, - UserVideoRateAttributes, - - UserVideoRateMethods -} from './user-video-rate-interface' - -let UserVideoRate: Sequelize.Model -let load: UserVideoRateMethods.Load - -export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { - UserVideoRate = sequelize.define('UserVideoRate', - { - type: { - type: DataTypes.ENUM(values(VIDEO_RATE_TYPES)), - allowNull: false - } - }, - { - indexes: [ - { - fields: [ 'videoId', 'userId', 'type' ], - unique: true - } - ] - } - ) - - const classMethods = [ - associate, - - load - ] - addMethodsToModel(UserVideoRate, classMethods) - - return UserVideoRate -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - UserVideoRate.belongsTo(models.Video, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'CASCADE' - }) - - UserVideoRate.belongsTo(models.User, { - foreignKey: { - name: 'userId', - allowNull: false - }, - onDelete: 'CASCADE' - }) -} - -load = function (userId: number, videoId: number, transaction: Sequelize.Transaction) { - const options: Sequelize.FindOptions = { - where: { - userId, - videoId - } - } - if (transaction) options.transaction = transaction - - return UserVideoRate.findOne(options) -} diff --git a/server/models/user/user.ts b/server/models/user/user.ts deleted file mode 100644 index b974418d4..000000000 --- a/server/models/user/user.ts +++ /dev/null @@ -1,312 +0,0 @@ -import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' - -import { getSort, addMethodsToModel } from '../utils' -import { - cryptPassword, - comparePassword, - isUserPasswordValid, - isUserUsernameValid, - isUserDisplayNSFWValid, - isUserVideoQuotaValid, - isUserRoleValid -} from '../../helpers' -import { UserRight, USER_ROLE_LABELS, hasUserRight } from '../../../shared' - -import { - UserInstance, - UserAttributes, - - UserMethods -} from './user-interface' - -let User: Sequelize.Model -let isPasswordMatch: UserMethods.IsPasswordMatch -let hasRight: UserMethods.HasRight -let toFormattedJSON: UserMethods.ToFormattedJSON -let countTotal: UserMethods.CountTotal -let getByUsername: UserMethods.GetByUsername -let listForApi: UserMethods.ListForApi -let loadById: UserMethods.LoadById -let loadByUsername: UserMethods.LoadByUsername -let loadByUsernameAndPopulateChannels: UserMethods.LoadByUsernameAndPopulateChannels -let loadByUsernameOrEmail: UserMethods.LoadByUsernameOrEmail -let isAbleToUploadVideo: UserMethods.IsAbleToUploadVideo - -export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { - User = sequelize.define('User', - { - password: { - type: DataTypes.STRING, - allowNull: false, - validate: { - passwordValid: value => { - const res = isUserPasswordValid(value) - if (res === false) throw new Error('Password not valid.') - } - } - }, - username: { - type: DataTypes.STRING, - allowNull: false, - validate: { - usernameValid: value => { - const res = isUserUsernameValid(value) - if (res === false) throw new Error('Username not valid.') - } - } - }, - email: { - type: DataTypes.STRING(400), - allowNull: false, - validate: { - isEmail: true - } - }, - displayNSFW: { - type: DataTypes.BOOLEAN, - allowNull: false, - defaultValue: false, - validate: { - nsfwValid: value => { - const res = isUserDisplayNSFWValid(value) - if (res === false) throw new Error('Display NSFW is not valid.') - } - } - }, - role: { - type: DataTypes.INTEGER, - allowNull: false, - validate: { - roleValid: value => { - const res = isUserRoleValid(value) - if (res === false) throw new Error('Role is not valid.') - } - } - }, - videoQuota: { - type: DataTypes.BIGINT, - allowNull: false, - validate: { - videoQuotaValid: value => { - const res = isUserVideoQuotaValid(value) - if (res === false) throw new Error('Video quota is not valid.') - } - } - } - }, - { - indexes: [ - { - fields: [ 'username' ], - unique: true - }, - { - fields: [ 'email' ], - unique: true - } - ], - hooks: { - beforeCreate: beforeCreateOrUpdate, - beforeUpdate: beforeCreateOrUpdate - } - } - ) - - const classMethods = [ - associate, - - countTotal, - getByUsername, - listForApi, - loadById, - loadByUsername, - loadByUsernameAndPopulateChannels, - loadByUsernameOrEmail - ] - const instanceMethods = [ - hasRight, - isPasswordMatch, - toFormattedJSON, - isAbleToUploadVideo - ] - addMethodsToModel(User, classMethods, instanceMethods) - - return User -} - -function beforeCreateOrUpdate (user: UserInstance) { - if (user.changed('password')) { - return cryptPassword(user.password) - .then(hash => { - user.password = hash - return undefined - }) - } -} - -// ------------------------------ METHODS ------------------------------ - -hasRight = function (this: UserInstance, right: UserRight) { - return hasUserRight(this.role, right) -} - -isPasswordMatch = function (this: UserInstance, password: string) { - return comparePassword(password, this.password) -} - -toFormattedJSON = function (this: UserInstance) { - const json = { - id: this.id, - username: this.username, - email: this.email, - displayNSFW: this.displayNSFW, - role: this.role, - roleLabel: USER_ROLE_LABELS[this.role], - videoQuota: this.videoQuota, - createdAt: this.createdAt, - author: { - id: this.Author.id, - uuid: this.Author.uuid - } - } - - if (Array.isArray(this.Author.VideoChannels) === true) { - const videoChannels = this.Author.VideoChannels - .map(c => c.toFormattedJSON()) - .sort((v1, v2) => { - if (v1.createdAt < v2.createdAt) return -1 - if (v1.createdAt === v2.createdAt) return 0 - - return 1 - }) - - json['videoChannels'] = videoChannels - } - - return json -} - -isAbleToUploadVideo = function (this: UserInstance, videoFile: Express.Multer.File) { - if (this.videoQuota === -1) return Promise.resolve(true) - - return getOriginalVideoFileTotalFromUser(this).then(totalBytes => { - return (videoFile.size + totalBytes) < this.videoQuota - }) -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - User.hasOne(models.Author, { - foreignKey: 'userId', - onDelete: 'cascade' - }) - - User.hasMany(models.OAuthToken, { - foreignKey: 'userId', - onDelete: 'cascade' - }) -} - -countTotal = function () { - return this.count() -} - -getByUsername = function (username: string) { - const query = { - where: { - username: username - }, - include: [ { model: User['sequelize'].models.Author, required: true } ] - } - - return User.findOne(query) -} - -listForApi = function (start: number, count: number, sort: string) { - const query = { - offset: start, - limit: count, - order: [ getSort(sort) ], - include: [ { model: User['sequelize'].models.Author, required: true } ] - } - - return User.findAndCountAll(query).then(({ rows, count }) => { - return { - data: rows, - total: count - } - }) -} - -loadById = function (id: number) { - const options = { - include: [ { model: User['sequelize'].models.Author, required: true } ] - } - - return User.findById(id, options) -} - -loadByUsername = function (username: string) { - const query = { - where: { - username - }, - include: [ { model: User['sequelize'].models.Author, required: true } ] - } - - return User.findOne(query) -} - -loadByUsernameAndPopulateChannels = function (username: string) { - const query = { - where: { - username - }, - include: [ - { - model: User['sequelize'].models.Author, - required: true, - include: [ User['sequelize'].models.VideoChannel ] - } - ] - } - - return User.findOne(query) -} - -loadByUsernameOrEmail = function (username: string, email: string) { - const query = { - include: [ { model: User['sequelize'].models.Author, required: true } ], - where: { - [Sequelize.Op.or]: [ { username }, { email } ] - } - } - - // FIXME: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/18387 - return (User as any).findOne(query) -} - -// --------------------------------------------------------------------------- - -function getOriginalVideoFileTotalFromUser (user: UserInstance) { - // Don't use sequelize because we need to use a sub query - const query = 'SELECT SUM("size") AS "total" FROM ' + - '(SELECT MAX("VideoFiles"."size") AS "size" FROM "VideoFiles" ' + - 'INNER JOIN "Videos" ON "VideoFiles"."videoId" = "Videos"."id" ' + - 'INNER JOIN "VideoChannels" ON "VideoChannels"."id" = "Videos"."channelId" ' + - 'INNER JOIN "Authors" ON "VideoChannels"."authorId" = "Authors"."id" ' + - 'INNER JOIN "Users" ON "Authors"."userId" = "Users"."id" ' + - 'WHERE "Users"."id" = $userId GROUP BY "Videos"."id") t' - - const options = { - bind: { userId: user.id }, - type: Sequelize.QueryTypes.SELECT - } - return User['sequelize'].query(query, options).then(([ { total } ]) => { - if (total === null) return 0 - - return parseInt(total, 10) - }) -} diff --git a/server/models/video/author-interface.ts b/server/models/video/author-interface.ts deleted file mode 100644 index fc69ff3c2..000000000 --- a/server/models/video/author-interface.ts +++ /dev/null @@ -1,45 +0,0 @@ -import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' - -import { PodInstance } from '../pod/pod-interface' -import { RemoteVideoAuthorCreateData } from '../../../shared/models/pods/remote-video/remote-video-author-create-request.model' -import { VideoChannelInstance } from './video-channel-interface' - -export namespace AuthorMethods { - export type Load = (id: number) => Promise - export type LoadByUUID = (uuid: string) => Promise - export type LoadAuthorByPodAndUUID = (uuid: string, podId: number, transaction: Sequelize.Transaction) => Promise - export type ListOwned = () => Promise - - export type ToAddRemoteJSON = (this: AuthorInstance) => RemoteVideoAuthorCreateData - export type IsOwned = (this: AuthorInstance) => boolean -} - -export interface AuthorClass { - loadAuthorByPodAndUUID: AuthorMethods.LoadAuthorByPodAndUUID - load: AuthorMethods.Load - loadByUUID: AuthorMethods.LoadByUUID - listOwned: AuthorMethods.ListOwned -} - -export interface AuthorAttributes { - name: string - uuid?: string - - podId?: number - userId?: number -} - -export interface AuthorInstance extends AuthorClass, AuthorAttributes, Sequelize.Instance { - isOwned: AuthorMethods.IsOwned - toAddRemoteJSON: AuthorMethods.ToAddRemoteJSON - - id: number - createdAt: Date - updatedAt: Date - - Pod: PodInstance - VideoChannels: VideoChannelInstance[] -} - -export interface AuthorModel extends AuthorClass, Sequelize.Model {} diff --git a/server/models/video/author.ts b/server/models/video/author.ts deleted file mode 100644 index 43f84c3ea..000000000 --- a/server/models/video/author.ts +++ /dev/null @@ -1,171 +0,0 @@ -import * as Sequelize from 'sequelize' - -import { isUserUsernameValid } from '../../helpers' -import { removeVideoAuthorToFriends } from '../../lib' - -import { addMethodsToModel } from '../utils' -import { - AuthorInstance, - AuthorAttributes, - - AuthorMethods -} from './author-interface' - -let Author: Sequelize.Model -let loadAuthorByPodAndUUID: AuthorMethods.LoadAuthorByPodAndUUID -let load: AuthorMethods.Load -let loadByUUID: AuthorMethods.LoadByUUID -let listOwned: AuthorMethods.ListOwned -let isOwned: AuthorMethods.IsOwned -let toAddRemoteJSON: AuthorMethods.ToAddRemoteJSON - -export default function defineAuthor (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { - Author = sequelize.define('Author', - { - uuid: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - allowNull: false, - validate: { - isUUID: 4 - } - }, - name: { - type: DataTypes.STRING, - allowNull: false, - validate: { - usernameValid: value => { - const res = isUserUsernameValid(value) - if (res === false) throw new Error('Username is not valid.') - } - } - } - }, - { - indexes: [ - { - fields: [ 'name' ] - }, - { - fields: [ 'podId' ] - }, - { - fields: [ 'userId' ], - unique: true - }, - { - fields: [ 'name', 'podId' ], - unique: true - } - ], - hooks: { afterDestroy } - } - ) - - const classMethods = [ - associate, - loadAuthorByPodAndUUID, - load, - loadByUUID, - listOwned - ] - const instanceMethods = [ - isOwned, - toAddRemoteJSON - ] - addMethodsToModel(Author, classMethods, instanceMethods) - - return Author -} - -// --------------------------------------------------------------------------- - -function associate (models) { - Author.belongsTo(models.Pod, { - foreignKey: { - name: 'podId', - allowNull: true - }, - onDelete: 'cascade' - }) - - Author.belongsTo(models.User, { - foreignKey: { - name: 'userId', - allowNull: true - }, - onDelete: 'cascade' - }) - - Author.hasMany(models.VideoChannel, { - foreignKey: { - name: 'authorId', - allowNull: false - }, - onDelete: 'cascade', - hooks: true - }) -} - -function afterDestroy (author: AuthorInstance) { - if (author.isOwned()) { - const removeVideoAuthorToFriendsParams = { - uuid: author.uuid - } - - return removeVideoAuthorToFriends(removeVideoAuthorToFriendsParams) - } - - return undefined -} - -toAddRemoteJSON = function (this: AuthorInstance) { - const json = { - uuid: this.uuid, - name: this.name - } - - return json -} - -isOwned = function (this: AuthorInstance) { - return this.podId === null -} - -// ------------------------------ STATICS ------------------------------ - -listOwned = function () { - const query: Sequelize.FindOptions = { - where: { - podId: null - } - } - - return Author.findAll(query) -} - -load = function (id: number) { - return Author.findById(id) -} - -loadByUUID = function (uuid: string) { - const query: Sequelize.FindOptions = { - where: { - uuid - } - } - - return Author.findOne(query) -} - -loadAuthorByPodAndUUID = function (uuid: string, podId: number, transaction: Sequelize.Transaction) { - const query: Sequelize.FindOptions = { - where: { - podId, - uuid - }, - transaction - } - - return Author.find(query) -} diff --git a/server/models/video/video-channel-interface.ts b/server/models/video/video-channel-interface.ts index b8d3e0f42..477f97cd4 100644 --- a/server/models/video/video-channel-interface.ts +++ b/server/models/video/video-channel-interface.ts @@ -1,42 +1,42 @@ import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' -import { ResultList, RemoteVideoChannelCreateData, RemoteVideoChannelUpdateData } from '../../../shared' +import { ResultList } from '../../../shared' // Don't use barrel, import just what we need import { VideoChannel as FormattedVideoChannel } from '../../../shared/models/videos/video-channel.model' -import { AuthorInstance } from './author-interface' import { VideoInstance } from './video-interface' +import { AccountInstance } from '../account/account-interface' +import { VideoChannelObject } from '../../../shared/models/activitypub/objects/video-channel-object' export namespace VideoChannelMethods { export type ToFormattedJSON = (this: VideoChannelInstance) => FormattedVideoChannel - export type ToAddRemoteJSON = (this: VideoChannelInstance) => RemoteVideoChannelCreateData - export type ToUpdateRemoteJSON = (this: VideoChannelInstance) => RemoteVideoChannelUpdateData + export type ToActivityPubObject = (this: VideoChannelInstance) => VideoChannelObject export type IsOwned = (this: VideoChannelInstance) => boolean - export type CountByAuthor = (authorId: number) => Promise + export type CountByAccount = (accountId: number) => Promise export type ListOwned = () => Promise export type ListForApi = (start: number, count: number, sort: string) => Promise< ResultList > - export type LoadByIdAndAuthor = (id: number, authorId: number) => Promise - export type ListByAuthor = (authorId: number) => Promise< ResultList > - export type LoadAndPopulateAuthor = (id: number) => Promise - export type LoadByUUIDAndPopulateAuthor = (uuid: string) => Promise + export type LoadByIdAndAccount = (id: number, accountId: number) => Promise + export type ListByAccount = (accountId: number) => Promise< ResultList > + export type LoadAndPopulateAccount = (id: number) => Promise + export type LoadByUUIDAndPopulateAccount = (uuid: string) => Promise export type LoadByUUID = (uuid: string, t?: Sequelize.Transaction) => Promise export type LoadByHostAndUUID = (uuid: string, podHost: string, t?: Sequelize.Transaction) => Promise - export type LoadAndPopulateAuthorAndVideos = (id: number) => Promise + export type LoadAndPopulateAccountAndVideos = (id: number) => Promise } export interface VideoChannelClass { - countByAuthor: VideoChannelMethods.CountByAuthor + countByAccount: VideoChannelMethods.CountByAccount listForApi: VideoChannelMethods.ListForApi - listByAuthor: VideoChannelMethods.ListByAuthor + listByAccount: VideoChannelMethods.ListByAccount listOwned: VideoChannelMethods.ListOwned - loadByIdAndAuthor: VideoChannelMethods.LoadByIdAndAuthor + loadByIdAndAccount: VideoChannelMethods.LoadByIdAndAccount loadByUUID: VideoChannelMethods.LoadByUUID loadByHostAndUUID: VideoChannelMethods.LoadByHostAndUUID - loadAndPopulateAuthor: VideoChannelMethods.LoadAndPopulateAuthor - loadByUUIDAndPopulateAuthor: VideoChannelMethods.LoadByUUIDAndPopulateAuthor - loadAndPopulateAuthorAndVideos: VideoChannelMethods.LoadAndPopulateAuthorAndVideos + loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount + loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount + loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos } export interface VideoChannelAttributes { @@ -45,8 +45,9 @@ export interface VideoChannelAttributes { name: string description: string remote: boolean + url: string - Author?: AuthorInstance + Account?: AccountInstance Videos?: VideoInstance[] } @@ -57,8 +58,7 @@ export interface VideoChannelInstance extends VideoChannelClass, VideoChannelAtt isOwned: VideoChannelMethods.IsOwned toFormattedJSON: VideoChannelMethods.ToFormattedJSON - toAddRemoteJSON: VideoChannelMethods.ToAddRemoteJSON - toUpdateRemoteJSON: VideoChannelMethods.ToUpdateRemoteJSON + toActivityPubObject: VideoChannelMethods.ToActivityPubObject } export interface VideoChannelModel extends VideoChannelClass, Sequelize.Model {} diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 46c2db63f..c17828f3e 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -13,19 +13,18 @@ import { let VideoChannel: Sequelize.Model let toFormattedJSON: VideoChannelMethods.ToFormattedJSON -let toAddRemoteJSON: VideoChannelMethods.ToAddRemoteJSON -let toUpdateRemoteJSON: VideoChannelMethods.ToUpdateRemoteJSON +let toActivityPubObject: VideoChannelMethods.ToActivityPubObject let isOwned: VideoChannelMethods.IsOwned -let countByAuthor: VideoChannelMethods.CountByAuthor +let countByAccount: VideoChannelMethods.CountByAccount let listOwned: VideoChannelMethods.ListOwned let listForApi: VideoChannelMethods.ListForApi -let listByAuthor: VideoChannelMethods.ListByAuthor -let loadByIdAndAuthor: VideoChannelMethods.LoadByIdAndAuthor +let listByAccount: VideoChannelMethods.ListByAccount +let loadByIdAndAccount: VideoChannelMethods.LoadByIdAndAccount let loadByUUID: VideoChannelMethods.LoadByUUID -let loadAndPopulateAuthor: VideoChannelMethods.LoadAndPopulateAuthor -let loadByUUIDAndPopulateAuthor: VideoChannelMethods.LoadByUUIDAndPopulateAuthor +let loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount +let loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount let loadByHostAndUUID: VideoChannelMethods.LoadByHostAndUUID -let loadAndPopulateAuthorAndVideos: VideoChannelMethods.LoadAndPopulateAuthorAndVideos +let loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { VideoChannel = sequelize.define('VideoChannel', @@ -62,12 +61,19 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false + }, + url: { + type: DataTypes.STRING, + allowNull: false, + validate: { + isUrl: true + } } }, { indexes: [ { - fields: [ 'authorId' ] + fields: [ 'accountId' ] } ], hooks: { @@ -80,21 +86,20 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da associate, listForApi, - listByAuthor, + listByAccount, listOwned, - loadByIdAndAuthor, - loadAndPopulateAuthor, - loadByUUIDAndPopulateAuthor, + loadByIdAndAccount, + loadAndPopulateAccount, + loadByUUIDAndPopulateAccount, loadByUUID, loadByHostAndUUID, - loadAndPopulateAuthorAndVideos, - countByAuthor + loadAndPopulateAccountAndVideos, + countByAccount ] const instanceMethods = [ isOwned, toFormattedJSON, - toAddRemoteJSON, - toUpdateRemoteJSON + toActivityPubObject, ] addMethodsToModel(VideoChannel, classMethods, instanceMethods) @@ -118,10 +123,10 @@ toFormattedJSON = function (this: VideoChannelInstance) { updatedAt: this.updatedAt } - if (this.Author !== undefined) { + if (this.Account !== undefined) { json['owner'] = { - name: this.Author.name, - uuid: this.Author.uuid + name: this.Account.name, + uuid: this.Account.uuid } } @@ -132,27 +137,14 @@ toFormattedJSON = function (this: VideoChannelInstance) { return json } -toAddRemoteJSON = function (this: VideoChannelInstance) { - const json = { - uuid: this.uuid, - name: this.name, - description: this.description, - createdAt: this.createdAt, - updatedAt: this.updatedAt, - ownerUUID: this.Author.uuid - } - - return json -} - -toUpdateRemoteJSON = function (this: VideoChannelInstance) { +toActivityPubObject = function (this: VideoChannelInstance) { const json = { uuid: this.uuid, name: this.name, description: this.description, createdAt: this.createdAt, updatedAt: this.updatedAt, - ownerUUID: this.Author.uuid + ownerUUID: this.Account.uuid } return json @@ -161,9 +153,9 @@ toUpdateRemoteJSON = function (this: VideoChannelInstance) { // ------------------------------ STATICS ------------------------------ function associate (models) { - VideoChannel.belongsTo(models.Author, { + VideoChannel.belongsTo(models.Account, { foreignKey: { - name: 'authorId', + name: 'accountId', allowNull: false }, onDelete: 'CASCADE' @@ -190,10 +182,10 @@ function afterDestroy (videoChannel: VideoChannelInstance) { return undefined } -countByAuthor = function (authorId: number) { +countByAccount = function (accountId: number) { const query = { where: { - authorId + accountId } } @@ -205,7 +197,7 @@ listOwned = function () { where: { remote: false }, - include: [ VideoChannel['sequelize'].models.Author ] + include: [ VideoChannel['sequelize'].models.Account ] } return VideoChannel.findAll(query) @@ -218,7 +210,7 @@ listForApi = function (start: number, count: number, sort: string) { order: [ getSort(sort) ], include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, required: true, include: [ { model: VideoChannel['sequelize'].models.Pod, required: false } ] } @@ -230,14 +222,14 @@ listForApi = function (start: number, count: number, sort: string) { }) } -listByAuthor = function (authorId: number) { +listByAccount = function (accountId: number) { const query = { order: [ getSort('createdAt') ], include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, where: { - id: authorId + id: accountId }, required: true, include: [ { model: VideoChannel['sequelize'].models.Pod, required: false } ] @@ -269,7 +261,7 @@ loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Tran }, include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, include: [ { model: VideoChannel['sequelize'].models.Pod, @@ -288,15 +280,15 @@ loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Tran return VideoChannel.findOne(query) } -loadByIdAndAuthor = function (id: number, authorId: number) { +loadByIdAndAccount = function (id: number, accountId: number) { const options = { where: { id, - authorId + accountId }, include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, include: [ { model: VideoChannel['sequelize'].models.Pod, required: false } ] } ] @@ -305,11 +297,11 @@ loadByIdAndAuthor = function (id: number, authorId: number) { return VideoChannel.findOne(options) } -loadAndPopulateAuthor = function (id: number) { +loadAndPopulateAccount = function (id: number) { const options = { include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, include: [ { model: VideoChannel['sequelize'].models.Pod, required: false } ] } ] @@ -318,14 +310,14 @@ loadAndPopulateAuthor = function (id: number) { return VideoChannel.findById(id, options) } -loadByUUIDAndPopulateAuthor = function (uuid: string) { +loadByUUIDAndPopulateAccount = function (uuid: string) { const options = { where: { uuid }, include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, include: [ { model: VideoChannel['sequelize'].models.Pod, required: false } ] } ] @@ -334,11 +326,11 @@ loadByUUIDAndPopulateAuthor = function (uuid: string) { return VideoChannel.findOne(options) } -loadAndPopulateAuthorAndVideos = function (id: number) { +loadAndPopulateAccountAndVideos = function (id: number) { const options = { include: [ { - model: VideoChannel['sequelize'].models.Author, + model: VideoChannel['sequelize'].models.Account, include: [ { model: VideoChannel['sequelize'].models.Pod, required: false } ] }, VideoChannel['sequelize'].models.Video diff --git a/server/models/video/video-interface.ts b/server/models/video/video-interface.ts index cfe65f9aa..e62e25a82 100644 --- a/server/models/video/video-interface.ts +++ b/server/models/video/video-interface.ts @@ -1,5 +1,5 @@ import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' +import * as Bluebird from 'bluebird' import { TagAttributes, TagInstance } from './tag-interface' import { VideoFileAttributes, VideoFileInstance } from './video-file-interface' @@ -13,6 +13,7 @@ import { RemoteVideoUpdateData } from '../../../shared/models/pods/remote-video/ import { RemoteVideoCreateData } from '../../../shared/models/pods/remote-video/remote-video-create-request.model' import { ResultList } from '../../../shared/models/result-list.model' import { VideoChannelInstance } from './video-channel-interface' +import { VideoTorrentObject } from '../../../shared/models/activitypub/objects/video-torrent-object' export namespace VideoMethods { export type GetThumbnailName = (this: VideoInstance) => string @@ -29,8 +30,7 @@ export namespace VideoMethods { export type GetVideoFilePath = (this: VideoInstance, videoFile: VideoFileInstance) => string export type CreateTorrentAndSetInfoHash = (this: VideoInstance, videoFile: VideoFileInstance) => Promise - export type ToAddRemoteJSON = (this: VideoInstance) => Promise - export type ToUpdateRemoteJSON = (this: VideoInstance) => RemoteVideoUpdateData + export type ToActivityPubObject = (this: VideoInstance) => VideoTorrentObject export type OptimizeOriginalVideofile = (this: VideoInstance) => Promise export type TranscodeOriginalVideofile = (this: VideoInstance, resolution: number) => Promise @@ -40,31 +40,35 @@ export namespace VideoMethods { export type GetPreviewPath = (this: VideoInstance) => string export type GetDescriptionPath = (this: VideoInstance) => string export type GetTruncatedDescription = (this: VideoInstance) => string + export type GetCategoryLabel = (this: VideoInstance) => string + export type GetLicenceLabel = (this: VideoInstance) => string + export type GetLanguageLabel = (this: VideoInstance) => string // Return thumbnail name export type GenerateThumbnailFromData = (video: VideoInstance, thumbnailData: string) => Promise - export type List = () => Promise - export type ListOwnedAndPopulateAuthorAndTags = () => Promise - export type ListOwnedByAuthor = (author: string) => Promise + export type List = () => Bluebird + export type ListOwnedAndPopulateAccountAndTags = () => Bluebird + export type ListOwnedByAccount = (account: string) => Bluebird - export type ListForApi = (start: number, count: number, sort: string) => Promise< ResultList > - export type ListUserVideosForApi = (userId: number, start: number, count: number, sort: string) => Promise< ResultList > - export type SearchAndPopulateAuthorAndPodAndTags = ( + export type ListForApi = (start: number, count: number, sort: string) => Bluebird< ResultList > + export type ListUserVideosForApi = (userId: number, start: number, count: number, sort: string) => Bluebird< ResultList > + export type SearchAndPopulateAccountAndPodAndTags = ( value: string, field: string, start: number, count: number, sort: string - ) => Promise< ResultList > + ) => Bluebird< ResultList > - export type Load = (id: number) => Promise - export type LoadByUUID = (uuid: string, t?: Sequelize.Transaction) => Promise - export type LoadLocalVideoByUUID = (uuid: string, t?: Sequelize.Transaction) => Promise - export type LoadByHostAndUUID = (fromHost: string, uuid: string, t?: Sequelize.Transaction) => Promise - export type LoadAndPopulateAuthor = (id: number) => Promise - export type LoadAndPopulateAuthorAndPodAndTags = (id: number) => Promise - export type LoadByUUIDAndPopulateAuthorAndPodAndTags = (uuid: string) => Promise + export type Load = (id: number) => Bluebird + export type LoadByUUID = (uuid: string, t?: Sequelize.Transaction) => Bluebird + export type LoadByUrl = (url: string, t?: Sequelize.Transaction) => Bluebird + export type LoadLocalVideoByUUID = (uuid: string, t?: Sequelize.Transaction) => Bluebird + export type LoadByHostAndUUID = (fromHost: string, uuid: string, t?: Sequelize.Transaction) => Bluebird + export type LoadAndPopulateAccount = (id: number) => Bluebird + export type LoadAndPopulateAccountAndPodAndTags = (id: number) => Bluebird + export type LoadByUUIDAndPopulateAccountAndPodAndTags = (uuid: string) => Bluebird export type RemoveThumbnail = (this: VideoInstance) => Promise export type RemovePreview = (this: VideoInstance) => Promise @@ -77,16 +81,17 @@ export interface VideoClass { list: VideoMethods.List listForApi: VideoMethods.ListForApi listUserVideosForApi: VideoMethods.ListUserVideosForApi - listOwnedAndPopulateAuthorAndTags: VideoMethods.ListOwnedAndPopulateAuthorAndTags - listOwnedByAuthor: VideoMethods.ListOwnedByAuthor + listOwnedAndPopulateAccountAndTags: VideoMethods.ListOwnedAndPopulateAccountAndTags + listOwnedByAccount: VideoMethods.ListOwnedByAccount load: VideoMethods.Load - loadAndPopulateAuthor: VideoMethods.LoadAndPopulateAuthor - loadAndPopulateAuthorAndPodAndTags: VideoMethods.LoadAndPopulateAuthorAndPodAndTags + loadAndPopulateAccount: VideoMethods.LoadAndPopulateAccount + loadAndPopulateAccountAndPodAndTags: VideoMethods.LoadAndPopulateAccountAndPodAndTags loadByHostAndUUID: VideoMethods.LoadByHostAndUUID loadByUUID: VideoMethods.LoadByUUID + loadByUrl: VideoMethods.LoadByUrl loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID - loadByUUIDAndPopulateAuthorAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAuthorAndPodAndTags - searchAndPopulateAuthorAndPodAndTags: VideoMethods.SearchAndPopulateAuthorAndPodAndTags + loadByUUIDAndPopulateAccountAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAccountAndPodAndTags + searchAndPopulateAccountAndPodAndTags: VideoMethods.SearchAndPopulateAccountAndPodAndTags } export interface VideoAttributes { @@ -104,7 +109,9 @@ export interface VideoAttributes { likes?: number dislikes?: number remote: boolean + url: string + parentId?: number channelId?: number VideoChannel?: VideoChannelInstance @@ -132,16 +139,18 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In removePreview: VideoMethods.RemovePreview removeThumbnail: VideoMethods.RemoveThumbnail removeTorrent: VideoMethods.RemoveTorrent - toAddRemoteJSON: VideoMethods.ToAddRemoteJSON + toActivityPubObject: VideoMethods.ToActivityPubObject toFormattedJSON: VideoMethods.ToFormattedJSON toFormattedDetailsJSON: VideoMethods.ToFormattedDetailsJSON - toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON optimizeOriginalVideofile: VideoMethods.OptimizeOriginalVideofile transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile getOriginalFileHeight: VideoMethods.GetOriginalFileHeight getEmbedPath: VideoMethods.GetEmbedPath getDescriptionPath: VideoMethods.GetDescriptionPath getTruncatedDescription: VideoMethods.GetTruncatedDescription + getCategoryLabel: VideoMethods.GetCategoryLabel + getLicenceLabel: VideoMethods.GetLicenceLabel + getLanguageLabel: VideoMethods.GetLanguageLabel setTags: Sequelize.HasManySetAssociationsMixin addVideoFile: Sequelize.HasManyAddAssociationMixin @@ -149,3 +158,4 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In } export interface VideoModel extends VideoClass, Sequelize.Model {} + diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 02dde1726..94af1ece5 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -5,7 +5,6 @@ import { map, maxBy, truncate } from 'lodash' import * as parseTorrent from 'parse-torrent' import { join } from 'path' import * as Sequelize from 'sequelize' -import * as Promise from 'bluebird' import { TagInstance } from './tag-interface' import { @@ -52,6 +51,7 @@ import { VideoMethods } from './video-interface' +import { VideoTorrentObject } from '../../../shared/models/activitypub/objects/video-torrent-object' let Video: Sequelize.Model let getOriginalFile: VideoMethods.GetOriginalFile @@ -64,8 +64,7 @@ let getTorrentFileName: VideoMethods.GetTorrentFileName let isOwned: VideoMethods.IsOwned let toFormattedJSON: VideoMethods.ToFormattedJSON let toFormattedDetailsJSON: VideoMethods.ToFormattedDetailsJSON -let toAddRemoteJSON: VideoMethods.ToAddRemoteJSON -let toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON +let toActivityPubObject: VideoMethods.ToActivityPubObject let optimizeOriginalVideofile: VideoMethods.OptimizeOriginalVideofile let transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile let createPreview: VideoMethods.CreatePreview @@ -76,21 +75,25 @@ let getOriginalFileHeight: VideoMethods.GetOriginalFileHeight let getEmbedPath: VideoMethods.GetEmbedPath let getDescriptionPath: VideoMethods.GetDescriptionPath let getTruncatedDescription: VideoMethods.GetTruncatedDescription +let getCategoryLabel: VideoMethods.GetCategoryLabel +let getLicenceLabel: VideoMethods.GetLicenceLabel +let getLanguageLabel: VideoMethods.GetLanguageLabel let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData let list: VideoMethods.List let listForApi: VideoMethods.ListForApi let listUserVideosForApi: VideoMethods.ListUserVideosForApi let loadByHostAndUUID: VideoMethods.LoadByHostAndUUID -let listOwnedAndPopulateAuthorAndTags: VideoMethods.ListOwnedAndPopulateAuthorAndTags -let listOwnedByAuthor: VideoMethods.ListOwnedByAuthor +let listOwnedAndPopulateAccountAndTags: VideoMethods.ListOwnedAndPopulateAccountAndTags +let listOwnedByAccount: VideoMethods.ListOwnedByAccount let load: VideoMethods.Load let loadByUUID: VideoMethods.LoadByUUID +let loadByUrl: VideoMethods.LoadByUrl let loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID -let loadAndPopulateAuthor: VideoMethods.LoadAndPopulateAuthor -let loadAndPopulateAuthorAndPodAndTags: VideoMethods.LoadAndPopulateAuthorAndPodAndTags -let loadByUUIDAndPopulateAuthorAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAuthorAndPodAndTags -let searchAndPopulateAuthorAndPodAndTags: VideoMethods.SearchAndPopulateAuthorAndPodAndTags +let loadAndPopulateAccount: VideoMethods.LoadAndPopulateAccount +let loadAndPopulateAccountAndPodAndTags: VideoMethods.LoadAndPopulateAccountAndPodAndTags +let loadByUUIDAndPopulateAccountAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAccountAndPodAndTags +let searchAndPopulateAccountAndPodAndTags: VideoMethods.SearchAndPopulateAccountAndPodAndTags let removeThumbnail: VideoMethods.RemoveThumbnail let removePreview: VideoMethods.RemovePreview let removeFile: VideoMethods.RemoveFile @@ -219,6 +222,13 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false + }, + url: { + type: DataTypes.STRING, + allowNull: false, + validate: { + isUrl: true + } } }, { @@ -243,6 +253,9 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da }, { fields: [ 'channelId' ] + }, + { + fields: [ 'parentId' ] } ], hooks: { @@ -258,16 +271,16 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da list, listForApi, listUserVideosForApi, - listOwnedAndPopulateAuthorAndTags, - listOwnedByAuthor, + listOwnedAndPopulateAccountAndTags, + listOwnedByAccount, load, - loadAndPopulateAuthor, - loadAndPopulateAuthorAndPodAndTags, + loadAndPopulateAccount, + loadAndPopulateAccountAndPodAndTags, loadByHostAndUUID, loadByUUID, loadLocalVideoByUUID, - loadByUUIDAndPopulateAuthorAndPodAndTags, - searchAndPopulateAuthorAndPodAndTags + loadByUUIDAndPopulateAccountAndPodAndTags, + searchAndPopulateAccountAndPodAndTags ] const instanceMethods = [ createPreview, @@ -286,16 +299,18 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da removePreview, removeThumbnail, removeTorrent, - toAddRemoteJSON, + toActivityPubObject, toFormattedJSON, toFormattedDetailsJSON, - toUpdateRemoteJSON, optimizeOriginalVideofile, transcodeOriginalVideofile, getOriginalFileHeight, getEmbedPath, getTruncatedDescription, - getDescriptionPath + getDescriptionPath, + getCategoryLabel, + getLicenceLabel, + getLanguageLabel ] addMethodsToModel(Video, classMethods, instanceMethods) @@ -313,6 +328,14 @@ function associate (models) { onDelete: 'cascade' }) + Video.belongsTo(models.VideoChannel, { + foreignKey: { + name: 'parentId', + allowNull: true + }, + onDelete: 'cascade' + }) + Video.belongsToMany(models.Tag, { foreignKey: 'videoId', through: models.VideoTag, @@ -423,7 +446,7 @@ getVideoFilePath = function (this: VideoInstance, videoFile: VideoFileInstance) return join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) } -createTorrentAndSetInfoHash = function (this: VideoInstance, videoFile: VideoFileInstance) { +createTorrentAndSetInfoHash = async function (this: VideoInstance, videoFile: VideoFileInstance) { const options = { announceList: [ [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ] @@ -433,18 +456,15 @@ createTorrentAndSetInfoHash = function (this: VideoInstance, videoFile: VideoFil ] } - return createTorrentPromise(this.getVideoFilePath(videoFile), options) - .then(torrent => { - const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) - logger.info('Creating torrent %s.', filePath) + const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) - return writeFilePromise(filePath, torrent).then(() => torrent) - }) - .then(torrent => { - const parsedTorrent = parseTorrent(torrent) + const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) + logger.info('Creating torrent %s.', filePath) - videoFile.infoHash = parsedTorrent.infoHash - }) + await writeFilePromise(filePath, torrent) + + const parsedTorrent = parseTorrent(torrent) + videoFile.infoHash = parsedTorrent.infoHash } getEmbedPath = function (this: VideoInstance) { @@ -462,40 +482,28 @@ getPreviewPath = function (this: VideoInstance) { toFormattedJSON = function (this: VideoInstance) { let podHost - if (this.VideoChannel.Author.Pod) { - podHost = this.VideoChannel.Author.Pod.host + if (this.VideoChannel.Account.Pod) { + podHost = this.VideoChannel.Account.Pod.host } else { // It means it's our video podHost = CONFIG.WEBSERVER.HOST } - // Maybe our pod is not up to date and there are new categories since our version - let categoryLabel = VIDEO_CATEGORIES[this.category] - if (!categoryLabel) categoryLabel = 'Misc' - - // Maybe our pod is not up to date and there are new licences since our version - let licenceLabel = VIDEO_LICENCES[this.licence] - if (!licenceLabel) licenceLabel = 'Unknown' - - // Language is an optional attribute - let languageLabel = VIDEO_LANGUAGES[this.language] - if (!languageLabel) languageLabel = 'Unknown' - const json = { id: this.id, uuid: this.uuid, name: this.name, category: this.category, - categoryLabel, + categoryLabel: this.getCategoryLabel(), licence: this.licence, - licenceLabel, + licenceLabel: this.getLicenceLabel(), language: this.language, - languageLabel, + languageLabel: this.getLanguageLabel(), nsfw: this.nsfw, description: this.getTruncatedDescription(), podHost, isLocal: this.isOwned(), - author: this.VideoChannel.Author.name, + account: this.VideoChannel.Account.name, duration: this.duration, views: this.views, likes: this.likes, @@ -552,75 +560,75 @@ toFormattedDetailsJSON = function (this: VideoInstance) { return Object.assign(formattedJson, detailsJson) } -toAddRemoteJSON = function (this: VideoInstance) { - // Get thumbnail data to send to the other pod - const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) +toActivityPubObject = function (this: VideoInstance) { + const { baseUrlHttp, baseUrlWs } = getBaseUrls(this) - return readFileBufferPromise(thumbnailPath).then(thumbnailData => { - const remoteVideo = { - uuid: this.uuid, - name: this.name, - category: this.category, - licence: this.licence, - language: this.language, - nsfw: this.nsfw, - truncatedDescription: this.getTruncatedDescription(), - channelUUID: this.VideoChannel.uuid, - duration: this.duration, - thumbnailData: thumbnailData.toString('binary'), - tags: map(this.Tags, 'name'), - createdAt: this.createdAt, - updatedAt: this.updatedAt, - views: this.views, - likes: this.likes, - dislikes: this.dislikes, - privacy: this.privacy, - files: [] - } + const tag = this.Tags.map(t => ({ + type: 'Hashtag', + name: t.name + })) + + const url = [] + for (const file of this.VideoFiles) { + url.push({ + type: 'Link', + mimeType: 'video/' + file.extname, + url: getVideoFileUrl(this, file, baseUrlHttp), + width: file.resolution, + size: file.size + }) - this.VideoFiles.forEach(videoFile => { - remoteVideo.files.push({ - infoHash: videoFile.infoHash, - resolution: videoFile.resolution, - extname: videoFile.extname, - size: videoFile.size - }) + url.push({ + type: 'Link', + mimeType: 'application/x-bittorrent', + url: getTorrentUrl(this, file, baseUrlHttp), + width: file.resolution }) - return remoteVideo - }) -} + url.push({ + type: 'Link', + mimeType: 'application/x-bittorrent;x-scheme-handler/magnet', + url: generateMagnetUri(this, file, baseUrlHttp, baseUrlWs), + width: file.resolution + }) + } -toUpdateRemoteJSON = function (this: VideoInstance) { - const json = { - uuid: this.uuid, + const videoObject: VideoTorrentObject = { + type: 'Video', name: this.name, - category: this.category, - licence: this.licence, - language: this.language, - nsfw: this.nsfw, - truncatedDescription: this.getTruncatedDescription(), - duration: this.duration, - tags: map(this.Tags, 'name'), - createdAt: this.createdAt, - updatedAt: this.updatedAt, + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + duration: 'PT' + this.duration + 'S', + uuid: this.uuid, + tag, + category: { + id: this.category, + label: this.getCategoryLabel() + }, + licence: { + id: this.licence, + name: this.getLicenceLabel() + }, + language: { + id: this.language, + name: this.getLanguageLabel() + }, views: this.views, - likes: this.likes, - dislikes: this.dislikes, - privacy: this.privacy, - files: [] + nsfw: this.nsfw, + published: this.createdAt, + updated: this.updatedAt, + mediaType: 'text/markdown', + content: this.getTruncatedDescription(), + icon: { + type: 'Image', + url: getThumbnailUrl(this, baseUrlHttp), + mediaType: 'image/jpeg', + width: THUMBNAILS_SIZE.width, + height: THUMBNAILS_SIZE.height + }, + url } - this.VideoFiles.forEach(videoFile => { - json.files.push({ - infoHash: videoFile.infoHash, - resolution: videoFile.resolution, - extname: videoFile.extname, - size: videoFile.size - }) - }) - - return json + return videoObject } getTruncatedDescription = function (this: VideoInstance) { @@ -631,7 +639,7 @@ getTruncatedDescription = function (this: VideoInstance) { return truncate(this.description, options) } -optimizeOriginalVideofile = function (this: VideoInstance) { +optimizeOriginalVideofile = async function (this: VideoInstance) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const newExtname = '.mp4' const inputVideoFile = this.getOriginalFile() @@ -643,40 +651,32 @@ optimizeOriginalVideofile = function (this: VideoInstance) { outputPath: videoOutputPath } - return transcode(transcodeOptions) - .then(() => { - return unlinkPromise(videoInputPath) - }) - .then(() => { - // Important to do this before getVideoFilename() to take in account the new file extension - inputVideoFile.set('extname', newExtname) + try { + // Could be very long! + await transcode(transcodeOptions) - return renamePromise(videoOutputPath, this.getVideoFilePath(inputVideoFile)) - }) - .then(() => { - return statPromise(this.getVideoFilePath(inputVideoFile)) - }) - .then(stats => { - return inputVideoFile.set('size', stats.size) - }) - .then(() => { - return this.createTorrentAndSetInfoHash(inputVideoFile) - }) - .then(() => { - return inputVideoFile.save() - }) - .then(() => { - return undefined - }) - .catch(err => { - // Auto destruction... - this.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', err)) + await unlinkPromise(videoInputPath) - throw err - }) + // Important to do this before getVideoFilename() to take in account the new file extension + inputVideoFile.set('extname', newExtname) + + await renamePromise(videoOutputPath, this.getVideoFilePath(inputVideoFile)) + const stats = await statPromise(this.getVideoFilePath(inputVideoFile)) + + inputVideoFile.set('size', stats.size) + + await this.createTorrentAndSetInfoHash(inputVideoFile) + await inputVideoFile.save() + + } catch (err) { + // Auto destruction... + this.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', err)) + + throw err + } } -transcodeOriginalVideofile = function (this: VideoInstance, resolution: VideoResolution) { +transcodeOriginalVideofile = async function (this: VideoInstance, resolution: VideoResolution) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const extname = '.mp4' @@ -696,25 +696,18 @@ transcodeOriginalVideofile = function (this: VideoInstance, resolution: VideoRes outputPath: videoOutputPath, resolution } - return transcode(transcodeOptions) - .then(() => { - return statPromise(videoOutputPath) - }) - .then(stats => { - newVideoFile.set('size', stats.size) - return undefined - }) - .then(() => { - return this.createTorrentAndSetInfoHash(newVideoFile) - }) - .then(() => { - return newVideoFile.save() - }) - .then(() => { - return this.VideoFiles.push(newVideoFile) - }) - .then(() => undefined) + await transcode(transcodeOptions) + + const stats = await statPromise(videoOutputPath) + + newVideoFile.set('size', stats.size) + + await this.createTorrentAndSetInfoHash(newVideoFile) + + await newVideoFile.save() + + this.VideoFiles.push(newVideoFile) } getOriginalFileHeight = function (this: VideoInstance) { @@ -727,6 +720,31 @@ getDescriptionPath = function (this: VideoInstance) { return `/api/${API_VERSION}/videos/${this.uuid}/description` } +getCategoryLabel = function (this: VideoInstance) { + let categoryLabel = VIDEO_CATEGORIES[this.category] + + // Maybe our pod is not up to date and there are new categories since our version + if (!categoryLabel) categoryLabel = 'Misc' + + return categoryLabel +} + +getLicenceLabel = function (this: VideoInstance) { + let licenceLabel = VIDEO_LICENCES[this.licence] + // Maybe our pod is not up to date and there are new licences since our version + if (!licenceLabel) licenceLabel = 'Unknown' + + return licenceLabel +} + +getLanguageLabel = function (this: VideoInstance) { + // Language is an optional attribute + let languageLabel = VIDEO_LANGUAGES[this.language] + if (!languageLabel) languageLabel = 'Unknown' + + return languageLabel +} + removeThumbnail = function (this: VideoInstance) { const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) return unlinkPromise(thumbnailPath) @@ -779,7 +797,7 @@ listUserVideosForApi = function (userId: number, start: number, count: number, s required: true, include: [ { - model: Video['sequelize'].models.Author, + model: Video['sequelize'].models.Account, where: { userId }, @@ -810,7 +828,7 @@ listForApi = function (start: number, count: number, sort: string) { model: Video['sequelize'].models.VideoChannel, include: [ { - model: Video['sequelize'].models.Author, + model: Video['sequelize'].models.Account, include: [ { model: Video['sequelize'].models.Pod, @@ -846,7 +864,7 @@ loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Tran model: Video['sequelize'].models.VideoChannel, include: [ { - model: Video['sequelize'].models.Author, + model: Video['sequelize'].models.Account, include: [ { model: Video['sequelize'].models.Pod, @@ -867,7 +885,7 @@ loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Tran return Video.findOne(query) } -listOwnedAndPopulateAuthorAndTags = function () { +listOwnedAndPopulateAccountAndTags = function () { const query = { where: { remote: false @@ -876,7 +894,7 @@ listOwnedAndPopulateAuthorAndTags = function () { Video['sequelize'].models.VideoFile, { model: Video['sequelize'].models.VideoChannel, - include: [ Video['sequelize'].models.Author ] + include: [ Video['sequelize'].models.Account ] }, Video['sequelize'].models.Tag ] @@ -885,7 +903,7 @@ listOwnedAndPopulateAuthorAndTags = function () { return Video.findAll(query) } -listOwnedByAuthor = function (author: string) { +listOwnedByAccount = function (account: string) { const query = { where: { remote: false @@ -898,9 +916,9 @@ listOwnedByAuthor = function (author: string) { model: Video['sequelize'].models.VideoChannel, include: [ { - model: Video['sequelize'].models.Author, + model: Video['sequelize'].models.Account, where: { - name: author + name: account } } ] @@ -942,13 +960,13 @@ loadLocalVideoByUUID = function (uuid: string, t?: Sequelize.Transaction) { return Video.findOne(query) } -loadAndPopulateAuthor = function (id: number) { +loadAndPopulateAccount = function (id: number) { const options = { include: [ Video['sequelize'].models.VideoFile, { model: Video['sequelize'].models.VideoChannel, - include: [ Video['sequelize'].models.Author ] + include: [ Video['sequelize'].models.Account ] } ] } @@ -956,14 +974,14 @@ loadAndPopulateAuthor = function (id: number) { return Video.findById(id, options) } -loadAndPopulateAuthorAndPodAndTags = function (id: number) { +loadAndPopulateAccountAndPodAndTags = function (id: number) { const options = { include: [ { model: Video['sequelize'].models.VideoChannel, include: [ { - model: Video['sequelize'].models.Author, + model: Video['sequelize'].models.Account, include: [ { model: Video['sequelize'].models.Pod, required: false } ] } ] @@ -976,7 +994,7 @@ loadAndPopulateAuthorAndPodAndTags = function (id: number) { return Video.findById(id, options) } -loadByUUIDAndPopulateAuthorAndPodAndTags = function (uuid: string) { +loadByUUIDAndPopulateAccountAndPodAndTags = function (uuid: string) { const options = { where: { uuid @@ -986,7 +1004,7 @@ loadByUUIDAndPopulateAuthorAndPodAndTags = function (uuid: string) { model: Video['sequelize'].models.VideoChannel, include: [ { - model: Video['sequelize'].models.Author, + model: Video['sequelize'].models.Account, include: [ { model: Video['sequelize'].models.Pod, required: false } ] } ] @@ -999,20 +1017,20 @@ loadByUUIDAndPopulateAuthorAndPodAndTags = function (uuid: string) { return Video.findOne(options) } -searchAndPopulateAuthorAndPodAndTags = function (value: string, field: string, start: number, count: number, sort: string) { +searchAndPopulateAccountAndPodAndTags = function (value: string, field: string, start: number, count: number, sort: string) { const podInclude: Sequelize.IncludeOptions = { model: Video['sequelize'].models.Pod, required: false } - const authorInclude: Sequelize.IncludeOptions = { - model: Video['sequelize'].models.Author, + const accountInclude: Sequelize.IncludeOptions = { + model: Video['sequelize'].models.Account, include: [ podInclude ] } const videoChannelInclude: Sequelize.IncludeOptions = { model: Video['sequelize'].models.VideoChannel, - include: [ authorInclude ], + include: [ accountInclude ], required: true } @@ -1045,8 +1063,8 @@ searchAndPopulateAuthorAndPodAndTags = function (value: string, field: string, s } } podInclude.required = true - } else if (field === 'author') { - authorInclude.where = { + } else if (field === 'account') { + accountInclude.where = { name: { [Sequelize.Op.iLike]: '%' + value + '%' } @@ -1090,13 +1108,17 @@ function getBaseUrls (video: VideoInstance) { baseUrlHttp = CONFIG.WEBSERVER.URL baseUrlWs = CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT } else { - baseUrlHttp = REMOTE_SCHEME.HTTP + '://' + video.VideoChannel.Author.Pod.host - baseUrlWs = REMOTE_SCHEME.WS + '://' + video.VideoChannel.Author.Pod.host + baseUrlHttp = REMOTE_SCHEME.HTTP + '://' + video.VideoChannel.Account.Pod.host + baseUrlWs = REMOTE_SCHEME.WS + '://' + video.VideoChannel.Account.Pod.host } return { baseUrlHttp, baseUrlWs } } +function getThumbnailUrl (video: VideoInstance, baseUrlHttp: string) { + return baseUrlHttp + STATIC_PATHS.THUMBNAILS + video.getThumbnailName() +} + function getTorrentUrl (video: VideoInstance, videoFile: VideoFileInstance, baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.TORRENTS + video.getTorrentFileName(videoFile) } -- cgit v1.2.3