From 0d0e8dd0904b380b70e19ebcb4763d65601c4632 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Nov 2017 14:34:45 +0100 Subject: Continue activitypub --- server/helpers/activitypub.ts | 45 ++++- .../custom-validators/activitypub/activity.ts | 34 ++++ .../helpers/custom-validators/activitypub/index.ts | 1 + .../helpers/custom-validators/activitypub/misc.ts | 12 +- .../custom-validators/activitypub/videos.ts | 223 +++++++-------------- server/helpers/custom-validators/index.ts | 1 - server/helpers/custom-validators/video-authors.ts | 45 ----- server/helpers/custom-validators/videos.ts | 16 +- server/helpers/requests.ts | 11 + 9 files changed, 190 insertions(+), 198 deletions(-) create mode 100644 server/helpers/custom-validators/activitypub/activity.ts delete mode 100644 server/helpers/custom-validators/video-authors.ts (limited to 'server/helpers') diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index ecb509b66..75de2278c 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -2,10 +2,48 @@ import * as url from 'url' import { database as db } from '../initializers' import { logger } from './logger' -import { doRequest } from './requests' +import { doRequest, doRequestAndSaveToFile } from './requests' import { isRemoteAccountValid } from './custom-validators' import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' import { ResultList } from '../../shared/models/result-list.model' +import { CONFIG } from '../initializers/constants' +import { VideoInstance } from '../models/video/video-interface' +import { ActivityIconObject } from '../../shared/index' +import { join } from 'path' + +function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) { + const thumbnailName = video.getThumbnailName() + const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) + + const options = { + method: 'GET', + uri: icon.url + } + return doRequestAndSaveToFile(options, thumbnailPath) +} + +function getActivityPubUrl (type: 'video' | 'videoChannel', uuid: string) { + if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + uuid + else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + uuid + + return '' +} + +async function getOrCreateAccount (accountUrl: string) { + let account = await db.Account.loadByUrl(accountUrl) + + // We don't have this account in our database, fetch it on remote + if (!account) { + const { account } = await fetchRemoteAccountAndCreatePod(accountUrl) + + if (!account) throw new Error('Cannot fetch remote account.') + + // Save our new account in database + await account.save() + } + + return account +} async function fetchRemoteAccountAndCreatePod (accountUrl: string) { const options = { @@ -100,7 +138,10 @@ function activityPubCollectionPagination (url: string, page: number, result: Res export { fetchRemoteAccountAndCreatePod, activityPubContextify, - activityPubCollectionPagination + activityPubCollectionPagination, + getActivityPubUrl, + generateThumbnailFromUrl, + getOrCreateAccount } // --------------------------------------------------------------------------- diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts new file mode 100644 index 000000000..dd671c4cf --- /dev/null +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -0,0 +1,34 @@ +import * as validator from 'validator' +import { + isVideoChannelCreateActivityValid, + isVideoTorrentAddActivityValid, + isVideoTorrentUpdateActivityValid, + isVideoChannelUpdateActivityValid +} from './videos' + +function isRootActivityValid (activity: any) { + return Array.isArray(activity['@context']) && + ( + (activity.type === 'Collection' || activity.type === 'OrderedCollection') && + validator.isInt(activity.totalItems, { min: 0 }) && + Array.isArray(activity.items) + ) || + ( + validator.isURL(activity.id) && + validator.isURL(activity.actor) + ) +} + +function isActivityValid (activity: any) { + return isVideoTorrentAddActivityValid(activity) || + isVideoChannelCreateActivityValid(activity) || + isVideoTorrentUpdateActivityValid(activity) || + isVideoChannelUpdateActivityValid(activity) +} + +// --------------------------------------------------------------------------- + +export { + isRootActivityValid, + isActivityValid +} diff --git a/server/helpers/custom-validators/activitypub/index.ts b/server/helpers/custom-validators/activitypub/index.ts index 800f0ddf3..0eba06a7b 100644 --- a/server/helpers/custom-validators/activitypub/index.ts +++ b/server/helpers/custom-validators/activitypub/index.ts @@ -1,4 +1,5 @@ export * from './account' +export * from './activity' export * from './signature' export * from './misc' export * from './videos' diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 806d33483..f049f5a8c 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -12,6 +12,16 @@ function isActivityPubUrlValid (url: string) { return exists(url) && validator.isURL(url, isURLOptions) } +function isBaseActivityValid (activity: any, type: string) { + return Array.isArray(activity['@context']) && + activity.type === type && + validator.isURL(activity.id) && + validator.isURL(activity.actor) && + Array.isArray(activity.to) && + activity.to.every(t => validator.isURL(t)) +} + export { - isActivityPubUrlValid + isActivityPubUrlValid, + isBaseActivityValid } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index e0ffba679..9233a1359 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -1,184 +1,117 @@ -import 'express-validator' -import { has, values } from 'lodash' +import * as validator from 'validator' import { - REQUEST_ENDPOINTS, - REQUEST_ENDPOINT_ACTIONS, - REQUEST_VIDEO_EVENT_TYPES + ACTIVITY_PUB } from '../../../initializers' -import { isArray, isDateValid, isUUIDValid } from '../misc' +import { isDateValid, isUUIDValid } from '../misc' import { - isVideoThumbnailDataValid, - isVideoAbuseReasonValid, - isVideoAbuseReporterUsernameValid, isVideoViewsValid, - isVideoLikesValid, - isVideoDislikesValid, - isVideoEventCountValid, - isRemoteVideoCategoryValid, - isRemoteVideoLicenceValid, - isRemoteVideoLanguageValid, isVideoNSFWValid, isVideoTruncatedDescriptionValid, isVideoDurationValid, - isVideoFileInfoHashValid, isVideoNameValid, - isVideoTagsValid, - isVideoFileExtnameValid, - isVideoFileResolutionValid + isVideoTagValid } from '../videos' import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../video-channels' -import { isVideoAuthorNameValid } from '../video-authors' - -const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] - -const checkers: { [ id: string ]: (obj: any) => boolean } = {} -checkers[ENDPOINT_ACTIONS.ADD_VIDEO] = checkAddVideo -checkers[ENDPOINT_ACTIONS.UPDATE_VIDEO] = checkUpdateVideo -checkers[ENDPOINT_ACTIONS.REMOVE_VIDEO] = checkRemoveVideo -checkers[ENDPOINT_ACTIONS.REPORT_ABUSE] = checkReportVideo -checkers[ENDPOINT_ACTIONS.ADD_CHANNEL] = checkAddVideoChannel -checkers[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = checkUpdateVideoChannel -checkers[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = checkRemoveVideoChannel -checkers[ENDPOINT_ACTIONS.ADD_AUTHOR] = checkAddAuthor -checkers[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = checkRemoveAuthor - -function removeBadRequestVideos (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const video = request.data - - if ( - !video || - checkers[request.type] === undefined || - checkers[request.type](video) === false - ) { - requests.splice(i, 1) - } - } +import { isBaseActivityValid } from './misc' + +function isVideoTorrentAddActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Add') && + isVideoTorrentObjectValid(activity.object) +} + +function isVideoTorrentUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + isVideoTorrentObjectValid(activity.object) } -function removeBadRequestVideosQadu (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const video = request.data - - if ( - !video || - ( - isUUIDValid(video.uuid) && - (has(video, 'views') === false || isVideoViewsValid(video.views)) && - (has(video, 'likes') === false || isVideoLikesValid(video.likes)) && - (has(video, 'dislikes') === false || isVideoDislikesValid(video.dislikes)) - ) === false - ) { - requests.splice(i, 1) - } - } +function isVideoTorrentObjectValid (video: any) { + return video.type === 'Video' && + isVideoNameValid(video.name) && + isVideoDurationValid(video.duration) && + isUUIDValid(video.uuid) && + setValidRemoteTags(video) && + isRemoteIdentifierValid(video.category) && + isRemoteIdentifierValid(video.licence) && + isRemoteIdentifierValid(video.language) && + isVideoViewsValid(video.video) && + isVideoNSFWValid(video.nsfw) && + isDateValid(video.published) && + isDateValid(video.updated) && + isRemoteVideoContentValid(video.mediaType, video.content) && + isRemoteVideoIconValid(video.icon) && + setValidRemoteVideoUrls(video.url) } -function removeBadRequestVideosEvents (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const eventData = request.data - - if ( - !eventData || - ( - isUUIDValid(eventData.uuid) && - values(REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && - isVideoEventCountValid(eventData.count) - ) === false - ) { - requests.splice(i, 1) - } - } +function isVideoChannelCreateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Create') && + isVideoChannelObjectValid(activity.object) +} + +function isVideoChannelUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + isVideoChannelObjectValid(activity.object) +} + +function isVideoChannelObjectValid (videoChannel: any) { + return videoChannel.type === 'VideoChannel' && + isVideoChannelNameValid(videoChannel.name) && + isVideoChannelDescriptionValid(videoChannel.description) && + isUUIDValid(videoChannel.uuid) } // --------------------------------------------------------------------------- export { - removeBadRequestVideos, - removeBadRequestVideosQadu, - removeBadRequestVideosEvents + isVideoTorrentAddActivityValid, + isVideoChannelCreateActivityValid, + isVideoTorrentUpdateActivityValid, + isVideoChannelUpdateActivityValid } // --------------------------------------------------------------------------- -function isCommonVideoAttributesValid (video: any) { - return isDateValid(video.createdAt) && - isDateValid(video.updatedAt) && - isRemoteVideoCategoryValid(video.category) && - isRemoteVideoLicenceValid(video.licence) && - isRemoteVideoLanguageValid(video.language) && - isVideoNSFWValid(video.nsfw) && - isVideoTruncatedDescriptionValid(video.truncatedDescription) && - isVideoDurationValid(video.duration) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isUUIDValid(video.uuid) && - isVideoViewsValid(video.views) && - isVideoLikesValid(video.likes) && - isVideoDislikesValid(video.dislikes) && - isArray(video.files) && - video.files.every(videoFile => { - if (!videoFile) return false - - return ( - isVideoFileInfoHashValid(videoFile.infoHash) && - isVideoFileExtnameValid(videoFile.extname) && - isVideoFileResolutionValid(videoFile.resolution) - ) - }) -} +function setValidRemoteTags (video: any) { + if (Array.isArray(video.tag) === false) return false -function checkAddVideo (video: any) { - return isCommonVideoAttributesValid(video) && - isUUIDValid(video.channelUUID) && - isVideoThumbnailDataValid(video.thumbnailData) -} + const newTag = video.tag.filter(t => { + return t.type === 'Hashtag' && + isVideoTagValid(t.name) + }) -function checkUpdateVideo (video: any) { - return isCommonVideoAttributesValid(video) + video.tag = newTag + return true } -function checkRemoveVideo (video: any) { - return isUUIDValid(video.uuid) +function isRemoteIdentifierValid (data: any) { + return validator.isInt(data.identifier, { min: 0 }) } -function checkReportVideo (abuse: any) { - return isUUIDValid(abuse.videoUUID) && - isVideoAbuseReasonValid(abuse.reportReason) && - isVideoAbuseReporterUsernameValid(abuse.reporterUsername) +function isRemoteVideoContentValid (mediaType: string, content: string) { + return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) } -function checkAddVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) && - isVideoChannelNameValid(videoChannel.name) && - isVideoChannelDescriptionValid(videoChannel.description) && - isDateValid(videoChannel.createdAt) && - isDateValid(videoChannel.updatedAt) && - isUUIDValid(videoChannel.ownerUUID) +function isRemoteVideoIconValid (icon: any) { + return icon.type === 'Image' && + validator.isURL(icon.url) && + icon.mediaType === 'image/jpeg' && + validator.isInt(icon.width, { min: 0 }) && + validator.isInt(icon.height, { min: 0 }) } -function checkUpdateVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) && - isVideoChannelNameValid(videoChannel.name) && - isVideoChannelDescriptionValid(videoChannel.description) && - isDateValid(videoChannel.createdAt) && - isDateValid(videoChannel.updatedAt) && - isUUIDValid(videoChannel.ownerUUID) -} +function setValidRemoteVideoUrls (video: any) { + if (Array.isArray(video.url) === false) return false -function checkRemoveVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) -} + const newUrl = video.url.filter(u => isRemoteVideoUrlValid(u)) + video.url = newUrl -function checkAddAuthor (author: any) { - return isUUIDValid(author.uuid) && - isVideoAuthorNameValid(author.name) + return true } -function checkRemoveAuthor (author: any) { - return isUUIDValid(author.uuid) +function isRemoteVideoUrlValid (url: any) { + return url.type === 'Link' && + ACTIVITY_PUB.VIDEO_URL_MIME_TYPES.indexOf(url.mimeType) !== -1 && + validator.isURL(url.url) && + validator.isInt(url.width, { min: 0 }) && + validator.isInt(url.size, { min: 0 }) } diff --git a/server/helpers/custom-validators/index.ts b/server/helpers/custom-validators/index.ts index 869b08870..58a40249b 100644 --- a/server/helpers/custom-validators/index.ts +++ b/server/helpers/custom-validators/index.ts @@ -3,6 +3,5 @@ export * from './misc' export * from './pods' export * from './pods' export * from './users' -export * from './video-authors' export * from './video-channels' export * from './videos' diff --git a/server/helpers/custom-validators/video-authors.ts b/server/helpers/custom-validators/video-authors.ts deleted file mode 100644 index 48ca9b200..000000000 --- a/server/helpers/custom-validators/video-authors.ts +++ /dev/null @@ -1,45 +0,0 @@ -import * as Promise from 'bluebird' -import * as validator from 'validator' -import * as express from 'express' -import 'express-validator' - -import { database as db } from '../../initializers' -import { AuthorInstance } from '../../models' -import { logger } from '../logger' - -import { isUserUsernameValid } from './users' - -function isVideoAuthorNameValid (value: string) { - return isUserUsernameValid(value) -} - -function checkVideoAuthorExists (id: string, res: express.Response, callback: () => void) { - let promise: Promise - if (validator.isInt(id)) { - promise = db.Author.load(+id) - } else { // UUID - promise = db.Author.loadByUUID(id) - } - - promise.then(author => { - if (!author) { - return res.status(404) - .json({ error: 'Video author not found' }) - .end() - } - - res.locals.author = author - callback() - }) - .catch(err => { - logger.error('Error in video author request validator.', err) - return res.sendStatus(500) - }) -} - -// --------------------------------------------------------------------------- - -export { - checkVideoAuthorExists, - isVideoAuthorNameValid -} diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index f3fdcaf2d..83407f17b 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -73,19 +73,26 @@ function isVideoDescriptionValid (value: string) { } function isVideoDurationValid (value: string) { - return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return exists(value) && + typeof value === 'string' && + value.startsWith('PT') && + value.endsWith('S') && + validator.isInt(value.replace(/[^0-9]+/, ''), VIDEOS_CONSTRAINTS_FIELDS.DURATION) } function isVideoNameValid (value: string) { return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) } +function isVideoTagValid (tag: string) { + return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) +} + function isVideoTagsValid (tags: string[]) { return isArray(tags) && validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) && - tags.every(tag => { - return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) - }) + tags.every(tag => isVideoTagValid(tag)) } function isVideoThumbnailValid (value: string) { @@ -209,6 +216,7 @@ export { isRemoteVideoPrivacyValid, isVideoFileResolutionValid, checkVideoExists, + isVideoTagValid, isRemoteVideoCategoryValid, isRemoteVideoLicenceValid, isRemoteVideoLanguageValid diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 8c4c983f7..31cedd768 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -10,6 +10,7 @@ import { import { PodInstance } from '../models' import { PodSignature } from '../../shared' import { signObject } from './peertube-crypto' +import { createWriteStream } from 'fs' function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { @@ -17,6 +18,15 @@ function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { }) } +function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.UriOptions, destPath: string) { + return new Promise((res, rej) => { + request(requestOptions) + .on('response', response => res(response as request.RequestResponse)) + .on('error', err => rej(err)) + .pipe(createWriteStream(destPath)) + }) +} + type MakeRetryRequestParams = { url: string, method: 'GET' | 'POST', @@ -88,6 +98,7 @@ function makeSecureRequest (params: MakeSecureRequestParams) { export { doRequest, + doRequestAndSaveToFile, makeRetryRequest, makeSecureRequest } -- cgit v1.2.3