From c48e82b5e0478434de30626d14594a97f2402e7c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Sep 2018 16:27:07 +0200 Subject: Basic video redundancy implementation --- server/helpers/activitypub.ts | 28 +++++----- .../custom-validators/activitypub/activity.ts | 14 +++-- .../custom-validators/activitypub/cache-file.ts | 28 ++++++++++ .../helpers/custom-validators/activitypub/misc.ts | 10 +++- .../helpers/custom-validators/activitypub/undo.ts | 4 +- .../custom-validators/activitypub/videos.ts | 49 +++++++++-------- server/helpers/webtorrent.ts | 61 ++++++++++++++++------ 7 files changed, 137 insertions(+), 57 deletions(-) create mode 100644 server/helpers/custom-validators/activitypub/cache-file.ts (limited to 'server/helpers') diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index a9de11fb0..1304c7559 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -14,20 +14,24 @@ function activityPubContextify (data: T) { 'https://w3id.org/security/v1', { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', + pt: 'https://joinpeertube.org/ns', + schema: 'http://schema.org#', Hashtag: 'as:Hashtag', - uuid: 'http://schema.org/identifier', - category: 'http://schema.org/category', - licence: 'http://schema.org/license', - subtitleLanguage: 'http://schema.org/subtitleLanguage', + uuid: 'schema:identifier', + category: 'schema:category', + licence: 'schema:license', + subtitleLanguage: 'schema:subtitleLanguage', sensitive: 'as:sensitive', - language: 'http://schema.org/inLanguage', - views: 'http://schema.org/Number', - stats: 'http://schema.org/Number', - size: 'http://schema.org/Number', - fps: 'http://schema.org/Number', - commentsEnabled: 'http://schema.org/Boolean', - waitTranscoding: 'http://schema.org/Boolean', - support: 'http://schema.org/Text' + language: 'schema:inLanguage', + views: 'schema:Number', + stats: 'schema:Number', + size: 'schema:Number', + fps: 'schema:Number', + commentsEnabled: 'schema:Boolean', + waitTranscoding: 'schema:Boolean', + expires: 'schema:expires', + support: 'schema:Text', + CacheFile: 'pt:CacheFile' }, { likes: { diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 381a29e66..2562ead9b 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -1,7 +1,10 @@ import * as validator from 'validator' import { Activity, ActivityType } from '../../../../shared/models/activitypub' import { - isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid, + isActorAcceptActivityValid, + isActorDeleteActivityValid, + isActorFollowActivityValid, + isActorRejectActivityValid, isActorUpdateActivityValid } from './actor' import { isAnnounceActivityValid } from './announce' @@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo' import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments' import { isVideoFlagValid, - sanitizeAndCheckVideoTorrentCreateActivity, isVideoTorrentDeleteActivityValid, + sanitizeAndCheckVideoTorrentCreateActivity, sanitizeAndCheckVideoTorrentUpdateActivity } from './videos' import { isViewActivityValid } from './view' import { exists } from '../misc' +import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file' function isRootActivityValid (activity: any) { return Array.isArray(activity['@context']) && ( @@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) { isDislikeActivityValid(activity) || sanitizeAndCheckVideoTorrentCreateActivity(activity) || isVideoFlagValid(activity) || - isVideoCommentCreateActivityValid(activity) + isVideoCommentCreateActivityValid(activity) || + isCacheFileCreateActivityValid(activity) } function checkUpdateActivity (activity: any) { - return sanitizeAndCheckVideoTorrentUpdateActivity(activity) || + return isCacheFileUpdateActivityValid(activity) || + sanitizeAndCheckVideoTorrentUpdateActivity(activity) || isActorUpdateActivityValid(activity) } diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts new file mode 100644 index 000000000..bd70934c8 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/cache-file.ts @@ -0,0 +1,28 @@ +import { isActivityPubUrlValid, isBaseActivityValid } from './misc' +import { isRemoteVideoUrlValid } from './videos' +import { isDateValid, exists } from '../misc' +import { CacheFileObject } from '../../../../shared/models/activitypub/objects' + +function isCacheFileCreateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Create') && + isCacheFileObjectValid(activity.object) +} + +function isCacheFileUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + isCacheFileObjectValid(activity.object) +} + +function isCacheFileObjectValid (object: CacheFileObject) { + return exists(object) && + object.type === 'CacheFile' && + isDateValid(object.expires) && + isActivityPubUrlValid(object.object) && + isRemoteVideoUrlValid(object.url) +} + +export { + isCacheFileUpdateActivityValid, + isCacheFileCreateActivityValid, + isCacheFileObjectValid +} diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 6c5c7abca..4e2c57f04 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers' import { isTestInstance } from '../../core-utils' import { exists } from '../misc' -function isActivityPubUrlValid (url: string) { +function isUrlValid (url: string) { const isURLOptions = { require_host: true, require_tld: true, @@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) { isURLOptions.require_tld = false } - return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) + return exists(url) && validator.isURL('' + url, isURLOptions) +} + +function isActivityPubUrlValid (url: string) { + return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) } function isBaseActivityValid (activity: any, type: string) { return (activity['@context'] === undefined || Array.isArray(activity['@context'])) && activity.type === type && isActivityPubUrlValid(activity.id) && + exists(activity.actor) && (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) && ( activity.to === undefined || @@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) { } export { + isUrlValid, isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo diff --git a/server/helpers/custom-validators/activitypub/undo.ts b/server/helpers/custom-validators/activitypub/undo.ts index f50f224fa..578035893 100644 --- a/server/helpers/custom-validators/activitypub/undo.ts +++ b/server/helpers/custom-validators/activitypub/undo.ts @@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor' import { isBaseActivityValid } from './misc' import { isDislikeActivityValid, isLikeActivityValid } from './rate' import { isAnnounceActivityValid } from './announce' +import { isCacheFileCreateActivityValid } from './cache-file' function isUndoActivityValid (activity: any) { return isBaseActivityValid(activity, 'Undo') && @@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) { isActorFollowActivityValid(activity.object) || isLikeActivityValid(activity.object) || isDislikeActivityValid(activity.object) || - isAnnounceActivityValid(activity.object) + isAnnounceActivityValid(activity.object) || + isCacheFileCreateActivityValid(activity.object) ) } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 0362f43ab..f76eba474 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { video.attributedTo.length !== 0 } +function isRemoteVideoUrlValid (url: any) { + // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11) + if (url.width && !url.height) url.height = url.width + + return url.type === 'Link' && + ( + ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && + isActivityPubUrlValid(url.href) && + validator.isInt(url.height + '', { min: 0 }) && + validator.isInt(url.size + '', { min: 0 }) && + (!url.fps || validator.isInt(url.fps + '', { min: 0 })) + ) || + ( + ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && + isActivityPubUrlValid(url.href) && + validator.isInt(url.height + '', { min: 0 }) + ) || + ( + ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 && + validator.isLength(url.href, { min: 5 }) && + validator.isInt(url.height + '', { min: 0 }) + ) +} + // --------------------------------------------------------------------------- export { @@ -83,7 +107,8 @@ export { isVideoTorrentDeleteActivityValid, isRemoteStringIdentifierValid, isVideoFlagValid, - sanitizeAndCheckVideoTorrentObject + sanitizeAndCheckVideoTorrentObject, + isRemoteVideoUrlValid } // --------------------------------------------------------------------------- @@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) { return true } -function isRemoteVideoUrlValid (url: any) { - // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11) - if (url.width && !url.height) url.height = url.width - return url.type === 'Link' && - ( - ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && - isActivityPubUrlValid(url.href) && - validator.isInt(url.height + '', { min: 0 }) && - validator.isInt(url.size + '', { min: 0 }) && - (!url.fps || validator.isInt(url.fps + '', { min: 0 })) - ) || - ( - ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && - isActivityPubUrlValid(url.href) && - validator.isInt(url.height + '', { min: 0 }) - ) || - ( - ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 && - validator.isLength(url.href, { min: 5 }) && - validator.isInt(url.height + '', { min: 0 }) - ) -} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 1c0cc7058..2fdfd1876 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra' import { CONFIG } from '../initializers' import { join } from 'path' -function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { +function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) { const id = target.magnetUri || target.torrentName + let timer const path = generateVideoTmpPath(id) logger.info('Importing torrent video %s', id) return new Promise((res, rej) => { const webtorrent = new WebTorrent() + let file: WebTorrent.TorrentFile const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) const options = { path: CONFIG.STORAGE.VIDEOS_DIR } const torrent = webtorrent.add(torrentId, options, torrent => { - if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) + if (torrent.files.length !== 1) { + if (timer) clearTimeout(timer) - const file = torrent.files[ 0 ] + return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) + .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId))) + } + + file = torrent.files[ 0 ] const writeStream = createWriteStream(path) writeStream.on('finish', () => { - webtorrent.destroy(async err => { - if (err) return rej(err) - - if (target.torrentName) { - remove(torrentId) - .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) - } + if (timer) clearTimeout(timer) - remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name)) - .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err })) - - res(path) - }) + return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) + .then(() => res(path)) }) file.createReadStream().pipe(writeStream) }) torrent.on('error', err => rej(err)) + + if (timeout) { + timer = setTimeout(async () => { + return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName) + .then(() => rej(new Error('Webtorrent download timeout.'))) + }, timeout) + } }) } @@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri export { downloadWebTorrentVideo } + +// --------------------------------------------------------------------------- + +function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) { + return new Promise(res => { + webtorrent.destroy(err => { + // Delete torrent file + if (torrentName) { + remove(torrentId) + .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) + } + + // Delete downloaded file + if (filename) { + remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename)) + .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err })) + } + + if (err) { + logger.warn('Cannot destroy webtorrent in timeout.', { err }) + } + + return res() + }) + }) +} -- cgit v1.2.3