From 4157cdb13748cb6e8ce7081d062a8778554cc5a7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 11:16:23 +0200 Subject: [PATCH] Refractor videos AP functions --- server/controllers/api/search.ts | 2 +- server/helpers/custom-validators/videos.ts | 12 +- server/helpers/video.ts | 25 ++ .../activitypub/process/process-announce.ts | 2 +- .../lib/activitypub/process/process-create.ts | 10 +- .../lib/activitypub/process/process-like.ts | 2 +- .../lib/activitypub/process/process-undo.ts | 6 +- .../lib/activitypub/process/process-update.ts | 4 +- server/lib/activitypub/video-comments.ts | 2 +- server/lib/activitypub/videos.ts | 420 +++++++++--------- .../handlers/activitypub-http-fetcher.ts | 4 +- server/middlewares/validators/videos.ts | 4 +- server/models/video/video.ts | 16 +- 13 files changed, 272 insertions(+), 237 deletions(-) create mode 100644 server/helpers/video.ts diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 58851d0b5..ea3166f5f 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -139,7 +139,7 @@ async function searchVideoURI (url: string, res: express.Response) { refreshVideo: false } - const result = await getOrCreateVideoAndAccountAndChannel(url, syncParam) + const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) video = result ? result.video : undefined } catch (err) { logger.info('Cannot search remote video %s.', url, { err }) diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index c9ef8445d..9875c68bd 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -18,6 +18,7 @@ import { exists, isArray, isFileValid } from './misc' import { VideoChannelModel } from '../../models/video/video-channel' import { UserModel } from '../../models/account/user' import * as magnetUtil from 'magnet-uri' +import { fetchVideo, VideoFetchType } from '../video' const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS @@ -152,17 +153,8 @@ function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: Use return true } -export type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' async function isVideoExist (id: string, res: Response, fetchType: VideoFetchType = 'all') { - let video: VideoModel | null - - if (fetchType === 'all') { - video = await VideoModel.loadAndPopulateAccountAndServerAndTags(id) - } else if (fetchType === 'only-video') { - video = await VideoModel.load(id) - } else if (fetchType === 'id' || fetchType === 'none') { - video = await VideoModel.loadOnlyId(id) - } + const video = await fetchVideo(id, fetchType) if (video === null) { res.status(404) diff --git a/server/helpers/video.ts b/server/helpers/video.ts new file mode 100644 index 000000000..b1577a6b0 --- /dev/null +++ b/server/helpers/video.ts @@ -0,0 +1,25 @@ +import { VideoModel } from '../models/video/video' + +type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' + +function fetchVideo (id: number | string, fetchType: VideoFetchType) { + if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id) + + if (fetchType === 'only-video') return VideoModel.load(id) + + if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) +} + +type VideoFetchByUrlType = 'all' | 'only-video' +function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType) { + if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) + + if (fetchType === 'only-video') return VideoModel.loadByUrl(url) +} + +export { + VideoFetchType, + VideoFetchByUrlType, + fetchVideo, + fetchVideoByUrl +} diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 814556817..b968389b3 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -25,7 +25,7 @@ export { async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id - const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) return sequelizeTypescript.transaction(async t => { // Add share entry diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 32e555acf..99841da14 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -48,7 +48,7 @@ export { async function processCreateVideo (activity: ActivityCreate) { const videoToCreateData = activity.object as VideoTorrentObject - const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) return video } @@ -59,7 +59,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) return sequelizeTypescript.transaction(async t => { const rate = { @@ -86,7 +86,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object }) const actor = await ActorModel.loadByUrl(view.actor) if (!actor) throw new Error('Unknown actor ' + view.actor) @@ -103,7 +103,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { const cacheFile = activity.object as CacheFileObject - const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) await createCacheFile(cacheFile, video, byActor) @@ -120,7 +120,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat const account = actor.Account if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) return sequelizeTypescript.transaction(async t => { const videoAbuseData = { diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 9e1664fd8..631a9dde7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) return sequelizeTypescript.transaction(async t => { const rate = { diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 0eb5fa392..b78de6697 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -54,7 +54,7 @@ export { async function processUndoLike (actorUrl: string, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike - const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) @@ -78,7 +78,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) { async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject - const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) @@ -102,7 +102,7 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { const cacheFileObject = activity.object.object as CacheFileObject - const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) return sequelizeTypescript.transaction(async t => { const byActor = await ActorModel.loadByUrl(actorUrl) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d3af1a181..935da5a54 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -48,7 +48,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) return undefined } - const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) @@ -64,7 +64,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) if (!redundancyModel) { - const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) return createCacheFile(cacheFileObject, video, byActor) } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index ffbd3a64e..4ca8bf659 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { try { // Maybe it's a reply to a video? // If yes, it's done: we resolved all the thread - const { video } = await getOrCreateVideoAndAccountAndChannel(url) + const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url }) if (comments.length !== 0) { const firstReply = comments[ comments.length - 1 ] diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5150c9975..5aabd3e0d 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -3,7 +3,7 @@ import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' import { join } from 'path' import * as request from 'request' -import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' +import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' @@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' +import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -50,13 +51,24 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr } } -function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { - const host = video.VideoChannel.Account.Actor.Server.host +async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { + const options = { + uri: videoUrl, + method: 'GET', + json: true, + activityPub: true + } - // We need to provide a callback, if no we could have an uncaught exception - return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { - if (err) reject(err) - }) + logger.info('Fetching remote video %s.', videoUrl) + + const { response, body } = await doRequest(options) + + if (sanitizeAndCheckVideoTorrentObject(body) === false) { + logger.debug('Remote video JSON is not valid.', { body }) + return { response, videoObject: undefined } + } + + return { response, videoObject: body } } async function fetchRemoteVideoDescription (video: VideoModel) { @@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) { return body.description ? body.description : '' } +function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { + const host = video.VideoChannel.Account.Actor.Server.host + + // We need to provide a callback, if no we could have an uncaught exception + return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { + if (err) reject(err) + }) +} + function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { const thumbnailName = video.getThumbnailName() const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) @@ -82,94 +103,6 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) return doRequestAndSaveToFile(options, thumbnailPath) } -async function videoActivityObjectToDBAttributes ( - videoChannel: VideoChannelModel, - videoObject: VideoTorrentObject, - to: string[] = [] -) { - const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED - const duration = videoObject.duration.replace(/[^\d]+/, '') - - let language: string | undefined - if (videoObject.language) { - language = videoObject.language.identifier - } - - let category: number | undefined - if (videoObject.category) { - category = parseInt(videoObject.category.identifier, 10) - } - - let licence: number | undefined - if (videoObject.licence) { - licence = parseInt(videoObject.licence.identifier, 10) - } - - const description = videoObject.content || null - const support = videoObject.support || null - - return { - name: videoObject.name, - uuid: videoObject.uuid, - url: videoObject.id, - category, - licence, - language, - description, - support, - nsfw: videoObject.sensitive, - commentsEnabled: videoObject.commentsEnabled, - waitTranscoding: videoObject.waitTranscoding, - state: videoObject.state, - channelId: videoChannel.id, - duration: parseInt(duration, 10), - createdAt: new Date(videoObject.published), - publishedAt: new Date(videoObject.published), - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: new Date(videoObject.updated), - views: videoObject.views, - likes: 0, - dislikes: 0, - remote: true, - privacy - } -} - -function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { - const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] - - if (fileUrls.length === 0) { - throw new Error('Cannot find video files for ' + videoCreated.url) - } - - const attributes: VideoFileModel[] = [] - for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = videoObject.url.find(u => { - return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height - }) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) - - const parsed = magnetUtil.decode(magnet.href) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { - throw new Error('Cannot parse magnet URI ' + magnet.href) - } - - const attribute = { - extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], - infoHash: parsed.infoHash, - resolution: fileUrl.height, - size: fileUrl.size, - videoId: videoCreated.id, - fps: fileUrl.fps - } as VideoFileModel - attributes.push(attribute) - } - - return attributes -} - function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { const channel = videoObject.attributedTo.find(a => a.type === 'Group') if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) @@ -177,51 +110,6 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject return getOrCreateActorAndServerAndModel(channel.id) } -async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { - logger.debug('Adding remote video %s.', videoObject.id) - - const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) - const video = VideoModel.build(videoData) - - const videoCreated = await video.save(sequelizeOptions) - - // Process files - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) - if (videoFileAttributes.length === 0) { - throw new Error('Cannot find valid files for video %s ' + videoObject.url) - } - - const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) - await Promise.all(videoFilePromises) - - // Process tags - const tags = videoObject.tag.map(t => t.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await videoCreated.$set('Tags', tagInstances, sequelizeOptions) - - // Process captions - const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) - }) - await Promise.all(videoCaptionsPromises) - - logger.info('Remote video with uuid %s inserted.', videoObject.uuid) - - videoCreated.VideoChannel = channelActor.VideoChannel - return videoCreated - }) - - const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) - - if (waitThumbnail === true) await p - - return videoCreated -} - type SyncParam = { likes: boolean dislikes: boolean @@ -230,28 +118,7 @@ type SyncParam = { thumbnail: boolean refreshVideo: boolean } -async function getOrCreateVideoAndAccountAndChannel ( - videoObject: VideoTorrentObject | string, - syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } -) { - const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id - - let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) - if (videoFromDatabase) { - const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) - if (syncParam.refreshVideo === true) videoFromDatabase = await p - - return { video: videoFromDatabase } - } - - const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) - if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - - const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) - const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) - - // Process outside the transaction because we could fetch remote data - +async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) const jobPayloads: ActivitypubHttpFetcherPayload[] = [] @@ -285,54 +152,37 @@ async function getOrCreateVideoAndAccountAndChannel ( } await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) - - return { video } } -async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { - const options = { - uri: videoUrl, - method: 'GET', - json: true, - activityPub: true - } +async function getOrCreateVideoAndAccountAndChannel (options: { + videoObject: VideoTorrentObject | string, + syncParam?: SyncParam, + fetchType?: VideoFetchByUrlType +}) { + // Default params + const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } + const fetchType = options.fetchType || 'all' - logger.info('Fetching remote video %s.', videoUrl) + // Get video url + const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id - const { response, body } = await doRequest(options) + let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + if (videoFromDatabase) { + const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam) + if (syncParam.refreshVideo === true) videoFromDatabase = await p - if (sanitizeAndCheckVideoTorrentObject(body) === false) { - logger.debug('Remote video JSON is not valid.', { body }) - return { response, videoObject: undefined } + return { video: videoFromDatabase } } - return { response, videoObject: body } -} - -async function refreshVideoIfNeeded (video: VideoModel): Promise { - if (!video.isOutdated()) return video - - try { - const { response, videoObject } = await fetchRemoteVideo(video.url) - if (response.statusCode === 404) { - // Video does not exist anymore - await video.destroy() - return undefined - } + const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) + if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video: invalid body.') - return video - } + const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) + const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) - const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) - const account = await AccountModel.load(channelActor.VideoChannel.accountId) + await syncVideoExternalAttributes(video, fetchedVideo, syncParam) - return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) - } catch (err) { - logger.warn('Cannot refresh video.', { err }) - return video - } + return { video } } async function updateVideoFromAP ( @@ -433,12 +283,7 @@ export { fetchRemoteVideoStaticFile, fetchRemoteVideoDescription, generateThumbnailFromUrl, - videoActivityObjectToDBAttributes, - videoFileActivityUrlToDBAttributes, - createVideo, - getOrCreateVideoChannelFromVideoObject, - addVideoShares, - createRates + getOrCreateVideoChannelFromVideoObject } // --------------------------------------------------------------------------- @@ -448,3 +293,166 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') } + +async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { + logger.debug('Adding remote video %s.', videoObject.id) + + const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) + const video = VideoModel.build(videoData) + + const videoCreated = await video.save(sequelizeOptions) + + // Process files + const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) + if (videoFileAttributes.length === 0) { + throw new Error('Cannot find valid files for video %s ' + videoObject.url) + } + + const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) + await Promise.all(videoFilePromises) + + // Process tags + const tags = videoObject.tag.map(t => t.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await videoCreated.$set('Tags', tagInstances, sequelizeOptions) + + // Process captions + const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) + }) + await Promise.all(videoCaptionsPromises) + + logger.info('Remote video with uuid %s inserted.', videoObject.uuid) + + videoCreated.VideoChannel = channelActor.VideoChannel + return videoCreated + }) + + const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) + + if (waitThumbnail === true) await p + + return videoCreated +} + +async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise { + // We need more attributes if the argument video was fetched with not enough joints + const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url) + + if (!video.isOutdated()) return video + + try { + const { response, videoObject } = await fetchRemoteVideo(video.url) + if (response.statusCode === 404) { + // Video does not exist anymore + await video.destroy() + return undefined + } + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video: invalid body.') + return video + } + + const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) + const account = await AccountModel.load(channelActor.VideoChannel.accountId) + + await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) + await syncVideoExternalAttributes(video, videoObject, syncParam) + } catch (err) { + logger.warn('Cannot refresh video.', { err }) + return video + } +} + +async function videoActivityObjectToDBAttributes ( + videoChannel: VideoChannelModel, + videoObject: VideoTorrentObject, + to: string[] = [] +) { + const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED + const duration = videoObject.duration.replace(/[^\d]+/, '') + + let language: string | undefined + if (videoObject.language) { + language = videoObject.language.identifier + } + + let category: number | undefined + if (videoObject.category) { + category = parseInt(videoObject.category.identifier, 10) + } + + let licence: number | undefined + if (videoObject.licence) { + licence = parseInt(videoObject.licence.identifier, 10) + } + + const description = videoObject.content || null + const support = videoObject.support || null + + return { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category, + licence, + language, + description, + support, + nsfw: videoObject.sensitive, + commentsEnabled: videoObject.commentsEnabled, + waitTranscoding: videoObject.waitTranscoding, + state: videoObject.state, + channelId: videoChannel.id, + duration: parseInt(duration, 10), + createdAt: new Date(videoObject.published), + publishedAt: new Date(videoObject.published), + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: new Date(videoObject.updated), + views: videoObject.views, + likes: 0, + dislikes: 0, + remote: true, + privacy + } +} + +function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { + const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] + + if (fileUrls.length === 0) { + throw new Error('Cannot find video files for ' + videoCreated.url) + } + + const attributes: VideoFileModel[] = [] + for (const fileUrl of fileUrls) { + // Fetch associated magnet uri + const magnet = videoObject.url.find(u => { + return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height + }) + + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) + + const parsed = magnetUtil.decode(magnet.href) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { + throw new Error('Cannot parse magnet URI ' + magnet.href) + } + + const attribute = { + extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], + infoHash: parsed.infoHash, + resolution: fileUrl.height, + size: fileUrl.size, + videoId: videoCreated.id, + fps: fileUrl.fps + } as VideoFileModel + attributes.push(attribute) + } + + return attributes +} diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 72d670277..42217c27c 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -1,10 +1,10 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { processActivities } from '../../activitypub/process' -import { VideoModel } from '../../../models/video/video' -import { addVideoShares, createRates } from '../../activitypub/videos' import { addVideoComments } from '../../activitypub/video-comments' import { crawlCollectionPage } from '../../activitypub/crawl' +import { VideoModel } from '../../../models/video/video' +import { addVideoShares, createRates } from '../../activitypub' type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 8aa7b3a39..67eabe468 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts @@ -26,8 +26,7 @@ import { isVideoPrivacyValid, isVideoRatingTypeValid, isVideoSupportValid, - isVideoTagsValid, - VideoFetchType + isVideoTagsValid } from '../../helpers/custom-validators/videos' import { getDurationFromVideoFile } from '../../helpers/ffmpeg-utils' import { logger } from '../../helpers/logger' @@ -42,6 +41,7 @@ import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } f import { VideoChangeOwnershipAccept } from '../../../shared/models/videos/video-change-ownership-accept.model' import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership' import { AccountModel } from '../../models/account/account' +import { VideoFetchType } from '../../helpers/video' const videosAddValidator = getCommonVideoAttributes().concat([ body('videofile') diff --git a/server/models/video/video.ts b/server/models/video/video.ts index ce2153f87..6c89c16bf 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1103,14 +1103,24 @@ export class VideoModel extends Model { .findOne(options) } - static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { + static loadByUrl (url: string, transaction?: Sequelize.Transaction) { const query: IFindOptions = { where: { url - } + }, + transaction } - if (t !== undefined) query.transaction = t + return VideoModel.findOne(query) + } + + static loadByUrlAndPopulateAccount (url: string, transaction?: Sequelize.Transaction) { + const query: IFindOptions = { + where: { + url + }, + transaction + } return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) } -- 2.41.0