Improve create import file job

[github/Chocobozzz/PeerTube.git] / server / lib / activitypub / videos.ts

diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index e7b516129aa4a87b4d14db81acb155fe5aa1ad6f..907f7e11e273e55a559af20f7f9762de0498846f 100644 (file)
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -5,7 +5,7 @@ import * as request from 'request'
 import { ActivityIconObject } from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
-import { isVideoTorrentObjectValid } from '../../helpers/custom-validators/activitypub/videos'
+import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
 import { retryTransactionWrapper } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
@@ -20,6 +20,7 @@ import { VideoFileModel } from '../../models/video/video-file'
 import { VideoShareModel } from '../../models/video/video-share'
 import { getOrCreateActorAndServerAndModel } from './actor'
 import { addVideoComments } from './video-comments'
+import { crawlCollectionPage } from './crawl'
 
 function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
   const host = video.VideoChannel.Account.Actor.Server.host
@@ -58,19 +59,19 @@ async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelMode
                                                   videoObject: VideoTorrentObject,
                                                   to: string[] = []) {
   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
-
   const duration = videoObject.duration.replace(/[^\d]+/, '')
-  let language = null
+
+  let language: string = null
   if (videoObject.language) {
-    language = parseInt(videoObject.language.identifier, 10)
+    language = videoObject.language.identifier
   }
 
-  let category = null
+  let category: number = null
   if (videoObject.category) {
     category = parseInt(videoObject.category.identifier, 10)
   }
 
-  let licence = null
+  let licence: number = null
   if (videoObject.licence) {
     licence = parseInt(videoObject.licence.identifier, 10)
   }
@@ -92,6 +93,7 @@ async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelMode
     channelId: videoChannel.id,
     duration: parseInt(duration, 10),
     createdAt: new Date(videoObject.published),
+    publishedAt: new Date(videoObject.published),
     // FIXME: updatedAt does not seems to be considered by Sequelize
     updatedAt: new Date(videoObject.updated),
     views: videoObject.views,
@@ -137,6 +139,13 @@ function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObje
   return attributes
 }
 
+function getOrCreateVideoChannel (videoObject: VideoTorrentObject) {
+  const channel = videoObject.attributedTo.find(a => a.type === 'Group')
+  if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
+
+  return getOrCreateActorAndServerAndModel(channel.id)
+}
+
 async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) {
   logger.debug('Adding remote video %s.', videoObject.id)
 
@@ -152,7 +161,7 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
 
     // Don't block on request
     generateThumbnailFromUrl(video, videoObject.icon)
-      .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, err))
+      .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
 
     const videoCreated = await video.save(sequelizeOptions)
 
@@ -177,7 +186,9 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
 
 async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) {
   if (typeof videoObject === 'string') {
-    const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoObject)
+    const videoUrl = videoObject
+
+    const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
     if (videoFromDatabase) {
       return {
         video: videoFromDatabase,
@@ -186,8 +197,8 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb
       }
     }
 
-    videoObject = await fetchRemoteVideo(videoObject)
-    if (!videoObject) throw new Error('Cannot fetch remote video')
+    videoObject = await fetchRemoteVideo(videoUrl)
+    if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
   }
 
   if (!actor) {
@@ -197,10 +208,7 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb
     actor = await getOrCreateActorAndServerAndModel(actorObj.id)
   }
 
-  const channel = videoObject.attributedTo.find(a => a.type === 'Group')
-  if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
-
-  const channelActor = await getOrCreateActorAndServerAndModel(channel.id)
+  const channelActor = await getOrCreateVideoChannel(videoObject)
 
   const options = {
     arguments: [ videoObject, channelActor ],
@@ -210,25 +218,17 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb
   const video = await retryTransactionWrapper(getOrCreateVideo, options)
 
   // Process outside the transaction because we could fetch remote data
-  if (videoObject.likes && Array.isArray(videoObject.likes.orderedItems)) {
-    logger.info('Adding likes of video %s.', video.uuid)
-    await createRates(videoObject.likes.orderedItems, video, 'like')
-  }
+  logger.info('Adding likes of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like'))
 
-  if (videoObject.dislikes && Array.isArray(videoObject.dislikes.orderedItems)) {
-    logger.info('Adding dislikes of video %s.', video.uuid)
-    await createRates(videoObject.dislikes.orderedItems, video, 'dislike')
-  }
+  logger.info('Adding dislikes of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike'))
 
-  if (videoObject.shares && Array.isArray(videoObject.shares.orderedItems)) {
-    logger.info('Adding shares of video %s.', video.uuid)
-    await addVideoShares(video, videoObject.shares.orderedItems)
-  }
+  logger.info('Adding shares of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video))
 
-  if (videoObject.comments && Array.isArray(videoObject.comments.orderedItems)) {
-    logger.info('Adding comments of video %s.', video.uuid)
-    await addVideoComments(video, videoObject.comments.orderedItems)
-  }
+  logger.info('Adding comments of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video))
 
   return { actor, channelActor, video }
 }
@@ -260,7 +260,7 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
   return
 }
 
-async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
+async function addVideoShares (shareUrls: string[], instance: VideoModel) {
   for (const shareUrl of shareUrls) {
     // Fetch url
     const { body } = await doRequest({
@@ -268,9 +268,12 @@ async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
       json: true,
       activityPub: true
     })
-    const actorUrl = body.actor
-    if (!actorUrl) continue
+    if (!body || !body.actor) {
+      logger.warn('Cannot add remote share with url: %s, skipping...', shareUrl)
+      continue
+    }
 
+    const actorUrl = body.actor
     const actor = await getOrCreateActorAndServerAndModel(actorUrl)
 
     const entry = {
@@ -296,7 +299,9 @@ export {
   videoActivityObjectToDBAttributes,
   videoFileActivityUrlToDBAttributes,
   getOrCreateVideo,
-  addVideoShares}
+  getOrCreateVideoChannel,
+  addVideoShares
+}
 
 // ---------------------------------------------------------------------------
 
@@ -312,7 +317,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject>
 
   const { body } = await doRequest(options)
 
-  if (isVideoTorrentObjectValid(body) === false) {
+  if (sanitizeAndCheckVideoTorrentObject(body) === false) {
     logger.debug('Remote video JSON is not valid.', { body })
     return undefined
   }