6 files changed, 198 insertions, 181 deletions

diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts
deleted file mode 100644
index 5113c9d7e..000000000
--- a/server/lib/activitypub/videos/fetch.ts
+++ /dev/null
@@ -1,180 +0,0 @@
-import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub'
-import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
-import { retryTransactionWrapper } from '@server/helpers/database-utils'
-import { logger } from '@server/helpers/logger'
-import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
-import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video'
-import { REMOTE_SCHEME } from '@server/initializers/constants'
-import { ActorFollowScoreCache } from '@server/lib/files-cache'
-import { JobQueue } from '@server/lib/job-queue'
-import { VideoModel } from '@server/models/video/video'
-import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
-import { HttpStatusCode } from '@shared/core-utils'
-import { VideoObject } from '@shared/models'
-import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared'
-import { APVideoUpdater } from './updater'
-
-async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
-  logger.info('Fetching remote video %s.', videoUrl)
-
-  const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
-
-  if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
-    logger.debug('Remote video JSON is not valid.', { body })
-    return { statusCode, videoObject: undefined }
-  }
-
-  return { statusCode, videoObject: body }
-}
-
-async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
-  const host = video.VideoChannel.Account.Actor.Server.host
-  const path = video.getDescriptionAPIPath()
-  const url = REMOTE_SCHEME.HTTP + '://' + host + path
-
-  const { body } = await doJSONRequest<any>(url)
-  return body.description || ''
-}
-
-type GetVideoResult <T> = Promise<{
-  video: T
-  created: boolean
-  autoBlacklisted?: boolean
-}>
-
-type GetVideoParamAll = {
-  videoObject: { id: string } | string
-  syncParam?: SyncParam
-  fetchType?: 'all'
-  allowRefresh?: boolean
-}
-
-type GetVideoParamImmutable = {
-  videoObject: { id: string } | string
-  syncParam?: SyncParam
-  fetchType: 'only-immutable-attributes'
-  allowRefresh: false
-}
-
-type GetVideoParamOther = {
-  videoObject: { id: string } | string
-  syncParam?: SyncParam
-  fetchType?: 'all' | 'only-video'
-  allowRefresh?: boolean
-}
-
-function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
-function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
-function getOrCreateVideoAndAccountAndChannel (
-  options: GetVideoParamOther
-): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
-async function getOrCreateVideoAndAccountAndChannel (
-  options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
-): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
-  // Default params
-  const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
-  const fetchType = options.fetchType || 'all'
-  const allowRefresh = options.allowRefresh !== false
-
-  // Get video url
-  const videoUrl = getAPId(options.videoObject)
-  let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
-
-  if (videoFromDatabase) {
-    // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
-    if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
-      const refreshOptions = {
-        video: videoFromDatabase as MVideoThumbnail,
-        fetchedType: fetchType,
-        syncParam
-      }
-
-      if (syncParam.refreshVideo === true) {
-        videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
-      } else {
-        await JobQueue.Instance.createJobWithPromise({
-          type: 'activitypub-refresher',
-          payload: { type: 'video', url: videoFromDatabase.url }
-        })
-      }
-    }
-
-    return { video: videoFromDatabase, created: false }
-  }
-
-  const { videoObject } = await fetchRemoteVideo(videoUrl)
-  if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
-
-  try {
-    const creator = new APVideoCreator(videoObject)
-    const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
-
-    await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
-
-    return { video: videoCreated, created: true, autoBlacklisted }
-  } catch (err) {
-    // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
-    if (err.name === 'SequelizeUniqueConstraintError') {
-      const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
-      if (fallbackVideo) return { video: fallbackVideo, created: false }
-    }
-
-    throw err
-  }
-}
-
-async function refreshVideoIfNeeded (options: {
-  video: MVideoThumbnail
-  fetchedType: VideoFetchByUrlType
-  syncParam: SyncParam
-}): Promise<MVideoThumbnail> {
-  if (!options.video.isOutdated()) return options.video
-
-  // We need more attributes if the argument video was fetched with not enough joints
-  const video = options.fetchedType === 'all'
-    ? options.video as MVideoAccountLightBlacklistAllFiles
-    : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
-
-  try {
-    const { videoObject } = await fetchRemoteVideo(video.url)
-
-    if (videoObject === undefined) {
-      logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
-
-      await video.setAsRefreshed()
-      return video
-    }
-
-    const videoUpdater = new APVideoUpdater(videoObject, video)
-    await videoUpdater.update()
-
-    await syncVideoExternalAttributes(video, videoObject, options.syncParam)
-
-    ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
-
-    return video
-  } catch (err) {
-    if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
-      logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
-
-      // Video does not exist anymore
-      await video.destroy()
-      return undefined
-    }
-
-    logger.warn('Cannot refresh video %s.', options.video.url, { err })
-
-    ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
-
-    // Don't refresh in loop
-    await video.setAsRefreshed()
-    return video
-  }
-}
-
-export {
-  fetchRemoteVideo,
-  fetchRemoteVideoDescription,
-  refreshVideoIfNeeded,
-  getOrCreateVideoAndAccountAndChannel
-}
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts
new file mode 100644
index 000000000..a8c41e178
--- /dev/null
+++ b/server/lib/activitypub/videos/get.ts
@@ -0,0 +1,109 @@
+import { getAPId } from '@server/helpers/activitypub'
+import { retryTransactionWrapper } from '@server/helpers/database-utils'
+import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video'
+import { JobQueue } from '@server/lib/job-queue'
+import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
+import { refreshVideoIfNeeded } from './refresh'
+import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
+
+type GetVideoResult <T> = Promise<{
+  video: T
+  created: boolean
+  autoBlacklisted?: boolean
+}>
+
+type GetVideoParamAll = {
+  videoObject: { id: string } | string
+  syncParam?: SyncParam
+  fetchType?: 'all'
+  allowRefresh?: boolean
+}
+
+type GetVideoParamImmutable = {
+  videoObject: { id: string } | string
+  syncParam?: SyncParam
+  fetchType: 'only-immutable-attributes'
+  allowRefresh: false
+}
+
+type GetVideoParamOther = {
+  videoObject: { id: string } | string
+  syncParam?: SyncParam
+  fetchType?: 'all' | 'only-video'
+  allowRefresh?: boolean
+}
+
+function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
+function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
+function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
+
+async function getOrCreateAPVideo (
+  options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
+): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
+  // Default params
+  const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
+  const fetchType = options.fetchType || 'all'
+  const allowRefresh = options.allowRefresh !== false
+
+  // Get video url
+  const videoUrl = getAPId(options.videoObject)
+  let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
+
+  if (videoFromDatabase) {
+    if (allowRefresh === true) {
+      // Typings ensure allowRefresh === false in only-immutable-attributes fetch type
+      videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam)
+    }
+
+    return { video: videoFromDatabase, created: false }
+  }
+
+  const { videoObject } = await fetchRemoteVideo(videoUrl)
+  if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
+
+  try {
+    const creator = new APVideoCreator(videoObject)
+    const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
+
+    await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
+
+    return { video: videoCreated, created: true, autoBlacklisted }
+  } catch (err) {
+    // Maybe a concurrent getOrCreateAPVideo call created this video
+    if (err.name === 'SequelizeUniqueConstraintError') {
+      const alreadyCreatedVideo = await fetchVideoByUrl(videoUrl, fetchType)
+      if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false }
+    }
+
+    throw err
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getOrCreateAPVideo
+}
+
+// ---------------------------------------------------------------------------
+
+async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoFetchByUrlType, syncParam: SyncParam) {
+  if (!video.isOutdated()) return video
+
+  const refreshOptions = {
+    video,
+    fetchedType: fetchType,
+    syncParam
+  }
+
+  if (syncParam.refreshVideo === true) {
+    return refreshVideoIfNeeded(refreshOptions)
+  }
+
+  await JobQueue.Instance.createJobWithPromise({
+    type: 'activitypub-refresher',
+    payload: { type: 'video', url: video.url }
+  })
+
+  return video
+}
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts
index b560acb76..b22062598 100644
--- a/server/lib/activitypub/videos/index.ts
+++ b/server/lib/activitypub/videos/index.ts
@@ -1,3 +1,4 @@
 export * from './federate'
-export * from './fetch'
+export * from './get'
+export * from './refresh'
 export * from './updater'
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts
new file mode 100644
index 000000000..205a3ccb1
--- /dev/null
+++ b/server/lib/activitypub/videos/refresh.ts
@@ -0,0 +1,64 @@
+import { logger } from '@server/helpers/logger'
+import { PeerTubeRequestError } from '@server/helpers/requests'
+import { VideoFetchByUrlType } from '@server/helpers/video'
+import { ActorFollowScoreCache } from '@server/lib/files-cache'
+import { VideoModel } from '@server/models/video/video'
+import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models'
+import { HttpStatusCode } from '@shared/core-utils'
+import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
+import { APVideoUpdater } from './updater'
+
+async function refreshVideoIfNeeded (options: {
+  video: MVideoThumbnail
+  fetchedType: VideoFetchByUrlType
+  syncParam: SyncParam
+}): Promise<MVideoThumbnail> {
+  if (!options.video.isOutdated()) return options.video
+
+  // We need more attributes if the argument video was fetched with not enough joints
+  const video = options.fetchedType === 'all'
+    ? options.video as MVideoAccountLightBlacklistAllFiles
+    : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
+
+  try {
+    const { videoObject } = await fetchRemoteVideo(video.url)
+
+    if (videoObject === undefined) {
+      logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
+
+      await video.setAsRefreshed()
+      return video
+    }
+
+    const videoUpdater = new APVideoUpdater(videoObject, video)
+    await videoUpdater.update()
+
+    await syncVideoExternalAttributes(video, videoObject, options.syncParam)
+
+    ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
+
+    return video
+  } catch (err) {
+    if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
+      logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
+
+      // Video does not exist anymore
+      await video.destroy()
+      return undefined
+    }
+
+    logger.warn('Cannot refresh video %s.', options.video.url, { err })
+
+    ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
+
+    // Don't refresh in loop
+    await video.setAsRefreshed()
+    return video
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  refreshVideoIfNeeded
+}
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts
index 208a43705..951403493 100644
--- a/server/lib/activitypub/videos/shared/index.ts
+++ b/server/lib/activitypub/videos/shared/index.ts
@@ -2,4 +2,5 @@ export * from './abstract-builder'
 export * from './creator'
 export * from './object-to-model-attributes'
 export * from './trackers'
+export * from './url-to-object'
 export * from './video-sync-attributes'
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts
new file mode 100644
index 000000000..b1ecac8ca
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/url-to-object.ts
@@ -0,0 +1,22 @@
+import { checkUrlsSameHost } from '@server/helpers/activitypub'
+import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
+import { logger } from '@server/helpers/logger'
+import { doJSONRequest } from '@server/helpers/requests'
+import { VideoObject } from '@shared/models'
+
+async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
+  logger.info('Fetching remote video %s.', videoUrl)
+
+  const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
+
+  if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
+    logger.debug('Remote video JSON is not valid.', { body })
+    return { statusCode, videoObject: undefined }
+  }
+
+  return { statusCode, videoObject: body }
+}
+
+export {
+  fetchRemoteVideo
+}