6 files changed, 281 insertions, 0 deletions

diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts
new file mode 100644
index 000000000..886b1f288
--- /dev/null
+++ b/server/lib/activitypub/playlists/create-update.ts
@@ -0,0 +1,146 @@
+import { isArray } from '@server/helpers/custom-validators/misc'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { createPlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
+import { FilteredModelAttributes } from '@server/types'
+import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
+import { PlaylistObject } from '@shared/models'
+import { getOrCreateActorAndServerAndModel } from '../actor'
+import { crawlCollectionPage } from '../crawl'
+import { getOrCreateAPVideo } from '../videos'
+import {
+  fetchRemotePlaylistElement,
+  fetchRemoteVideoPlaylist,
+  playlistElementObjectToDBAttributes,
+  playlistObjectToDBAttributes
+} from './shared'
+
+import Bluebird = require('bluebird')
+
+const lTags = loggerTagsFactory('ap', 'video-playlist')
+
+async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
+  await Bluebird.map(playlistUrls, async playlistUrl => {
+    try {
+      const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
+      if (exists === true) return
+
+      const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
+
+      if (playlistObject === undefined) {
+        throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
+      }
+
+      return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
+    } catch (err) {
+      logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+}
+
+async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
+  const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
+
+  await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
+
+  const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
+
+  const playlistElementUrls = await fetchElementUrls(playlistObject)
+
+  // Refetch playlist from DB since elements fetching could be long in time
+  const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
+
+  try {
+    await updatePlaylistThumbnail(playlistObject, playlist)
+  } catch (err) {
+    logger.warn('Cannot update thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
+  }
+
+  return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  createAccountPlaylists,
+  createOrUpdateVideoPlaylist
+}
+
+// ---------------------------------------------------------------------------
+
+async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
+  if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
+
+  const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
+
+  if (!actor.VideoChannel) {
+    logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
+    return
+  }
+
+  playlistAttributes.videoChannelId = actor.VideoChannel.id
+}
+
+async function fetchElementUrls (playlistObject: PlaylistObject) {
+  let accItems: string[] = []
+  await crawlCollectionPage<string>(playlistObject.id, items => {
+    accItems = accItems.concat(items)
+
+    return Promise.resolve()
+  })
+
+  return accItems
+}
+
+async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
+  if (playlistObject.icon) {
+    const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
+    await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
+
+    return
+  }
+
+  // Playlist does not have an icon, destroy existing one
+  if (playlist.hasThumbnail()) {
+    await playlist.Thumbnail.destroy()
+    playlist.Thumbnail = null
+  }
+}
+
+async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
+  const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
+
+  await sequelizeTypescript.transaction(async t => {
+    await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
+
+    for (const element of elementsToCreate) {
+      await VideoPlaylistElementModel.create(element, { transaction: t })
+    }
+  })
+
+  logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
+
+  return undefined
+}
+
+async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
+  const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
+
+  await Bluebird.map(elementUrls, async elementUrl => {
+    try {
+      const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
+
+      const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
+
+      elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
+    } catch (err) {
+      logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+
+  return elementsToCreate
+}
diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts
new file mode 100644
index 000000000..2885830b4
--- /dev/null
+++ b/server/lib/activitypub/playlists/index.ts
@@ -0,0 +1,2 @@
+export * from './create-update'
+export * from './refresh'
diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts
new file mode 100644
index 000000000..ff9e5471a
--- /dev/null
+++ b/server/lib/activitypub/playlists/refresh.ts
@@ -0,0 +1,44 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { PeerTubeRequestError } from '@server/helpers/requests'
+import { MVideoPlaylistOwner } from '@server/types/models'
+import { HttpStatusCode } from '@shared/core-utils'
+import { createOrUpdateVideoPlaylist } from './create-update'
+import { fetchRemoteVideoPlaylist } from './shared'
+
+async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
+  if (!videoPlaylist.isOutdated()) return videoPlaylist
+
+  const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
+
+  try {
+    const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
+
+    if (playlistObject === undefined) {
+      logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags())
+
+      await videoPlaylist.setAsRefreshed()
+      return videoPlaylist
+    }
+
+    const byAccount = videoPlaylist.OwnerAccount
+    await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
+
+    return videoPlaylist
+  } catch (err) {
+    if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
+      logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags())
+
+      await videoPlaylist.destroy()
+      return undefined
+    }
+
+    logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() })
+
+    await videoPlaylist.setAsRefreshed()
+    return videoPlaylist
+  }
+}
+
+export {
+  refreshVideoPlaylistIfNeeded
+}
diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts
new file mode 100644
index 000000000..a217f2291
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/index.ts
@@ -0,0 +1,2 @@
+export * from './object-to-model-attributes'
+export * from './url-to-object'
diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
new file mode 100644
index 000000000..6ec44485e
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts
@@ -0,0 +1,40 @@
+import { ACTIVITY_PUB } from '@server/initializers/constants'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
+import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
+import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
+
+function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
+  const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
+    ? VideoPlaylistPrivacy.PUBLIC
+    : VideoPlaylistPrivacy.UNLISTED
+
+  return {
+    name: playlistObject.name,
+    description: playlistObject.content,
+    privacy,
+    url: playlistObject.id,
+    uuid: playlistObject.uuid,
+    ownerAccountId: byAccount.id,
+    videoChannelId: null,
+    createdAt: new Date(playlistObject.published),
+    updatedAt: new Date(playlistObject.updated)
+  } as AttributesOnly<VideoPlaylistModel>
+}
+
+function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
+  return {
+    position: elementObject.position,
+    url: elementObject.id,
+    startTimestamp: elementObject.startTimestamp || null,
+    stopTimestamp: elementObject.stopTimestamp || null,
+    videoPlaylistId: videoPlaylist.id,
+    videoId: video.id
+  } as AttributesOnly<VideoPlaylistElementModel>
+}
+
+export {
+  playlistObjectToDBAttributes,
+  playlistElementObjectToDBAttributes
+}
diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts
new file mode 100644
index 000000000..ec8c01255
--- /dev/null
+++ b/server/lib/activitypub/playlists/shared/url-to-object.ts
@@ -0,0 +1,47 @@
+import { isArray } from 'lodash'
+import { checkUrlsSameHost } from '@server/helpers/activitypub'
+import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { doJSONRequest } from '@server/helpers/requests'
+import { PlaylistElementObject, PlaylistObject } from '@shared/models'
+
+async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
+  const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl)
+
+  logger.info('Fetching remote playlist %s.', playlistUrl, lTags())
+
+  const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
+
+  if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
+    logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() })
+    return { statusCode, playlistObject: undefined }
+  }
+
+  if (!isArray(body.to)) {
+    logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() })
+    return { statusCode, playlistObject: undefined }
+  }
+
+  return { statusCode, playlistObject: body }
+}
+
+async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> {
+  const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl)
+
+  logger.debug('Fetching remote playlist element %s.', elementUrl, lTags())
+
+  const { body, statusCode } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
+
+  if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`)
+
+  if (checkUrlsSameHost(body.id, elementUrl) !== true) {
+    throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
+  }
+
+  return { statusCode, elementObject: body }
+}
+
+export {
+  fetchRemoteVideoPlaylist,
+  fetchRemotePlaylistElement
+}