Playlist server API

10 files changed, 292 insertions, 7 deletions

diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index a3f379b76..f77df8b78 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -44,6 +44,7 @@ async function getOrCreateActorAndServerAndModel (
 ) {
   const actorUrl = getAPId(activityActor)
   let created = false
+  let accountPlaylistsUrl: string
 
   let actor = await fetchActorByUrl(actorUrl, fetchType)
   // Orphan actor (not associated to an account of channel) so recreate it
@@ -70,7 +71,8 @@ async function getOrCreateActorAndServerAndModel (
 
       try {
         // Don't recurse another time
-        ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
+        const recurseIfNeeded = false
+        ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
       } catch (err) {
         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
         throw new Error(err)
@@ -79,6 +81,7 @@ async function getOrCreateActorAndServerAndModel (
 
     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
     created = true
+    accountPlaylistsUrl = result.playlists
   }
 
   if (actor.Account) actor.Account.Actor = actor
@@ -92,6 +95,12 @@ async function getOrCreateActorAndServerAndModel (
     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
   }
 
+  // We created a new account: fetch the playlists
+  if (created === true && actor.Account && accountPlaylistsUrl) {
+    const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
+    await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
+  }
+
   return actorRefreshed
 }
 
@@ -342,6 +351,7 @@ type FetchRemoteActorResult = {
   name: string
   summary: string
   support?: string
+  playlists?: string
   avatarName?: string
   attributedTo: ActivityPubAttributedTo[]
 }
@@ -398,6 +408,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
       avatarName,
       summary: actorJSON.summary,
       support: actorJSON.support,
+      playlists: actorJSON.playlists,
       attributedTo: actorJSON.attributedTo
     }
   }
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 9a40414bb..597003135 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,4 +1,4 @@
-import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
+import { CacheFileObject } from '../../../shared/index'
 import { VideoModel } from '../../models/video/video'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { Transaction } from 'sequelize'
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 1b9b14c2e..2675524c6 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -4,7 +4,7 @@ import { logger } from '../../helpers/logger'
 import * as Bluebird from 'bluebird'
 import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
 
-async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) {
+async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (Promise<any> | Bluebird<any>)) {
   logger.info('Crawling ActivityPub data on %s.', uri)
 
   const options = {
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
new file mode 100644
index 000000000..c9b428c92
--- /dev/null
+++ b/server/lib/activitypub/playlist.ts
@@ -0,0 +1,162 @@
+import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
+import { crawlCollectionPage } from './crawl'
+import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers'
+import { AccountModel } from '../../models/account/account'
+import { isArray } from '../../helpers/custom-validators/misc'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { logger } from '../../helpers/logger'
+import { VideoPlaylistModel } from '../../models/video/video-playlist'
+import { doRequest, downloadImage } from '../../helpers/requests'
+import { checkUrlsSameHost } from '../../helpers/activitypub'
+import * as Bluebird from 'bluebird'
+import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
+import { getOrCreateVideoAndAccountAndChannel } from './videos'
+import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
+import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
+import { VideoModel } from '../../models/video/video'
+import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
+import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
+import { ActivityIconObject } from '../../../shared/models/activitypub/objects'
+
+function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
+  const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
+
+  return {
+    name: playlistObject.name,
+    description: playlistObject.content,
+    privacy,
+    url: playlistObject.id,
+    uuid: playlistObject.uuid,
+    ownerAccountId: byAccount.id,
+    videoChannelId: null
+  }
+}
+
+function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) {
+  return {
+    position: elementObject.position,
+    url: elementObject.id,
+    startTimestamp: elementObject.startTimestamp || null,
+    stopTimestamp: elementObject.stopTimestamp || null,
+    videoPlaylistId: videoPlaylist.id,
+    videoId: video.id
+  }
+}
+
+async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) {
+  await Bluebird.map(playlistUrls, async playlistUrl => {
+    try {
+      const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
+      if (exists === true) return
+
+      // Fetch url
+      const { body } = await doRequest<PlaylistObject>({
+        uri: playlistUrl,
+        json: true,
+        activityPub: true
+      })
+
+      if (!isPlaylistObjectValid(body)) {
+        throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
+      }
+
+      if (!isArray(body.to)) {
+        throw new Error('Playlist does not have an audience.')
+      }
+
+      return createOrUpdateVideoPlaylist(body, account, body.to)
+    } catch (err) {
+      logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+}
+
+async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
+  const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
+
+  if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
+    const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
+
+    if (actor.VideoChannel) {
+      playlistAttributes.videoChannelId = actor.VideoChannel.id
+    } else {
+      logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
+    }
+  }
+
+  const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true })
+
+  let accItems: string[] = []
+  await crawlCollectionPage<string>(playlistObject.id, items => {
+    accItems = accItems.concat(items)
+
+    return Promise.resolve()
+  })
+
+  // Empty playlists generally do not have a miniature, so skip it
+  if (accItems.length !== 0) {
+    try {
+      await generateThumbnailFromUrl(playlist, playlistObject.icon)
+    } catch (err) {
+      logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
+    }
+  }
+
+  return resetVideoPlaylistElements(accItems, playlist)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  createAccountPlaylists,
+  playlistObjectToDBAttributes,
+  playlistElementObjectToDBAttributes,
+  createOrUpdateVideoPlaylist
+}
+
+// ---------------------------------------------------------------------------
+
+async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) {
+  const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
+
+  await Bluebird.map(elementUrls, async elementUrl => {
+    try {
+      // Fetch url
+      const { body } = await doRequest<PlaylistElementObject>({
+        uri: elementUrl,
+        json: true,
+        activityPub: true
+      })
+
+      if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
+
+      if (checkUrlsSameHost(body.id, elementUrl) !== true) {
+        throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
+      }
+
+      const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
+
+      elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
+    } catch (err) {
+      logger.warn('Cannot add playlist element %s.', elementUrl, { err })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+
+  await sequelizeTypescript.transaction(async t => {
+    await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
+
+    for (const element of elementsToCreate) {
+      await VideoPlaylistElementModel.create(element, { transaction: t })
+    }
+  })
+
+  logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
+
+  return undefined
+}
+
+function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityIconObject) {
+  const thumbnailName = playlist.getThumbnailName()
+
+  return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
+}
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 5f4d793a5..e882669ce 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -12,6 +12,8 @@ import { Notifier } from '../../notifier'
 import { processViewActivity } from './process-view'
 import { processDislikeActivity } from './process-dislike'
 import { processFlagActivity } from './process-flag'
+import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
+import { createOrUpdateVideoPlaylist } from '../playlist'
 
 async function processCreateActivity (activity: ActivityCreate, byActor: ActorModel) {
   const activityObject = activity.object
@@ -38,7 +40,11 @@ async function processCreateActivity (activity: ActivityCreate, byActor: ActorMo
   }
 
   if (activityType === 'CacheFile') {
-    return retryTransactionWrapper(processCacheFile, activity, byActor)
+    return retryTransactionWrapper(processCreateCacheFile, activity, byActor)
+  }
+
+  if (activityType === 'Playlist') {
+    return retryTransactionWrapper(processCreatePlaylist, activity, byActor)
   }
 
   logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
@@ -63,7 +69,7 @@ async function processCreateVideo (activity: ActivityCreate) {
   return video
 }
 
-async function processCacheFile (activity: ActivityCreate, byActor: ActorModel) {
+async function processCreateCacheFile (activity: ActivityCreate, byActor: ActorModel) {
   const cacheFile = activity.object as CacheFileObject
 
   const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object })
@@ -98,3 +104,12 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act
 
   if (created === true) Notifier.Instance.notifyOnNewComment(comment)
 }
+
+async function processCreatePlaylist (activity: ActivityCreate, byActor: ActorModel) {
+  const playlistObject = activity.object as PlaylistObject
+  const byAccount = byActor.Account
+
+  if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
+
+  await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to)
+}
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index c6b42d846..0b96ba352 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -12,6 +12,8 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-vali
 import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
 import { createOrUpdateCacheFile } from '../cache-file'
 import { forwardVideoRelatedActivity } from '../send/utils'
+import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
+import { createOrUpdateVideoPlaylist } from '../playlist'
 
 async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorModel) {
   const objectType = activity.object.type
@@ -32,6 +34,10 @@ async function processUpdateActivity (activity: ActivityUpdate, byActor: ActorMo
     return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity)
   }
 
+  if (objectType === 'Playlist') {
+    return retryTransactionWrapper(processUpdatePlaylist, byActor, activity)
+  }
+
   return undefined
 }
 
@@ -135,3 +141,12 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
     throw err
   }
 }
+
+async function processUpdatePlaylist (byActor: ActorModel, activity: ActivityUpdate) {
+  const playlistObject = activity.object as PlaylistObject
+  const byAccount = byActor.Account
+
+  if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
+
+  await createOrUpdateVideoPlaylist(playlistObject, byAccount, activity.to)
+}
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index ef20e404c..bacdb97e3 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -8,6 +8,9 @@ import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unic
 import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
 import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { VideoPlaylistModel } from '../../../models/video/video-playlist'
+import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
+import { getServerActor } from '../../../helpers/utils'
 
 async function sendCreateVideo (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -34,6 +37,25 @@ async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, file
   })
 }
 
+async function sendCreateVideoPlaylist (playlist: VideoPlaylistModel, t: Transaction) {
+  if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
+
+  logger.info('Creating job to send create video playlist of %s.', playlist.url)
+
+  const byActor = playlist.OwnerAccount.Actor
+  const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC)
+
+  const object = await playlist.toActivityPubObject()
+  const createActivity = buildCreateActivity(playlist.url, byActor, object, audience)
+
+  const serverActor = await getServerActor()
+  const toFollowersOf = [ byActor, serverActor ]
+
+  if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor)
+
+  return broadcastToFollowers(createActivity, byActor, toFollowersOf, t)
+}
+
 async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
   logger.info('Creating job to send comment %s.', comment.url)
 
@@ -92,6 +114,7 @@ export {
   sendCreateVideo,
   buildCreateActivity,
   sendCreateVideoComment,
+  sendCreateVideoPlaylist,
   sendCreateCacheFile
 }
 
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 18969433a..016811e60 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -8,6 +8,8 @@ import { getDeleteActivityPubUrl } from '../url'
 import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
 import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
+import { VideoPlaylistModel } from '../../../models/video/video-playlist'
+import { getServerActor } from '../../../helpers/utils'
 
 async function sendDeleteVideo (video: VideoModel, transaction: Transaction) {
   logger.info('Creating job to broadcast delete of video %s.', video.url)
@@ -64,12 +66,29 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
   return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
 }
 
+async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
+  logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url)
+
+  const byActor = videoPlaylist.OwnerAccount.Actor
+
+  const url = getDeleteActivityPubUrl(videoPlaylist.url)
+  const activity = buildDeleteActivity(url, videoPlaylist.url, byActor)
+
+  const serverActor = await getServerActor()
+  const toFollowersOf = [ byActor, serverActor ]
+
+  if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor)
+
+  return broadcastToFollowers(activity, byActor, toFollowersOf, t)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   sendDeleteVideo,
   sendDeleteActor,
-  sendDeleteVideoComment
+  sendDeleteVideoComment,
+  sendDeleteVideoPlaylist
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 839f66470..3eb2704fd 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -12,8 +12,13 @@ import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience'
 import { logger } from '../../../helpers/logger'
 import { VideoCaptionModel } from '../../../models/video/video-caption'
 import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { VideoPlaylistModel } from '../../../models/video/video-playlist'
+import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
+import { getServerActor } from '../../../helpers/utils'
 
 async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) {
+  if (video.privacy === VideoPrivacy.PRIVATE) return undefined
+
   logger.info('Creating job to update video %s.', video.url)
 
   const byActor = overrodeByActor ? overrodeByActor : video.VideoChannel.Account.Actor
@@ -73,12 +78,35 @@ async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoR
   return sendVideoRelatedActivity(activityBuilder, { byActor, video })
 }
 
+async function sendUpdateVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
+  if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
+
+  const byActor = videoPlaylist.OwnerAccount.Actor
+
+  logger.info('Creating job to update video playlist %s.', videoPlaylist.url)
+
+  const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString())
+
+  const object = await videoPlaylist.toActivityPubObject()
+  const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC)
+
+  const updateActivity = buildUpdateActivity(url, byActor, object, audience)
+
+  const serverActor = await getServerActor()
+  const toFollowersOf = [ byActor, serverActor ]
+
+  if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor)
+
+  return broadcastToFollowers(updateActivity, byActor, toFollowersOf, t)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   sendUpdateActor,
   sendUpdateVideo,
-  sendUpdateCacheFile
+  sendUpdateCacheFile,
+  sendUpdateVideoPlaylist
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 4229fe094..00bbbba2d 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -7,11 +7,21 @@ import { VideoCommentModel } from '../../models/video/video-comment'
 import { VideoFileModel } from '../../models/video/video-file'
 import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
+import { VideoPlaylistModel } from '../../models/video/video-playlist'
+import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
 
 function getVideoActivityPubUrl (video: VideoModel) {
   return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
 }
 
+function getVideoPlaylistActivityPubUrl (videoPlaylist: VideoPlaylistModel) {
+  return CONFIG.WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid
+}
+
+function getVideoPlaylistElementActivityPubUrl (videoPlaylist: VideoPlaylistModel, video: VideoModel) {
+  return CONFIG.WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/' + video.uuid
+}
+
 function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
   const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : ''
 
@@ -98,6 +108,8 @@ function getUndoActivityPubUrl (originalUrl: string) {
 
 export {
   getVideoActivityPubUrl,
+  getVideoPlaylistElementActivityPubUrl,
+  getVideoPlaylistActivityPubUrl,
   getVideoCacheStreamingPlaylistActivityPubUrl,
   getVideoChannelActivityPubUrl,
   getAccountActivityPubUrl,