server/lib/activitypub/playlist.ts

   1 import * as Bluebird from 'bluebird'
   2 import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
   3 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
   4 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   5 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
   6 import { checkUrlsSameHost } from '../../helpers/activitypub'
   7 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
   8 import { isArray } from '../../helpers/custom-validators/misc'
   9 import { logger } from '../../helpers/logger'
  10 import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
  11 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
  12 import { sequelizeTypescript } from '../../initializers/database'
  13 import { VideoPlaylistModel } from '../../models/video/video-playlist'
  14 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  15 import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
  16 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
  17 import { FilteredModelAttributes } from '../../types/sequelize'
  18 import { createPlaylistMiniatureFromUrl } from '../thumbnail'
  19 import { getOrCreateActorAndServerAndModel } from './actor'
  20 import { crawlCollectionPage } from './crawl'
  21 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  22
  23 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  24   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
  25     ? VideoPlaylistPrivacy.PUBLIC
  26     : VideoPlaylistPrivacy.UNLISTED
  27
  28   return {
  29     name: playlistObject.name,
  30     description: playlistObject.content,
  31     privacy,
  32     url: playlistObject.id,
  33     uuid: playlistObject.uuid,
  34     ownerAccountId: byAccount.id,
  35     videoChannelId: null,
  36     createdAt: new Date(playlistObject.published),
  37     updatedAt: new Date(playlistObject.updated)
  38   }
  39 }
  40
  41 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
  42   return {
  43     position: elementObject.position,
  44     url: elementObject.id,
  45     startTimestamp: elementObject.startTimestamp || null,
  46     stopTimestamp: elementObject.stopTimestamp || null,
  47     videoPlaylistId: videoPlaylist.id,
  48     videoId: video.id
  49   }
  50 }
  51
  52 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  53   await Bluebird.map(playlistUrls, async playlistUrl => {
  54     try {
  55       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  56       if (exists === true) return
  57
  58       // Fetch url
  59       const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
  60
  61       if (!isPlaylistObjectValid(body)) {
  62         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  63       }
  64
  65       if (!isArray(body.to)) {
  66         throw new Error('Playlist does not have an audience.')
  67       }
  68
  69       return createOrUpdateVideoPlaylist(body, account, body.to)
  70     } catch (err) {
  71       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  72     }
  73   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  74 }
  75
  76 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  77   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  78
  79   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  80     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  81
  82     if (actor.VideoChannel) {
  83       playlistAttributes.videoChannelId = actor.VideoChannel.id
  84     } else {
  85       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  86     }
  87   }
  88
  89   const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  90
  91   let accItems: string[] = []
  92   await crawlCollectionPage<string>(playlistObject.id, items => {
  93     accItems = accItems.concat(items)
  94
  95     return Promise.resolve()
  96   })
  97
  98   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
  99
 100   if (playlistObject.icon) {
 101     try {
 102       const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
 103       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
 104     } catch (err) {
 105       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 106     }
 107   } else if (refreshedPlaylist.hasThumbnail()) {
 108     await refreshedPlaylist.Thumbnail.destroy()
 109     refreshedPlaylist.Thumbnail = null
 110   }
 111
 112   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 113 }
 114
 115 async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
 116   if (!videoPlaylist.isOutdated()) return videoPlaylist
 117
 118   try {
 119     const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 120
 121     if (playlistObject === undefined) {
 122       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 123
 124       await videoPlaylist.setAsRefreshed()
 125       return videoPlaylist
 126     }
 127
 128     const byAccount = videoPlaylist.OwnerAccount
 129     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 130
 131     return videoPlaylist
 132   } catch (err) {
 133     if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
 134       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 135
 136       await videoPlaylist.destroy()
 137       return undefined
 138     }
 139
 140     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 141
 142     await videoPlaylist.setAsRefreshed()
 143     return videoPlaylist
 144   }
 145 }
 146
 147 // ---------------------------------------------------------------------------
 148
 149 export {
 150   createAccountPlaylists,
 151   playlistObjectToDBAttributes,
 152   playlistElementObjectToDBAttributes,
 153   createOrUpdateVideoPlaylist,
 154   refreshVideoPlaylistIfNeeded
 155 }
 156
 157 // ---------------------------------------------------------------------------
 158
 159 async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 160   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 161
 162   await Bluebird.map(elementUrls, async elementUrl => {
 163     try {
 164       const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
 165
 166       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 167
 168       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 169         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 170       }
 171
 172       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 173
 174       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 175     } catch (err) {
 176       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 177     }
 178   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 179
 180   await sequelizeTypescript.transaction(async t => {
 181     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 182
 183     for (const element of elementsToCreate) {
 184       await VideoPlaylistElementModel.create(element, { transaction: t })
 185     }
 186   })
 187
 188   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 189
 190   return undefined
 191 }
 192
 193 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 194   logger.info('Fetching remote playlist %s.', playlistUrl)
 195
 196   const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
 197
 198   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 199     logger.debug('Remote video playlist JSON is not valid.', { body })
 200     return { statusCode, playlistObject: undefined }
 201   }
 202
 203   return { statusCode, playlistObject: body }
 204 }