server/lib/activitypub/playlist.ts

   1 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   2 import { crawlCollectionPage } from './crawl'
   3 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
   4 import { isArray } from '../../helpers/custom-validators/misc'
   5 import { getOrCreateActorAndServerAndModel } from './actor'
   6 import { logger } from '../../helpers/logger'
   7 import { VideoPlaylistModel } from '../../models/video/video-playlist'
   8 import { doRequest } from '../../helpers/requests'
   9 import { checkUrlsSameHost } from '../../helpers/activitypub'
  10 import * as Bluebird from 'bluebird'
  11 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
  12 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  13 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
  14 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  15 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
  16 import { sequelizeTypescript } from '../../initializers/database'
  17 import { createPlaylistMiniatureFromUrl } from '../thumbnail'
  18 import { FilteredModelAttributes } from '../../types/sequelize'
  19 import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
  20 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
  21 import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
  22
  23 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  24   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
  25     ? VideoPlaylistPrivacy.PUBLIC
  26     : VideoPlaylistPrivacy.UNLISTED
  27
  28   return {
  29     name: playlistObject.name,
  30     description: playlistObject.content,
  31     privacy,
  32     url: playlistObject.id,
  33     uuid: playlistObject.uuid,
  34     ownerAccountId: byAccount.id,
  35     videoChannelId: null,
  36     createdAt: new Date(playlistObject.published),
  37     updatedAt: new Date(playlistObject.updated)
  38   }
  39 }
  40
  41 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
  42   return {
  43     position: elementObject.position,
  44     url: elementObject.id,
  45     startTimestamp: elementObject.startTimestamp || null,
  46     stopTimestamp: elementObject.stopTimestamp || null,
  47     videoPlaylistId: videoPlaylist.id,
  48     videoId: video.id
  49   }
  50 }
  51
  52 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  53   await Bluebird.map(playlistUrls, async playlistUrl => {
  54     try {
  55       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  56       if (exists === true) return
  57
  58       // Fetch url
  59       const { body } = await doRequest<PlaylistObject>({
  60         uri: playlistUrl,
  61         json: true,
  62         activityPub: true
  63       })
  64
  65       if (!isPlaylistObjectValid(body)) {
  66         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  67       }
  68
  69       if (!isArray(body.to)) {
  70         throw new Error('Playlist does not have an audience.')
  71       }
  72
  73       return createOrUpdateVideoPlaylist(body, account, body.to)
  74     } catch (err) {
  75       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  76     }
  77   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  78 }
  79
  80 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  81   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  82
  83   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  84     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  85
  86     if (actor.VideoChannel) {
  87       playlistAttributes.videoChannelId = actor.VideoChannel.id
  88     } else {
  89       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  90     }
  91   }
  92
  93   const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  94
  95   let accItems: string[] = []
  96   await crawlCollectionPage<string>(playlistObject.id, items => {
  97     accItems = accItems.concat(items)
  98
  99     return Promise.resolve()
 100   })
 101
 102   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
 103
 104   if (playlistObject.icon) {
 105     try {
 106       const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
 107       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
 108     } catch (err) {
 109       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 110     }
 111   } else if (refreshedPlaylist.hasThumbnail()) {
 112     await refreshedPlaylist.Thumbnail.destroy()
 113     refreshedPlaylist.Thumbnail = null
 114   }
 115
 116   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 117 }
 118
 119 async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
 120   if (!videoPlaylist.isOutdated()) return videoPlaylist
 121
 122   try {
 123     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 124     if (statusCode === HttpStatusCode.NOT_FOUND_404) {
 125       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 126
 127       await videoPlaylist.destroy()
 128       return undefined
 129     }
 130
 131     if (playlistObject === undefined) {
 132       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 133
 134       await videoPlaylist.setAsRefreshed()
 135       return videoPlaylist
 136     }
 137
 138     const byAccount = videoPlaylist.OwnerAccount
 139     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 140
 141     return videoPlaylist
 142   } catch (err) {
 143     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 144
 145     await videoPlaylist.setAsRefreshed()
 146     return videoPlaylist
 147   }
 148 }
 149
 150 // ---------------------------------------------------------------------------
 151
 152 export {
 153   createAccountPlaylists,
 154   playlistObjectToDBAttributes,
 155   playlistElementObjectToDBAttributes,
 156   createOrUpdateVideoPlaylist,
 157   refreshVideoPlaylistIfNeeded
 158 }
 159
 160 // ---------------------------------------------------------------------------
 161
 162 async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 163   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 164
 165   await Bluebird.map(elementUrls, async elementUrl => {
 166     try {
 167       // Fetch url
 168       const { body } = await doRequest<PlaylistElementObject>({
 169         uri: elementUrl,
 170         json: true,
 171         activityPub: true
 172       })
 173
 174       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 175
 176       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 177         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 178       }
 179
 180       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 181
 182       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 183     } catch (err) {
 184       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 185     }
 186   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 187
 188   await sequelizeTypescript.transaction(async t => {
 189     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 190
 191     for (const element of elementsToCreate) {
 192       await VideoPlaylistElementModel.create(element, { transaction: t })
 193     }
 194   })
 195
 196   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 197
 198   return undefined
 199 }
 200
 201 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 202   const options = {
 203     uri: playlistUrl,
 204     method: 'GET',
 205     json: true,
 206     activityPub: true
 207   }
 208
 209   logger.info('Fetching remote playlist %s.', playlistUrl)
 210
 211   const { response, body } = await doRequest<any>(options)
 212
 213   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 214     logger.debug('Remote video playlist JSON is not valid.', { body })
 215     return { statusCode: response.statusCode, playlistObject: undefined }
 216   }
 217
 218   return { statusCode: response.statusCode, playlistObject: body }
 219 }