server/lib/activitypub/playlist.ts

   1 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   2 import { crawlCollectionPage } from './crawl'
   3 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers'
   4 import { AccountModel } from '../../models/account/account'
   5 import { isArray } from '../../helpers/custom-validators/misc'
   6 import { getOrCreateActorAndServerAndModel } from './actor'
   7 import { logger } from '../../helpers/logger'
   8 import { VideoPlaylistModel } from '../../models/video/video-playlist'
   9 import { doRequest, downloadImage } from '../../helpers/requests'
  10 import { checkUrlsSameHost } from '../../helpers/activitypub'
  11 import * as Bluebird from 'bluebird'
  12 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
  13 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  14 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
  15 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  16 import { VideoModel } from '../../models/video/video'
  17 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
  18 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
  19 import { ActivityIconObject } from '../../../shared/models/activitypub/objects'
  20 import { CONFIG } from '../../initializers/config'
  21
  22 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
  23   const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
  24
  25   return {
  26     name: playlistObject.name,
  27     description: playlistObject.content,
  28     privacy,
  29     url: playlistObject.id,
  30     uuid: playlistObject.uuid,
  31     ownerAccountId: byAccount.id,
  32     videoChannelId: null,
  33     createdAt: new Date(playlistObject.published),
  34     updatedAt: new Date(playlistObject.updated)
  35   }
  36 }
  37
  38 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) {
  39   return {
  40     position: elementObject.position,
  41     url: elementObject.id,
  42     startTimestamp: elementObject.startTimestamp || null,
  43     stopTimestamp: elementObject.stopTimestamp || null,
  44     videoPlaylistId: videoPlaylist.id,
  45     videoId: video.id
  46   }
  47 }
  48
  49 async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) {
  50   await Bluebird.map(playlistUrls, async playlistUrl => {
  51     try {
  52       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  53       if (exists === true) return
  54
  55       // Fetch url
  56       const { body } = await doRequest<PlaylistObject>({
  57         uri: playlistUrl,
  58         json: true,
  59         activityPub: true
  60       })
  61
  62       if (!isPlaylistObjectValid(body)) {
  63         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  64       }
  65
  66       if (!isArray(body.to)) {
  67         throw new Error('Playlist does not have an audience.')
  68       }
  69
  70       return createOrUpdateVideoPlaylist(body, account, body.to)
  71     } catch (err) {
  72       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  73     }
  74   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  75 }
  76
  77 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
  78   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  79
  80   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  81     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  82
  83     if (actor.VideoChannel) {
  84       playlistAttributes.videoChannelId = actor.VideoChannel.id
  85     } else {
  86       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  87     }
  88   }
  89
  90   const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true })
  91
  92   let accItems: string[] = []
  93   await crawlCollectionPage<string>(playlistObject.id, items => {
  94     accItems = accItems.concat(items)
  95
  96     return Promise.resolve()
  97   })
  98
  99   // Empty playlists generally do not have a miniature, so skip this
 100   if (accItems.length !== 0) {
 101     try {
 102       await generateThumbnailFromUrl(playlist, playlistObject.icon)
 103     } catch (err) {
 104       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 105     }
 106   }
 107
 108   return resetVideoPlaylistElements(accItems, playlist)
 109 }
 110
 111 async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
 112   if (!videoPlaylist.isOutdated()) return videoPlaylist
 113
 114   try {
 115     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 116     if (statusCode === 404) {
 117       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 118
 119       await videoPlaylist.destroy()
 120       return undefined
 121     }
 122
 123     if (playlistObject === undefined) {
 124       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 125
 126       await videoPlaylist.setAsRefreshed()
 127       return videoPlaylist
 128     }
 129
 130     const byAccount = videoPlaylist.OwnerAccount
 131     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 132
 133     return videoPlaylist
 134   } catch (err) {
 135     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 136
 137     await videoPlaylist.setAsRefreshed()
 138     return videoPlaylist
 139   }
 140 }
 141
 142 // ---------------------------------------------------------------------------
 143
 144 export {
 145   createAccountPlaylists,
 146   playlistObjectToDBAttributes,
 147   playlistElementObjectToDBAttributes,
 148   createOrUpdateVideoPlaylist,
 149   refreshVideoPlaylistIfNeeded
 150 }
 151
 152 // ---------------------------------------------------------------------------
 153
 154 async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) {
 155   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 156
 157   await Bluebird.map(elementUrls, async elementUrl => {
 158     try {
 159       // Fetch url
 160       const { body } = await doRequest<PlaylistElementObject>({
 161         uri: elementUrl,
 162         json: true,
 163         activityPub: true
 164       })
 165
 166       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 167
 168       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 169         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 170       }
 171
 172       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 173
 174       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 175     } catch (err) {
 176       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 177     }
 178   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 179
 180   await sequelizeTypescript.transaction(async t => {
 181     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 182
 183     for (const element of elementsToCreate) {
 184       await VideoPlaylistElementModel.create(element, { transaction: t })
 185     }
 186   })
 187
 188   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 189
 190   return undefined
 191 }
 192
 193 function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityIconObject) {
 194   const thumbnailName = playlist.getThumbnailName()
 195
 196   return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
 197 }
 198
 199 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 200   const options = {
 201     uri: playlistUrl,
 202     method: 'GET',
 203     json: true,
 204     activityPub: true
 205   }
 206
 207   logger.info('Fetching remote playlist %s.', playlistUrl)
 208
 209   const { response, body } = await doRequest(options)
 210
 211   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 212     logger.debug('Remote video playlist JSON is not valid.', { body })
 213     return { statusCode: response.statusCode, playlistObject: undefined }
 214   }
 215
 216   return { statusCode: response.statusCode, playlistObject: body }
 217 }