1 import { map } from 'bluebird'
2 import { isArray } from '@server/helpers/custom-validators/misc'
3 import { retryTransactionWrapper } from '@server/helpers/database-utils'
4 import { logger, loggerTagsFactory } from '@server/helpers/logger'
5 import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
6 import { sequelizeTypescript } from '@server/initializers/database'
7 import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
8 import { VideoPlaylistModel } from '@server/models/video/video-playlist'
9 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
10 import { FilteredModelAttributes } from '@server/types'
11 import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models'
12 import { PlaylistObject } from '@shared/models'
13 import { AttributesOnly } from '@shared/typescript-utils'
14 import { getAPId } from '../activity'
15 import { getOrCreateAPActor } from '../actors'
16 import { crawlCollectionPage } from '../crawl'
17 import { getOrCreateAPVideo } from '../videos'
19 fetchRemotePlaylistElement,
20 fetchRemoteVideoPlaylist,
21 playlistElementObjectToDBAttributes,
22 playlistObjectToDBAttributes
25 const lTags = loggerTagsFactory('ap', 'video-playlist')
27 async function createAccountPlaylists (playlistUrls: string[]) {
28 await map(playlistUrls, async playlistUrl => {
30 const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
31 if (exists === true) return
33 const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
35 if (playlistObject === undefined) {
36 throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
39 return createOrUpdateVideoPlaylist(playlistObject)
41 logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
43 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
46 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
47 const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
49 await setVideoChannel(playlistObject, playlistAttributes)
51 const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
53 const playlistElementUrls = await fetchElementUrls(playlistObject)
55 // Refetch playlist from DB since elements fetching could be long in time
56 const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
58 await updatePlaylistThumbnail(playlistObject, playlist)
60 const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist)
61 playlist.setVideosLength(elementsLength)
66 // ---------------------------------------------------------------------------
69 createAccountPlaylists,
70 createOrUpdateVideoPlaylist
73 // ---------------------------------------------------------------------------
75 async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
76 if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
77 throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
80 const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all')
82 if (!actor.VideoChannel) {
83 logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
87 playlistAttributes.videoChannelId = actor.VideoChannel.id
88 playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
91 async function fetchElementUrls (playlistObject: PlaylistObject) {
92 let accItems: string[] = []
93 await crawlCollectionPage<string>(playlistObject.id, items => {
94 accItems = accItems.concat(items)
96 return Promise.resolve()
102 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
103 if (playlistObject.icon) {
104 let thumbnailModel: MThumbnail
107 thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
108 await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
110 logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
112 if (thumbnailModel) await thumbnailModel.removeThumbnail()
118 // Playlist does not have an icon, destroy existing one
119 if (playlist.hasThumbnail()) {
120 await playlist.Thumbnail.destroy()
121 playlist.Thumbnail = null
125 async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
126 const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
128 await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => {
129 await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
131 for (const element of elementsToCreate) {
132 await VideoPlaylistElementModel.create(element, { transaction: t })
136 logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
138 return elementsToCreate.length
141 async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
142 const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
144 await map(elementUrls, async elementUrl => {
146 const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
148 const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
150 elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
152 logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
154 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
156 return elementsToCreate