1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
import { isArray } from '@server/helpers/custom-validators/misc'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
import { sequelizeTypescript } from '@server/initializers/database'
import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
import { VideoPlaylistModel } from '@server/models/video/video-playlist'
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
import { FilteredModelAttributes } from '@server/types'
import { MAccountDefault, MAccountId, MThumbnail, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
import { AttributesOnly } from '@shared/core-utils'
import { PlaylistObject } from '@shared/models'
import { getOrCreateAPActor } from '../actors'
import { crawlCollectionPage } from '../crawl'
import { getOrCreateAPVideo } from '../videos'
import {
fetchRemotePlaylistElement,
fetchRemoteVideoPlaylist,
playlistElementObjectToDBAttributes,
playlistObjectToDBAttributes
} from './shared'
import Bluebird = require('bluebird')
const lTags = loggerTagsFactory('ap', 'video-playlist')
async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
await Bluebird.map(playlistUrls, async playlistUrl => {
try {
const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
if (exists === true) return
const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
if (playlistObject === undefined) {
throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
}
return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
} catch (err) {
logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
}
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
}
async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
const playlistElementUrls = await fetchElementUrls(playlistObject)
// Refetch playlist from DB since elements fetching could be long in time
const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
await updatePlaylistThumbnail(playlistObject, playlist)
return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
}
// ---------------------------------------------------------------------------
export {
createAccountPlaylists,
createOrUpdateVideoPlaylist
}
// ---------------------------------------------------------------------------
async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
const actor = await getOrCreateAPActor(playlistObject.attributedTo[0])
if (!actor.VideoChannel) {
logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
return
}
playlistAttributes.videoChannelId = actor.VideoChannel.id
}
async function fetchElementUrls (playlistObject: PlaylistObject) {
let accItems: string[] = []
await crawlCollectionPage<string>(playlistObject.id, items => {
accItems = accItems.concat(items)
return Promise.resolve()
})
return accItems
}
async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
if (playlistObject.icon) {
let thumbnailModel: MThumbnail
try {
thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
} catch (err) {
logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
if (thumbnailModel) await thumbnailModel.removeThumbnail()
}
return
}
// Playlist does not have an icon, destroy existing one
if (playlist.hasThumbnail()) {
await playlist.Thumbnail.destroy()
playlist.Thumbnail = null
}
}
async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
await sequelizeTypescript.transaction(async t => {
await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
for (const element of elementsToCreate) {
await VideoPlaylistElementModel.create(element, { transaction: t })
}
})
logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
return undefined
}
async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
await Bluebird.map(elementUrls, async elementUrl => {
try {
const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
} catch (err) {
logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
}
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
return elementsToCreate
}
|
