]>
Commit | Line | Data |
---|---|---|
418d092a C |
1 | import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' |
2 | import { crawlCollectionPage } from './crawl' | |
e8bafea3 | 3 | import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' |
418d092a C |
4 | import { AccountModel } from '../../models/account/account' |
5 | import { isArray } from '../../helpers/custom-validators/misc' | |
6 | import { getOrCreateActorAndServerAndModel } from './actor' | |
7 | import { logger } from '../../helpers/logger' | |
8 | import { VideoPlaylistModel } from '../../models/video/video-playlist' | |
e8bafea3 | 9 | import { doRequest } from '../../helpers/requests' |
418d092a C |
10 | import { checkUrlsSameHost } from '../../helpers/activitypub' |
11 | import * as Bluebird from 'bluebird' | |
12 | import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' | |
13 | import { getOrCreateVideoAndAccountAndChannel } from './videos' | |
14 | import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' | |
15 | import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' | |
16 | import { VideoModel } from '../../models/video/video' | |
418d092a | 17 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' |
74dc3bca | 18 | import { sequelizeTypescript } from '../../initializers/database' |
3acc5084 C |
19 | import { createPlaylistMiniatureFromUrl } from '../thumbnail' |
20 | import { FilteredModelAttributes } from '../../typings/sequelize' | |
5224c394 | 21 | import { AccountModelId } from '../../typings/models' |
418d092a | 22 | |
5224c394 | 23 | function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModelId, to: string[]) { |
418d092a C |
24 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED |
25 | ||
26 | return { | |
27 | name: playlistObject.name, | |
28 | description: playlistObject.content, | |
29 | privacy, | |
30 | url: playlistObject.id, | |
31 | uuid: playlistObject.uuid, | |
32 | ownerAccountId: byAccount.id, | |
df0b219d C |
33 | videoChannelId: null, |
34 | createdAt: new Date(playlistObject.published), | |
35 | updatedAt: new Date(playlistObject.updated) | |
418d092a C |
36 | } |
37 | } | |
38 | ||
39 | function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: VideoPlaylistModel, video: VideoModel) { | |
40 | return { | |
41 | position: elementObject.position, | |
42 | url: elementObject.id, | |
43 | startTimestamp: elementObject.startTimestamp || null, | |
44 | stopTimestamp: elementObject.stopTimestamp || null, | |
45 | videoPlaylistId: videoPlaylist.id, | |
46 | videoId: video.id | |
47 | } | |
48 | } | |
49 | ||
50 | async function createAccountPlaylists (playlistUrls: string[], account: AccountModel) { | |
51 | await Bluebird.map(playlistUrls, async playlistUrl => { | |
52 | try { | |
53 | const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) | |
54 | if (exists === true) return | |
55 | ||
56 | // Fetch url | |
57 | const { body } = await doRequest<PlaylistObject>({ | |
58 | uri: playlistUrl, | |
59 | json: true, | |
60 | activityPub: true | |
61 | }) | |
62 | ||
63 | if (!isPlaylistObjectValid(body)) { | |
64 | throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) | |
65 | } | |
66 | ||
67 | if (!isArray(body.to)) { | |
68 | throw new Error('Playlist does not have an audience.') | |
69 | } | |
70 | ||
71 | return createOrUpdateVideoPlaylist(body, account, body.to) | |
72 | } catch (err) { | |
73 | logger.warn('Cannot add playlist element %s.', playlistUrl, { err }) | |
74 | } | |
75 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | |
76 | } | |
77 | ||
5224c394 | 78 | async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: AccountModelId, to: string[]) { |
418d092a C |
79 | const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to) |
80 | ||
81 | if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) { | |
82 | const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0]) | |
83 | ||
84 | if (actor.VideoChannel) { | |
85 | playlistAttributes.videoChannelId = actor.VideoChannel.id | |
86 | } else { | |
87 | logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject }) | |
88 | } | |
89 | } | |
90 | ||
3acc5084 | 91 | const [ playlist ] = await VideoPlaylistModel.upsert<VideoPlaylistModel>(playlistAttributes, { returning: true }) |
418d092a C |
92 | |
93 | let accItems: string[] = [] | |
94 | await crawlCollectionPage<string>(playlistObject.id, items => { | |
95 | accItems = accItems.concat(items) | |
96 | ||
97 | return Promise.resolve() | |
98 | }) | |
99 | ||
e8bafea3 C |
100 | const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null) |
101 | ||
102 | if (playlistObject.icon) { | |
418d092a | 103 | try { |
3acc5084 C |
104 | const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist) |
105 | await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined) | |
418d092a C |
106 | } catch (err) { |
107 | logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err }) | |
108 | } | |
65af03a2 C |
109 | } else if (refreshedPlaylist.hasThumbnail()) { |
110 | await refreshedPlaylist.Thumbnail.destroy() | |
111 | refreshedPlaylist.Thumbnail = null | |
418d092a C |
112 | } |
113 | ||
e8bafea3 | 114 | return resetVideoPlaylistElements(accItems, refreshedPlaylist) |
418d092a C |
115 | } |
116 | ||
9f79ade6 C |
117 | async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> { |
118 | if (!videoPlaylist.isOutdated()) return videoPlaylist | |
119 | ||
120 | try { | |
121 | const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) | |
122 | if (statusCode === 404) { | |
123 | logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url) | |
124 | ||
125 | await videoPlaylist.destroy() | |
126 | return undefined | |
127 | } | |
128 | ||
129 | if (playlistObject === undefined) { | |
130 | logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) | |
131 | ||
132 | await videoPlaylist.setAsRefreshed() | |
133 | return videoPlaylist | |
134 | } | |
135 | ||
136 | const byAccount = videoPlaylist.OwnerAccount | |
137 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) | |
138 | ||
139 | return videoPlaylist | |
140 | } catch (err) { | |
141 | logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) | |
142 | ||
143 | await videoPlaylist.setAsRefreshed() | |
144 | return videoPlaylist | |
145 | } | |
146 | } | |
147 | ||
418d092a C |
148 | // --------------------------------------------------------------------------- |
149 | ||
150 | export { | |
151 | createAccountPlaylists, | |
152 | playlistObjectToDBAttributes, | |
153 | playlistElementObjectToDBAttributes, | |
9f79ade6 C |
154 | createOrUpdateVideoPlaylist, |
155 | refreshVideoPlaylistIfNeeded | |
418d092a C |
156 | } |
157 | ||
158 | // --------------------------------------------------------------------------- | |
159 | ||
160 | async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) { | |
3acc5084 | 161 | const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = [] |
418d092a C |
162 | |
163 | await Bluebird.map(elementUrls, async elementUrl => { | |
164 | try { | |
165 | // Fetch url | |
166 | const { body } = await doRequest<PlaylistElementObject>({ | |
167 | uri: elementUrl, | |
168 | json: true, | |
169 | activityPub: true | |
170 | }) | |
171 | ||
172 | if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) | |
173 | ||
174 | if (checkUrlsSameHost(body.id, elementUrl) !== true) { | |
175 | throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) | |
176 | } | |
177 | ||
178 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' }) | |
179 | ||
180 | elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) | |
181 | } catch (err) { | |
182 | logger.warn('Cannot add playlist element %s.', elementUrl, { err }) | |
183 | } | |
184 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | |
185 | ||
186 | await sequelizeTypescript.transaction(async t => { | |
187 | await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) | |
188 | ||
189 | for (const element of elementsToCreate) { | |
190 | await VideoPlaylistElementModel.create(element, { transaction: t }) | |
191 | } | |
192 | }) | |
193 | ||
194 | logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length) | |
195 | ||
196 | return undefined | |
197 | } | |
198 | ||
9f79ade6 C |
199 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { |
200 | const options = { | |
201 | uri: playlistUrl, | |
202 | method: 'GET', | |
203 | json: true, | |
204 | activityPub: true | |
205 | } | |
206 | ||
207 | logger.info('Fetching remote playlist %s.', playlistUrl) | |
208 | ||
209 | const { response, body } = await doRequest(options) | |
210 | ||
211 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | |
212 | logger.debug('Remote video playlist JSON is not valid.', { body }) | |
213 | return { statusCode: response.statusCode, playlistObject: undefined } | |
214 | } | |
215 | ||
216 | return { statusCode: response.statusCode, playlistObject: body } | |
217 | } |