]>
Commit | Line | Data |
---|---|---|
41fb13c3 | 1 | import { map } from 'bluebird' |
49af5ac8 | 2 | import { isArray } from '@server/helpers/custom-validators/misc' |
d9dd51ae | 3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
49af5ac8 C |
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' | |
6 | import { sequelizeTypescript } from '@server/initializers/database' | |
91f8f8db | 7 | import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' |
49af5ac8 C |
8 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' |
9 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' | |
10 | import { FilteredModelAttributes } from '@server/types' | |
37a44fc9 | 11 | import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' |
49af5ac8 | 12 | import { PlaylistObject } from '@shared/models' |
7e98a7df C |
13 | import { AttributesOnly } from '@shared/typescript-utils' |
14 | import { getAPId } from '../activity' | |
136d7efd | 15 | import { getOrCreateAPActor } from '../actors' |
49af5ac8 C |
16 | import { crawlCollectionPage } from '../crawl' |
17 | import { getOrCreateAPVideo } from '../videos' | |
18 | import { | |
19 | fetchRemotePlaylistElement, | |
20 | fetchRemoteVideoPlaylist, | |
21 | playlistElementObjectToDBAttributes, | |
22 | playlistObjectToDBAttributes | |
23 | } from './shared' | |
24 | ||
49af5ac8 C |
25 | const lTags = loggerTagsFactory('ap', 'video-playlist') |
26 | ||
37a44fc9 | 27 | async function createAccountPlaylists (playlistUrls: string[]) { |
41fb13c3 | 28 | await map(playlistUrls, async playlistUrl => { |
49af5ac8 C |
29 | try { |
30 | const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) | |
31 | if (exists === true) return | |
32 | ||
33 | const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) | |
34 | ||
35 | if (playlistObject === undefined) { | |
36 | throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) | |
37 | } | |
38 | ||
37a44fc9 | 39 | return createOrUpdateVideoPlaylist(playlistObject) |
49af5ac8 C |
40 | } catch (err) { |
41 | logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) | |
42 | } | |
43 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | |
44 | } | |
45 | ||
37a44fc9 C |
46 | async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { |
47 | const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) | |
49af5ac8 | 48 | |
37a44fc9 | 49 | await setVideoChannel(playlistObject, playlistAttributes) |
49af5ac8 | 50 | |
37a44fc9 | 51 | const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true }) |
49af5ac8 C |
52 | |
53 | const playlistElementUrls = await fetchElementUrls(playlistObject) | |
54 | ||
55 | // Refetch playlist from DB since elements fetching could be long in time | |
56 | const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) | |
57 | ||
52fb1d97 | 58 | await updatePlaylistThumbnail(playlistObject, playlist) |
49af5ac8 | 59 | |
37a44fc9 C |
60 | const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) |
61 | playlist.setVideosLength(elementsLength) | |
62 | ||
63 | return playlist | |
49af5ac8 C |
64 | } |
65 | ||
66 | // --------------------------------------------------------------------------- | |
67 | ||
68 | export { | |
69 | createAccountPlaylists, | |
70 | createOrUpdateVideoPlaylist | |
71 | } | |
72 | ||
73 | // --------------------------------------------------------------------------- | |
74 | ||
37a44fc9 C |
75 | async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) { |
76 | if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { | |
77 | throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) | |
78 | } | |
49af5ac8 | 79 | |
37a44fc9 | 80 | const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all') |
49af5ac8 C |
81 | |
82 | if (!actor.VideoChannel) { | |
83 | logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) | |
84 | return | |
85 | } | |
86 | ||
87 | playlistAttributes.videoChannelId = actor.VideoChannel.id | |
37a44fc9 | 88 | playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id |
49af5ac8 C |
89 | } |
90 | ||
91 | async function fetchElementUrls (playlistObject: PlaylistObject) { | |
92 | let accItems: string[] = [] | |
93 | await crawlCollectionPage<string>(playlistObject.id, items => { | |
94 | accItems = accItems.concat(items) | |
95 | ||
96 | return Promise.resolve() | |
97 | }) | |
98 | ||
99 | return accItems | |
100 | } | |
101 | ||
102 | async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { | |
103 | if (playlistObject.icon) { | |
52fb1d97 C |
104 | let thumbnailModel: MThumbnail |
105 | ||
106 | try { | |
107 | thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) | |
108 | await playlist.setAndSaveThumbnail(thumbnailModel, undefined) | |
109 | } catch (err) { | |
110 | logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) | |
111 | ||
112 | if (thumbnailModel) await thumbnailModel.removeThumbnail() | |
113 | } | |
49af5ac8 C |
114 | |
115 | return | |
116 | } | |
117 | ||
118 | // Playlist does not have an icon, destroy existing one | |
119 | if (playlist.hasThumbnail()) { | |
120 | await playlist.Thumbnail.destroy() | |
121 | playlist.Thumbnail = null | |
122 | } | |
123 | } | |
124 | ||
125 | async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { | |
126 | const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) | |
127 | ||
d9dd51ae | 128 | await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { |
49af5ac8 C |
129 | await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) |
130 | ||
131 | for (const element of elementsToCreate) { | |
132 | await VideoPlaylistElementModel.create(element, { transaction: t }) | |
133 | } | |
d9dd51ae | 134 | })) |
49af5ac8 C |
135 | |
136 | logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) | |
137 | ||
37a44fc9 | 138 | return elementsToCreate.length |
49af5ac8 C |
139 | } |
140 | ||
141 | async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { | |
142 | const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = [] | |
143 | ||
41fb13c3 | 144 | await map(elementUrls, async elementUrl => { |
49af5ac8 C |
145 | try { |
146 | const { elementObject } = await fetchRemotePlaylistElement(elementUrl) | |
147 | ||
148 | const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) | |
149 | ||
150 | elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) | |
151 | } catch (err) { | |
152 | logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) | |
153 | } | |
154 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | |
155 | ||
156 | return elementsToCreate | |
157 | } |