diff options
author | Chocobozzz <me@florianbigard.com> | 2018-09-20 16:24:31 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-09-20 16:24:31 +0200 |
commit | 0491173a61aed66205c017e0d7e0503ea316c144 (patch) | |
tree | ce6621597505f9518cfdf0981977d097c63f9fad /server/lib/activitypub/videos.ts | |
parent | 8704acf49efc770d73bf07c10468ed8c74d28a83 (diff) | |
parent | 6247b2057b792cea155a1abd9788c363ae7d2cc2 (diff) | |
download | PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.tar.gz PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.tar.zst PeerTube-0491173a61aed66205c017e0d7e0503ea316c144.zip |
Merge branch 'develop' into cli-wrapper
Diffstat (limited to 'server/lib/activitypub/videos.ts')
-rw-r--r-- | server/lib/activitypub/videos.ts | 531 |
1 files changed, 280 insertions, 251 deletions
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 783f78d3e..48c0e0a5c 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize' | |||
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import { join } from 'path' | 4 | import { join } from 'path' |
5 | import * as request from 'request' | 5 | import * as request from 'request' |
6 | import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' | 6 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' |
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy } from '../../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub | |||
28 | import { createRates } from './video-rates' | 28 | import { createRates } from './video-rates' |
29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | 29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' |
30 | import { AccountModel } from '../../models/account/account' | 30 | import { AccountModel } from '../../models/account/account' |
31 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | ||
31 | 32 | ||
32 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 33 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
33 | // If the video is not private and published, we federate it | 34 | // If the video is not private and published, we federate it |
@@ -50,18 +51,29 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr | |||
50 | } | 51 | } |
51 | } | 52 | } |
52 | 53 | ||
53 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | 54 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { |
54 | const host = video.VideoChannel.Account.Actor.Server.host | 55 | const options = { |
56 | uri: videoUrl, | ||
57 | method: 'GET', | ||
58 | json: true, | ||
59 | activityPub: true | ||
60 | } | ||
55 | 61 | ||
56 | // We need to provide a callback, if no we could have an uncaught exception | 62 | logger.info('Fetching remote video %s.', videoUrl) |
57 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | 63 | |
58 | if (err) reject(err) | 64 | const { response, body } = await doRequest(options) |
59 | }) | 65 | |
66 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | ||
67 | logger.debug('Remote video JSON is not valid.', { body }) | ||
68 | return { response, videoObject: undefined } | ||
69 | } | ||
70 | |||
71 | return { response, videoObject: body } | ||
60 | } | 72 | } |
61 | 73 | ||
62 | async function fetchRemoteVideoDescription (video: VideoModel) { | 74 | async function fetchRemoteVideoDescription (video: VideoModel) { |
63 | const host = video.VideoChannel.Account.Actor.Server.host | 75 | const host = video.VideoChannel.Account.Actor.Server.host |
64 | const path = video.getDescriptionPath() | 76 | const path = video.getDescriptionAPIPath() |
65 | const options = { | 77 | const options = { |
66 | uri: REMOTE_SCHEME.HTTP + '://' + host + path, | 78 | uri: REMOTE_SCHEME.HTTP + '://' + host + path, |
67 | json: true | 79 | json: true |
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) { | |||
71 | return body.description ? body.description : '' | 83 | return body.description ? body.description : '' |
72 | } | 84 | } |
73 | 85 | ||
86 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | ||
87 | const host = video.VideoChannel.Account.Actor.Server.host | ||
88 | |||
89 | // We need to provide a callback, if no we could have an uncaught exception | ||
90 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | ||
91 | if (err) reject(err) | ||
92 | }) | ||
93 | } | ||
94 | |||
74 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { | 95 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { |
75 | const thumbnailName = video.getThumbnailName() | 96 | const thumbnailName = video.getThumbnailName() |
76 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) | 97 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) |
@@ -82,144 +103,11 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) | |||
82 | return doRequestAndSaveToFile(options, thumbnailPath) | 103 | return doRequestAndSaveToFile(options, thumbnailPath) |
83 | } | 104 | } |
84 | 105 | ||
85 | async function videoActivityObjectToDBAttributes ( | ||
86 | videoChannel: VideoChannelModel, | ||
87 | videoObject: VideoTorrentObject, | ||
88 | to: string[] = [] | ||
89 | ) { | ||
90 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED | ||
91 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
92 | |||
93 | let language: string | undefined | ||
94 | if (videoObject.language) { | ||
95 | language = videoObject.language.identifier | ||
96 | } | ||
97 | |||
98 | let category: number | undefined | ||
99 | if (videoObject.category) { | ||
100 | category = parseInt(videoObject.category.identifier, 10) | ||
101 | } | ||
102 | |||
103 | let licence: number | undefined | ||
104 | if (videoObject.licence) { | ||
105 | licence = parseInt(videoObject.licence.identifier, 10) | ||
106 | } | ||
107 | |||
108 | const description = videoObject.content || null | ||
109 | const support = videoObject.support || null | ||
110 | |||
111 | return { | ||
112 | name: videoObject.name, | ||
113 | uuid: videoObject.uuid, | ||
114 | url: videoObject.id, | ||
115 | category, | ||
116 | licence, | ||
117 | language, | ||
118 | description, | ||
119 | support, | ||
120 | nsfw: videoObject.sensitive, | ||
121 | commentsEnabled: videoObject.commentsEnabled, | ||
122 | waitTranscoding: videoObject.waitTranscoding, | ||
123 | state: videoObject.state, | ||
124 | channelId: videoChannel.id, | ||
125 | duration: parseInt(duration, 10), | ||
126 | createdAt: new Date(videoObject.published), | ||
127 | publishedAt: new Date(videoObject.published), | ||
128 | // FIXME: updatedAt does not seems to be considered by Sequelize | ||
129 | updatedAt: new Date(videoObject.updated), | ||
130 | views: videoObject.views, | ||
131 | likes: 0, | ||
132 | dislikes: 0, | ||
133 | remote: true, | ||
134 | privacy | ||
135 | } | ||
136 | } | ||
137 | |||
138 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | ||
139 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
140 | |||
141 | if (fileUrls.length === 0) { | ||
142 | throw new Error('Cannot find video files for ' + videoCreated.url) | ||
143 | } | ||
144 | |||
145 | const attributes: VideoFileModel[] = [] | ||
146 | for (const fileUrl of fileUrls) { | ||
147 | // Fetch associated magnet uri | ||
148 | const magnet = videoObject.url.find(u => { | ||
149 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | ||
150 | }) | ||
151 | |||
152 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
153 | |||
154 | const parsed = magnetUtil.decode(magnet.href) | ||
155 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
156 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
157 | } | ||
158 | |||
159 | const attribute = { | ||
160 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | ||
161 | infoHash: parsed.infoHash, | ||
162 | resolution: fileUrl.height, | ||
163 | size: fileUrl.size, | ||
164 | videoId: videoCreated.id, | ||
165 | fps: fileUrl.fps | ||
166 | } as VideoFileModel | ||
167 | attributes.push(attribute) | ||
168 | } | ||
169 | |||
170 | return attributes | ||
171 | } | ||
172 | |||
173 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { | 106 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { |
174 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | 107 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') |
175 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | 108 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) |
176 | 109 | ||
177 | return getOrCreateActorAndServerAndModel(channel.id) | 110 | return getOrCreateActorAndServerAndModel(channel.id, 'all') |
178 | } | ||
179 | |||
180 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | ||
181 | logger.debug('Adding remote video %s.', videoObject.id) | ||
182 | |||
183 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
184 | const sequelizeOptions = { transaction: t } | ||
185 | |||
186 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
187 | const video = VideoModel.build(videoData) | ||
188 | |||
189 | const videoCreated = await video.save(sequelizeOptions) | ||
190 | |||
191 | // Process files | ||
192 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | ||
193 | if (videoFileAttributes.length === 0) { | ||
194 | throw new Error('Cannot find valid files for video %s ' + videoObject.url) | ||
195 | } | ||
196 | |||
197 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
198 | await Promise.all(videoFilePromises) | ||
199 | |||
200 | // Process tags | ||
201 | const tags = videoObject.tag.map(t => t.name) | ||
202 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
203 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
204 | |||
205 | // Process captions | ||
206 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
207 | return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) | ||
208 | }) | ||
209 | await Promise.all(videoCaptionsPromises) | ||
210 | |||
211 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
212 | |||
213 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
214 | return videoCreated | ||
215 | }) | ||
216 | |||
217 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | ||
218 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
219 | |||
220 | if (waitThumbnail === true) await p | ||
221 | |||
222 | return videoCreated | ||
223 | } | 111 | } |
224 | 112 | ||
225 | type SyncParam = { | 113 | type SyncParam = { |
@@ -230,28 +118,7 @@ type SyncParam = { | |||
230 | thumbnail: boolean | 118 | thumbnail: boolean |
231 | refreshVideo: boolean | 119 | refreshVideo: boolean |
232 | } | 120 | } |
233 | async function getOrCreateVideoAndAccountAndChannel ( | 121 | async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { |
234 | videoObject: VideoTorrentObject | string, | ||
235 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
236 | ) { | ||
237 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id | ||
238 | |||
239 | let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) | ||
240 | if (videoFromDatabase) { | ||
241 | const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) | ||
242 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
243 | |||
244 | return { video: videoFromDatabase } | ||
245 | } | ||
246 | |||
247 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | ||
248 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
249 | |||
250 | const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) | ||
251 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) | ||
252 | |||
253 | // Process outside the transaction because we could fetch remote data | ||
254 | |||
255 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | 122 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) |
256 | 123 | ||
257 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] | 124 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] |
@@ -285,64 +152,56 @@ async function getOrCreateVideoAndAccountAndChannel ( | |||
285 | } | 152 | } |
286 | 153 | ||
287 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) | 154 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) |
288 | |||
289 | return { video } | ||
290 | } | 155 | } |
291 | 156 | ||
292 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { | 157 | async function getOrCreateVideoAndAccountAndChannel (options: { |
293 | const options = { | 158 | videoObject: VideoTorrentObject | string, |
294 | uri: videoUrl, | 159 | syncParam?: SyncParam, |
295 | method: 'GET', | 160 | fetchType?: VideoFetchByUrlType, |
296 | json: true, | 161 | refreshViews?: boolean |
297 | activityPub: true | 162 | }) { |
298 | } | 163 | // Default params |
299 | 164 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | |
300 | logger.info('Fetching remote video %s.', videoUrl) | 165 | const fetchType = options.fetchType || 'all' |
301 | 166 | const refreshViews = options.refreshViews || false | |
302 | const { response, body } = await doRequest(options) | 167 | |
168 | // Get video url | ||
169 | const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id | ||
170 | |||
171 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) | ||
172 | if (videoFromDatabase) { | ||
173 | const refreshOptions = { | ||
174 | video: videoFromDatabase, | ||
175 | fetchedType: fetchType, | ||
176 | syncParam, | ||
177 | refreshViews | ||
178 | } | ||
179 | const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) | ||
180 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
303 | 181 | ||
304 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | 182 | return { video: videoFromDatabase } |
305 | logger.debug('Remote video JSON is not valid.', { body }) | ||
306 | return { response, videoObject: undefined } | ||
307 | } | 183 | } |
308 | 184 | ||
309 | return { response, videoObject: body } | 185 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) |
310 | } | 186 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) |
311 | |||
312 | async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { | ||
313 | if (!video.isOutdated()) return video | ||
314 | |||
315 | try { | ||
316 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
317 | if (response.statusCode === 404) { | ||
318 | // Video does not exist anymore | ||
319 | await video.destroy() | ||
320 | return undefined | ||
321 | } | ||
322 | 187 | ||
323 | if (videoObject === undefined) { | 188 | const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) |
324 | logger.warn('Cannot refresh remote video: invalid body.') | 189 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) |
325 | return video | ||
326 | } | ||
327 | 190 | ||
328 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 191 | await syncVideoExternalAttributes(video, fetchedVideo, syncParam) |
329 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
330 | 192 | ||
331 | return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) | 193 | return { video } |
332 | } catch (err) { | ||
333 | logger.warn('Cannot refresh video.', { err }) | ||
334 | return video | ||
335 | } | ||
336 | } | 194 | } |
337 | 195 | ||
338 | async function updateVideoFromAP ( | 196 | async function updateVideoFromAP (options: { |
339 | video: VideoModel, | 197 | video: VideoModel, |
340 | videoObject: VideoTorrentObject, | 198 | videoObject: VideoTorrentObject, |
341 | account: AccountModel, | 199 | account: AccountModel, |
342 | channel: VideoChannelModel, | 200 | channel: VideoChannelModel, |
201 | updateViews: boolean, | ||
343 | overrideTo?: string[] | 202 | overrideTo?: string[] |
344 | ) { | 203 | }) { |
345 | logger.debug('Updating remote video "%s".', videoObject.uuid) | 204 | logger.debug('Updating remote video "%s".', options.videoObject.uuid) |
346 | let videoFieldsSave: any | 205 | let videoFieldsSave: any |
347 | 206 | ||
348 | try { | 207 | try { |
@@ -351,72 +210,72 @@ async function updateVideoFromAP ( | |||
351 | transaction: t | 210 | transaction: t |
352 | } | 211 | } |
353 | 212 | ||
354 | videoFieldsSave = video.toJSON() | 213 | videoFieldsSave = options.video.toJSON() |
355 | 214 | ||
356 | // Check actor has the right to update the video | 215 | // Check actor has the right to update the video |
357 | const videoChannel = video.VideoChannel | 216 | const videoChannel = options.video.VideoChannel |
358 | if (videoChannel.Account.id !== account.id) { | 217 | if (videoChannel.Account.id !== options.account.id) { |
359 | throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) | 218 | throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) |
360 | } | 219 | } |
361 | 220 | ||
362 | const to = overrideTo ? overrideTo : videoObject.to | 221 | const to = options.overrideTo ? options.overrideTo : options.videoObject.to |
363 | const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) | 222 | const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to) |
364 | video.set('name', videoData.name) | 223 | options.video.set('name', videoData.name) |
365 | video.set('uuid', videoData.uuid) | 224 | options.video.set('uuid', videoData.uuid) |
366 | video.set('url', videoData.url) | 225 | options.video.set('url', videoData.url) |
367 | video.set('category', videoData.category) | 226 | options.video.set('category', videoData.category) |
368 | video.set('licence', videoData.licence) | 227 | options.video.set('licence', videoData.licence) |
369 | video.set('language', videoData.language) | 228 | options.video.set('language', videoData.language) |
370 | video.set('description', videoData.description) | 229 | options.video.set('description', videoData.description) |
371 | video.set('support', videoData.support) | 230 | options.video.set('support', videoData.support) |
372 | video.set('nsfw', videoData.nsfw) | 231 | options.video.set('nsfw', videoData.nsfw) |
373 | video.set('commentsEnabled', videoData.commentsEnabled) | 232 | options.video.set('commentsEnabled', videoData.commentsEnabled) |
374 | video.set('waitTranscoding', videoData.waitTranscoding) | 233 | options.video.set('waitTranscoding', videoData.waitTranscoding) |
375 | video.set('state', videoData.state) | 234 | options.video.set('state', videoData.state) |
376 | video.set('duration', videoData.duration) | 235 | options.video.set('duration', videoData.duration) |
377 | video.set('createdAt', videoData.createdAt) | 236 | options.video.set('createdAt', videoData.createdAt) |
378 | video.set('publishedAt', videoData.publishedAt) | 237 | options.video.set('publishedAt', videoData.publishedAt) |
379 | video.set('views', videoData.views) | 238 | options.video.set('privacy', videoData.privacy) |
380 | video.set('privacy', videoData.privacy) | 239 | options.video.set('channelId', videoData.channelId) |
381 | video.set('channelId', videoData.channelId) | 240 | |
382 | 241 | if (options.updateViews === true) options.video.set('views', videoData.views) | |
383 | await video.save(sequelizeOptions) | 242 | await options.video.save(sequelizeOptions) |
384 | 243 | ||
385 | // Don't block on request | 244 | // Don't block on request |
386 | generateThumbnailFromUrl(video, videoObject.icon) | 245 | generateThumbnailFromUrl(options.video, options.videoObject.icon) |
387 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | 246 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) |
388 | 247 | ||
389 | // Remove old video files | 248 | // Remove old video files |
390 | const videoFileDestroyTasks: Bluebird<void>[] = [] | 249 | const videoFileDestroyTasks: Bluebird<void>[] = [] |
391 | for (const videoFile of video.VideoFiles) { | 250 | for (const videoFile of options.video.VideoFiles) { |
392 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) | 251 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) |
393 | } | 252 | } |
394 | await Promise.all(videoFileDestroyTasks) | 253 | await Promise.all(videoFileDestroyTasks) |
395 | 254 | ||
396 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) | 255 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) |
397 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) | 256 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) |
398 | await Promise.all(tasks) | 257 | await Promise.all(tasks) |
399 | 258 | ||
400 | // Update Tags | 259 | // Update Tags |
401 | const tags = videoObject.tag.map(tag => tag.name) | 260 | const tags = options.videoObject.tag.map(tag => tag.name) |
402 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 261 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
403 | await video.$set('Tags', tagInstances, sequelizeOptions) | 262 | await options.video.$set('Tags', tagInstances, sequelizeOptions) |
404 | 263 | ||
405 | // Update captions | 264 | // Update captions |
406 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) | 265 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) |
407 | 266 | ||
408 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | 267 | const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { |
409 | return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) | 268 | return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) |
410 | }) | 269 | }) |
411 | await Promise.all(videoCaptionsPromises) | 270 | await Promise.all(videoCaptionsPromises) |
412 | }) | 271 | }) |
413 | 272 | ||
414 | logger.info('Remote video with uuid %s updated', videoObject.uuid) | 273 | logger.info('Remote video with uuid %s updated', options.videoObject.uuid) |
415 | 274 | ||
416 | return updatedVideo | 275 | return updatedVideo |
417 | } catch (err) { | 276 | } catch (err) { |
418 | if (video !== undefined && videoFieldsSave !== undefined) { | 277 | if (options.video !== undefined && videoFieldsSave !== undefined) { |
419 | resetSequelizeInstance(video, videoFieldsSave) | 278 | resetSequelizeInstance(options.video, videoFieldsSave) |
420 | } | 279 | } |
421 | 280 | ||
422 | // This is just a debug because we will retry the insert | 281 | // This is just a debug because we will retry the insert |
@@ -433,12 +292,7 @@ export { | |||
433 | fetchRemoteVideoStaticFile, | 292 | fetchRemoteVideoStaticFile, |
434 | fetchRemoteVideoDescription, | 293 | fetchRemoteVideoDescription, |
435 | generateThumbnailFromUrl, | 294 | generateThumbnailFromUrl, |
436 | videoActivityObjectToDBAttributes, | 295 | getOrCreateVideoChannelFromVideoObject |
437 | videoFileActivityUrlToDBAttributes, | ||
438 | createVideo, | ||
439 | getOrCreateVideoChannelFromVideoObject, | ||
440 | addVideoShares, | ||
441 | createRates | ||
442 | } | 296 | } |
443 | 297 | ||
444 | // --------------------------------------------------------------------------- | 298 | // --------------------------------------------------------------------------- |
@@ -448,3 +302,178 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo | |||
448 | 302 | ||
449 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') | 303 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') |
450 | } | 304 | } |
305 | |||
306 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | ||
307 | logger.debug('Adding remote video %s.', videoObject.id) | ||
308 | |||
309 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
310 | const sequelizeOptions = { transaction: t } | ||
311 | |||
312 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
313 | const video = VideoModel.build(videoData) | ||
314 | |||
315 | const videoCreated = await video.save(sequelizeOptions) | ||
316 | |||
317 | // Process files | ||
318 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | ||
319 | if (videoFileAttributes.length === 0) { | ||
320 | throw new Error('Cannot find valid files for video %s ' + videoObject.url) | ||
321 | } | ||
322 | |||
323 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
324 | await Promise.all(videoFilePromises) | ||
325 | |||
326 | // Process tags | ||
327 | const tags = videoObject.tag.map(t => t.name) | ||
328 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
329 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
330 | |||
331 | // Process captions | ||
332 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
333 | return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) | ||
334 | }) | ||
335 | await Promise.all(videoCaptionsPromises) | ||
336 | |||
337 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
338 | |||
339 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
340 | return videoCreated | ||
341 | }) | ||
342 | |||
343 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | ||
344 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
345 | |||
346 | if (waitThumbnail === true) await p | ||
347 | |||
348 | return videoCreated | ||
349 | } | ||
350 | |||
351 | async function refreshVideoIfNeeded (options: { | ||
352 | video: VideoModel, | ||
353 | fetchedType: VideoFetchByUrlType, | ||
354 | syncParam: SyncParam, | ||
355 | refreshViews: boolean | ||
356 | }): Promise<VideoModel> { | ||
357 | if (!options.video.isOutdated()) return options.video | ||
358 | |||
359 | // We need more attributes if the argument video was fetched with not enough joints | ||
360 | const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) | ||
361 | |||
362 | try { | ||
363 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
364 | if (response.statusCode === 404) { | ||
365 | // Video does not exist anymore | ||
366 | await video.destroy() | ||
367 | return undefined | ||
368 | } | ||
369 | |||
370 | if (videoObject === undefined) { | ||
371 | logger.warn('Cannot refresh remote video: invalid body.') | ||
372 | return video | ||
373 | } | ||
374 | |||
375 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
376 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
377 | |||
378 | const updateOptions = { | ||
379 | video, | ||
380 | videoObject, | ||
381 | account, | ||
382 | channel: channelActor.VideoChannel, | ||
383 | updateViews: options.refreshViews | ||
384 | } | ||
385 | await updateVideoFromAP(updateOptions) | ||
386 | await syncVideoExternalAttributes(video, videoObject, options.syncParam) | ||
387 | } catch (err) { | ||
388 | logger.warn('Cannot refresh video.', { err }) | ||
389 | return video | ||
390 | } | ||
391 | } | ||
392 | |||
393 | async function videoActivityObjectToDBAttributes ( | ||
394 | videoChannel: VideoChannelModel, | ||
395 | videoObject: VideoTorrentObject, | ||
396 | to: string[] = [] | ||
397 | ) { | ||
398 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED | ||
399 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
400 | |||
401 | let language: string | undefined | ||
402 | if (videoObject.language) { | ||
403 | language = videoObject.language.identifier | ||
404 | } | ||
405 | |||
406 | let category: number | undefined | ||
407 | if (videoObject.category) { | ||
408 | category = parseInt(videoObject.category.identifier, 10) | ||
409 | } | ||
410 | |||
411 | let licence: number | undefined | ||
412 | if (videoObject.licence) { | ||
413 | licence = parseInt(videoObject.licence.identifier, 10) | ||
414 | } | ||
415 | |||
416 | const description = videoObject.content || null | ||
417 | const support = videoObject.support || null | ||
418 | |||
419 | return { | ||
420 | name: videoObject.name, | ||
421 | uuid: videoObject.uuid, | ||
422 | url: videoObject.id, | ||
423 | category, | ||
424 | licence, | ||
425 | language, | ||
426 | description, | ||
427 | support, | ||
428 | nsfw: videoObject.sensitive, | ||
429 | commentsEnabled: videoObject.commentsEnabled, | ||
430 | waitTranscoding: videoObject.waitTranscoding, | ||
431 | state: videoObject.state, | ||
432 | channelId: videoChannel.id, | ||
433 | duration: parseInt(duration, 10), | ||
434 | createdAt: new Date(videoObject.published), | ||
435 | publishedAt: new Date(videoObject.published), | ||
436 | // FIXME: updatedAt does not seems to be considered by Sequelize | ||
437 | updatedAt: new Date(videoObject.updated), | ||
438 | views: videoObject.views, | ||
439 | likes: 0, | ||
440 | dislikes: 0, | ||
441 | remote: true, | ||
442 | privacy | ||
443 | } | ||
444 | } | ||
445 | |||
446 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | ||
447 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
448 | |||
449 | if (fileUrls.length === 0) { | ||
450 | throw new Error('Cannot find video files for ' + videoCreated.url) | ||
451 | } | ||
452 | |||
453 | const attributes: VideoFileModel[] = [] | ||
454 | for (const fileUrl of fileUrls) { | ||
455 | // Fetch associated magnet uri | ||
456 | const magnet = videoObject.url.find(u => { | ||
457 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | ||
458 | }) | ||
459 | |||
460 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
461 | |||
462 | const parsed = magnetUtil.decode(magnet.href) | ||
463 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
464 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
465 | } | ||
466 | |||
467 | const attribute = { | ||
468 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | ||
469 | infoHash: parsed.infoHash, | ||
470 | resolution: fileUrl.height, | ||
471 | size: fileUrl.size, | ||
472 | videoId: videoCreated.id, | ||
473 | fps: fileUrl.fps | ||
474 | } as VideoFileModel | ||
475 | attributes.push(attribute) | ||
476 | } | ||
477 | |||
478 | return attributes | ||
479 | } | ||