]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blame - server/lib/activitypub/videos.ts
Upgrade sequelize
[github/Chocobozzz/PeerTube.git] / server / lib / activitypub / videos.ts
CommitLineData
7acee6f1 1import * as Bluebird from 'bluebird'
2186386c 2import * as sequelize from 'sequelize'
2ccaeeb3 3import * as magnetUtil from 'magnet-uri'
892211e8 4import * as request from 'request'
09209296 5import {
09209296
C
6 ActivityPlaylistSegmentHashesObject,
7 ActivityPlaylistUrlObject,
8 ActivityUrlObject,
1735c825 9 ActivityVideoUrlObject,
09209296
C
10 VideoState
11} from '../../../shared/index'
2ccaeeb3 12import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
1297eb5d 13import { VideoPrivacy } from '../../../shared/models/videos'
1d6e5dfc 14import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
2ccaeeb3 15import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
c48e82b5 16import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
2ccaeeb3 17import { logger } from '../../helpers/logger'
e8bafea3
C
18import { doRequest } from '../../helpers/requests'
19import {
20 ACTIVITY_PUB,
21 MIMETYPES,
22 P2P_MEDIA_LOADER_PEER_VERSION,
23 PREVIEWS_SIZE,
24 REMOTE_SCHEME,
25 STATIC_PATHS
26} from '../../initializers/constants'
2ccaeeb3
C
27import { ActorModel } from '../../models/activitypub/actor'
28import { TagModel } from '../../models/video/tag'
3fd3ab2d 29import { VideoModel } from '../../models/video/video'
2ccaeeb3
C
30import { VideoChannelModel } from '../../models/video/video-channel'
31import { VideoFileModel } from '../../models/video/video-file'
c48e82b5 32import { getOrCreateActorAndServerAndModel } from './actor'
7acee6f1 33import { addVideoComments } from './video-comments'
8fffe21a 34import { crawlCollectionPage } from './crawl'
2186386c 35import { sendCreateVideo, sendUpdateVideo } from './send'
40e87e9e
C
36import { isArray } from '../../helpers/custom-validators/misc'
37import { VideoCaptionModel } from '../../models/video/video-caption'
f6eebcb3
C
38import { JobQueue } from '../job-queue'
39import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
1297eb5d
C
40import { createRates } from './video-rates'
41import { addVideoShares, shareVideoByServerAndChannel } from './share'
42import { AccountModel } from '../../models/account/account'
4157cdb1 43import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
848f499d 44import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
cef534ed 45import { Notifier } from '../notifier'
09209296
C
46import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
47import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
2ba92871
C
48import { AccountVideoRateModel } from '../../models/account/account-video-rate'
49import { VideoShareModel } from '../../models/video/video-share'
50import { VideoCommentModel } from '../../models/video/video-comment'
74dc3bca 51import { sequelizeTypescript } from '../../initializers/database'
3acc5084 52import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
e8bafea3
C
53import { ThumbnailModel } from '../../models/video/thumbnail'
54import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
55import { join } from 'path'
3acc5084 56import { FilteredModelAttributes } from '../../typings/sequelize'
2186386c
C
57
58async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
7ccddd7b 59 // If the video is not private and is published, we federate it
2186386c 60 if (video.privacy !== VideoPrivacy.PRIVATE && video.state === VideoState.PUBLISHED) {
40e87e9e
C
61 // Fetch more attributes that we will need to serialize in AP object
62 if (isArray(video.VideoCaptions) === false) {
63 video.VideoCaptions = await video.$get('VideoCaptions', {
64 attributes: [ 'language' ],
65 transaction
66 }) as VideoCaptionModel[]
67 }
68
2cebd797 69 if (isNewVideo) {
2186386c
C
70 // Now we'll add the video's meta data to our followers
71 await sendCreateVideo(video, transaction)
72 await shareVideoByServerAndChannel(video, transaction)
73 } else {
74 await sendUpdateVideo(video, transaction)
75 }
76 }
77}
892211e8 78
4157cdb1
C
79async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
80 const options = {
81 uri: videoUrl,
82 method: 'GET',
83 json: true,
84 activityPub: true
85 }
892211e8 86
4157cdb1
C
87 logger.info('Fetching remote video %s.', videoUrl)
88
89 const { response, body } = await doRequest(options)
90
5c6d985f 91 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
4157cdb1
C
92 logger.debug('Remote video JSON is not valid.', { body })
93 return { response, videoObject: undefined }
94 }
95
96 return { response, videoObject: body }
892211e8
C
97}
98
3fd3ab2d 99async function fetchRemoteVideoDescription (video: VideoModel) {
50d6de9c 100 const host = video.VideoChannel.Account.Actor.Server.host
96f29c0f 101 const path = video.getDescriptionAPIPath()
892211e8
C
102 const options = {
103 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
104 json: true
105 }
106
107 const { body } = await doRequest(options)
108 return body.description ? body.description : ''
109}
110
4157cdb1 111function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
e8bafea3 112 const url = buildRemoteBaseUrl(video, path)
4157cdb1
C
113
114 // We need to provide a callback, if no we could have an uncaught exception
e8bafea3 115 return request.get(url, err => {
4157cdb1
C
116 if (err) reject(err)
117 })
118}
119
e8bafea3
C
120function buildRemoteBaseUrl (video: VideoModel, path: string) {
121 const host = video.VideoChannel.Account.Actor.Server.host
892211e8 122
e8bafea3 123 return REMOTE_SCHEME.HTTP + '://' + host + path
892211e8
C
124}
125
f37dc0dd 126function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
0f320037
C
127 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
128 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
129
5c6d985f
C
130 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
131 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
132 }
133
e587e0ec 134 return getOrCreateActorAndServerAndModel(channel.id, 'all')
0f320037
C
135}
136
f6eebcb3 137type SyncParam = {
1297eb5d
C
138 likes: boolean
139 dislikes: boolean
140 shares: boolean
141 comments: boolean
f6eebcb3 142 thumbnail: boolean
04b8c3fb 143 refreshVideo?: boolean
f6eebcb3 144}
4157cdb1 145async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
f6eebcb3 146 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
2ccaeeb3 147
f6eebcb3 148 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
2ccaeeb3 149
f6eebcb3 150 if (syncParam.likes === true) {
2ba92871
C
151 const handler = items => createRates(items, video, 'like')
152 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
153
154 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
f6eebcb3
C
155 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
156 } else {
157 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
158 }
7acee6f1 159
f6eebcb3 160 if (syncParam.dislikes === true) {
2ba92871
C
161 const handler = items => createRates(items, video, 'dislike')
162 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
163
164 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
f6eebcb3
C
165 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
166 } else {
167 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
168 }
169
170 if (syncParam.shares === true) {
2ba92871
C
171 const handler = items => addVideoShares(items, video)
172 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
173
174 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
f6eebcb3
C
175 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
176 } else {
177 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
178 }
7acee6f1 179
f6eebcb3 180 if (syncParam.comments === true) {
2ba92871
C
181 const handler = items => addVideoComments(items, video)
182 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
183
184 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
f6eebcb3
C
185 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
186 } else {
2ba92871 187 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
f6eebcb3 188 }
7acee6f1 189
f6eebcb3 190 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
2ccaeeb3
C
191}
192
4157cdb1 193async function getOrCreateVideoAndAccountAndChannel (options: {
848f499d 194 videoObject: { id: string } | string,
4157cdb1 195 syncParam?: SyncParam,
d4defe07 196 fetchType?: VideoFetchByUrlType,
74577825 197 allowRefresh?: boolean // true by default
4157cdb1
C
198}) {
199 // Default params
200 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
201 const fetchType = options.fetchType || 'all'
74577825 202 const allowRefresh = options.allowRefresh !== false
1297eb5d 203
4157cdb1 204 // Get video url
848f499d 205 const videoUrl = getAPId(options.videoObject)
1297eb5d 206
4157cdb1
C
207 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
208 if (videoFromDatabase) {
f58094b2 209 if (videoFromDatabase.isOutdated() && allowRefresh === true) {
74577825
C
210 const refreshOptions = {
211 video: videoFromDatabase,
212 fetchedType: fetchType,
213 syncParam
214 }
215
216 if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
744d0eca 217 else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
d4defe07 218 }
1297eb5d 219
cef534ed 220 return { video: videoFromDatabase, created: false }
1297eb5d
C
221 }
222
4157cdb1
C
223 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
224 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
7acee6f1 225
4157cdb1
C
226 const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
227 const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
7acee6f1 228
4157cdb1 229 await syncVideoExternalAttributes(video, fetchedVideo, syncParam)
7acee6f1 230
cef534ed 231 return { video, created: true }
7acee6f1
C
232}
233
d4defe07 234async function updateVideoFromAP (options: {
1297eb5d
C
235 video: VideoModel,
236 videoObject: VideoTorrentObject,
c48e82b5
C
237 account: AccountModel,
238 channel: VideoChannelModel,
1297eb5d 239 overrideTo?: string[]
d4defe07
C
240}) {
241 logger.debug('Updating remote video "%s".', options.videoObject.uuid)
e8d246d5 242
1297eb5d 243 let videoFieldsSave: any
e8d246d5
C
244 const wasPrivateVideo = options.video.privacy === VideoPrivacy.PRIVATE
245 const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED
1297eb5d
C
246
247 try {
e8bafea3
C
248 let thumbnailModel: ThumbnailModel
249
250 try {
3acc5084 251 thumbnailModel = await createVideoMiniatureFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.MINIATURE)
e8bafea3
C
252 } catch (err) {
253 logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
254 }
255
d382f4e9 256 await sequelizeTypescript.transaction(async t => {
e8d246d5 257 const sequelizeOptions = { transaction: t }
2ccaeeb3 258
d4defe07 259 videoFieldsSave = options.video.toJSON()
2ccaeeb3 260
1297eb5d 261 // Check actor has the right to update the video
d4defe07
C
262 const videoChannel = options.video.VideoChannel
263 if (videoChannel.Account.id !== options.account.id) {
264 throw new Error('Account ' + options.account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
f6eebcb3
C
265 }
266
d4defe07
C
267 const to = options.overrideTo ? options.overrideTo : options.videoObject.to
268 const videoData = await videoActivityObjectToDBAttributes(options.channel, options.videoObject, to)
269 options.video.set('name', videoData.name)
270 options.video.set('uuid', videoData.uuid)
271 options.video.set('url', videoData.url)
272 options.video.set('category', videoData.category)
273 options.video.set('licence', videoData.licence)
274 options.video.set('language', videoData.language)
275 options.video.set('description', videoData.description)
276 options.video.set('support', videoData.support)
277 options.video.set('nsfw', videoData.nsfw)
278 options.video.set('commentsEnabled', videoData.commentsEnabled)
7f2cfe3a 279 options.video.set('downloadEnabled', videoData.downloadEnabled)
d4defe07
C
280 options.video.set('waitTranscoding', videoData.waitTranscoding)
281 options.video.set('state', videoData.state)
282 options.video.set('duration', videoData.duration)
283 options.video.set('createdAt', videoData.createdAt)
284 options.video.set('publishedAt', videoData.publishedAt)
7519127b 285 options.video.set('originallyPublishedAt', videoData.originallyPublishedAt)
d4defe07
C
286 options.video.set('privacy', videoData.privacy)
287 options.video.set('channelId', videoData.channelId)
04b8c3fb 288 options.video.set('views', videoData.views)
d4defe07 289
d4defe07 290 await options.video.save(sequelizeOptions)
1297eb5d 291
3acc5084 292 if (thumbnailModel) if (thumbnailModel) await options.video.addAndSaveThumbnail(thumbnailModel, t)
e8bafea3
C
293
294 // FIXME: use icon URL instead
295 const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename))
296 const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
3acc5084 297 await options.video.addAndSaveThumbnail(previewModel, t)
e8bafea3 298
e5565833
C
299 {
300 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
301 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
0032ebe9 302
e5565833
C
303 // Remove video files that do not exist anymore
304 const destroyTasks = options.video.VideoFiles
305 .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
306 .map(f => f.destroy(sequelizeOptions))
307 await Promise.all(destroyTasks)
2ccaeeb3 308
e5565833 309 // Update or add other one
d382f4e9 310 const upsertTasks = videoFileAttributes.map(a => {
3acc5084 311 return VideoFileModel.upsert<VideoFileModel>(a, { returning: true, transaction: t })
d382f4e9
C
312 .then(([ file ]) => file)
313 })
314
315 options.video.VideoFiles = await Promise.all(upsertTasks)
e5565833 316 }
2ccaeeb3 317
09209296 318 {
ae9bbed4
C
319 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
320 options.video,
321 options.videoObject,
322 options.video.VideoFiles
323 )
09209296
C
324 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
325
326 // Remove video files that do not exist anymore
327 const destroyTasks = options.video.VideoStreamingPlaylists
328 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
329 .map(f => f.destroy(sequelizeOptions))
330 await Promise.all(destroyTasks)
331
332 // Update or add other one
333 const upsertTasks = streamingPlaylistAttributes.map(a => {
3acc5084 334 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
09209296
C
335 .then(([ streamingPlaylist ]) => streamingPlaylist)
336 })
337
338 options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
339 }
340
e5565833
C
341 {
342 // Update Tags
343 const tags = options.videoObject.tag.map(tag => tag.name)
344 const tagInstances = await TagModel.findOrCreateTags(tags, t)
345 await options.video.$set('Tags', tagInstances, sequelizeOptions)
346 }
2ccaeeb3 347
e5565833
C
348 {
349 // Update captions
350 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t)
351
352 const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => {
353 return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t)
354 })
d382f4e9 355 options.video.VideoCaptions = await Promise.all(videoCaptionsPromises)
e5565833 356 }
1297eb5d
C
357 })
358
e8d246d5
C
359 // Notify our users?
360 if (wasPrivateVideo || wasUnlistedVideo) {
361 Notifier.Instance.notifyOnNewVideo(options.video)
362 }
363
d4defe07 364 logger.info('Remote video with uuid %s updated', options.videoObject.uuid)
1297eb5d 365 } catch (err) {
d4defe07
C
366 if (options.video !== undefined && videoFieldsSave !== undefined) {
367 resetSequelizeInstance(options.video, videoFieldsSave)
1297eb5d
C
368 }
369
370 // This is just a debug because we will retry the insert
371 logger.debug('Cannot update the remote video.', { err })
372 throw err
373 }
892211e8 374}
2186386c 375
04b8c3fb
C
376async function refreshVideoIfNeeded (options: {
377 video: VideoModel,
378 fetchedType: VideoFetchByUrlType,
379 syncParam: SyncParam
380}): Promise<VideoModel> {
381 if (!options.video.isOutdated()) return options.video
382
383 // We need more attributes if the argument video was fetched with not enough joints
384 const video = options.fetchedType === 'all' ? options.video : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
385
386 try {
387 const { response, videoObject } = await fetchRemoteVideo(video.url)
388 if (response.statusCode === 404) {
389 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
390
391 // Video does not exist anymore
392 await video.destroy()
393 return undefined
394 }
395
396 if (videoObject === undefined) {
397 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
398
399 await video.setAsRefreshed()
400 return video
401 }
402
403 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
404 const account = await AccountModel.load(channelActor.VideoChannel.accountId)
405
406 const updateOptions = {
407 video,
408 videoObject,
409 account,
410 channel: channelActor.VideoChannel
411 }
412 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
413 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
414
415 return video
416 } catch (err) {
417 logger.warn('Cannot refresh video %s.', options.video.url, { err })
418
419 // Don't refresh in loop
420 await video.setAsRefreshed()
421 return video
422 }
892211e8 423}
2186386c
C
424
425export {
1297eb5d 426 updateVideoFromAP,
04b8c3fb 427 refreshVideoIfNeeded,
2186386c
C
428 federateVideoIfNeeded,
429 fetchRemoteVideo,
1297eb5d 430 getOrCreateVideoAndAccountAndChannel,
40e87e9e 431 fetchRemoteVideoStaticFile,
2186386c 432 fetchRemoteVideoDescription,
4157cdb1 433 getOrCreateVideoChannelFromVideoObject
2186386c 434}
c48e82b5
C
435
436// ---------------------------------------------------------------------------
437
09209296 438function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
14e2014a 439 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
c48e82b5 440
e27ff5da
C
441 const urlMediaType = url.mediaType || url.mimeType
442 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
c48e82b5 443}
4157cdb1 444
09209296
C
445function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
446 const urlMediaType = url.mediaType || url.mimeType
447
448 return urlMediaType === 'application/x-mpegURL'
449}
450
451function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
452 const urlMediaType = tag.mediaType || tag.mimeType
453
454 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
c48e82b5 455}
4157cdb1
C
456
457async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
458 logger.debug('Adding remote video %s.', videoObject.id)
459
e8bafea3
C
460 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
461 const video = VideoModel.build(videoData)
462
3acc5084 463 const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
e8bafea3
C
464
465 let thumbnailModel: ThumbnailModel
466 if (waitThumbnail === true) {
467 thumbnailModel = await promiseThumbnail
468 }
469
4157cdb1
C
470 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
471 const sequelizeOptions = { transaction: t }
472
4157cdb1 473 const videoCreated = await video.save(sequelizeOptions)
e8bafea3
C
474 videoCreated.VideoChannel = channelActor.VideoChannel
475
3acc5084 476 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
e8bafea3
C
477
478 // FIXME: use icon URL instead
479 const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
480 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
3acc5084 481 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
4157cdb1
C
482
483 // Process files
484 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
485 if (videoFileAttributes.length === 0) {
486 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
487 }
488
489 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
ae9bbed4 490 const videoFiles = await Promise.all(videoFilePromises)
4157cdb1 491
ae9bbed4 492 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
09209296
C
493 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
494 await Promise.all(playlistPromises)
495
4157cdb1 496 // Process tags
09209296
C
497 const tags = videoObject.tag
498 .filter(t => t.type === 'Hashtag')
499 .map(t => t.name)
4157cdb1
C
500 const tagInstances = await TagModel.findOrCreateTags(tags, t)
501 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
502
503 // Process captions
504 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
505 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
506 })
507 await Promise.all(videoCaptionsPromises)
508
509 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
510
4157cdb1
C
511 return videoCreated
512 })
513
e8bafea3
C
514 if (waitThumbnail === false) {
515 promiseThumbnail.then(thumbnailModel => {
516 thumbnailModel = videoCreated.id
4157cdb1 517
e8bafea3
C
518 return thumbnailModel.save()
519 })
520 }
4157cdb1
C
521
522 return videoCreated
523}
524
4157cdb1
C
525async function videoActivityObjectToDBAttributes (
526 videoChannel: VideoChannelModel,
527 videoObject: VideoTorrentObject,
528 to: string[] = []
529) {
530 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
531 const duration = videoObject.duration.replace(/[^\d]+/, '')
532
533 let language: string | undefined
534 if (videoObject.language) {
535 language = videoObject.language.identifier
536 }
537
538 let category: number | undefined
539 if (videoObject.category) {
540 category = parseInt(videoObject.category.identifier, 10)
541 }
542
543 let licence: number | undefined
544 if (videoObject.licence) {
545 licence = parseInt(videoObject.licence.identifier, 10)
546 }
547
548 const description = videoObject.content || null
549 const support = videoObject.support || null
550
551 return {
552 name: videoObject.name,
553 uuid: videoObject.uuid,
554 url: videoObject.id,
555 category,
556 licence,
557 language,
558 description,
559 support,
560 nsfw: videoObject.sensitive,
561 commentsEnabled: videoObject.commentsEnabled,
7f2cfe3a 562 downloadEnabled: videoObject.downloadEnabled,
4157cdb1
C
563 waitTranscoding: videoObject.waitTranscoding,
564 state: videoObject.state,
565 channelId: videoChannel.id,
566 duration: parseInt(duration, 10),
567 createdAt: new Date(videoObject.published),
568 publishedAt: new Date(videoObject.published),
7519127b 569 originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
4157cdb1
C
570 // FIXME: updatedAt does not seems to be considered by Sequelize
571 updatedAt: new Date(videoObject.updated),
572 views: videoObject.views,
573 likes: 0,
574 dislikes: 0,
575 remote: true,
576 privacy
577 }
578}
579
a3737cbf 580function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
09209296 581 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
4157cdb1
C
582
583 if (fileUrls.length === 0) {
a3737cbf 584 throw new Error('Cannot find video files for ' + video.url)
4157cdb1
C
585 }
586
3acc5084 587 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
4157cdb1
C
588 for (const fileUrl of fileUrls) {
589 // Fetch associated magnet uri
590 const magnet = videoObject.url.find(u => {
e27ff5da
C
591 const mediaType = u.mediaType || u.mimeType
592 return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
4157cdb1
C
593 })
594
595 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
596
597 const parsed = magnetUtil.decode(magnet.href)
598 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
599 throw new Error('Cannot parse magnet URI ' + magnet.href)
600 }
601
e27ff5da 602 const mediaType = fileUrl.mediaType || fileUrl.mimeType
4157cdb1 603 const attribute = {
14e2014a 604 extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
4157cdb1
C
605 infoHash: parsed.infoHash,
606 resolution: fileUrl.height,
607 size: fileUrl.size,
a3737cbf 608 videoId: video.id,
2e7cf5ae 609 fps: fileUrl.fps || -1
09209296
C
610 }
611
612 attributes.push(attribute)
613 }
614
615 return attributes
616}
617
ae9bbed4 618function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
09209296
C
619 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
620 if (playlistUrls.length === 0) return []
621
3acc5084 622 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
09209296 623 for (const playlistUrlObject of playlistUrls) {
09209296
C
624 const segmentsSha256UrlObject = playlistUrlObject.tag
625 .find(t => {
626 return isAPPlaylistSegmentHashesUrlObject(t)
627 }) as ActivityPlaylistSegmentHashesObject
628 if (!segmentsSha256UrlObject) {
629 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
630 continue
631 }
632
633 const attribute = {
634 type: VideoStreamingPlaylistType.HLS,
635 playlistUrl: playlistUrlObject.href,
636 segmentsSha256Url: segmentsSha256UrlObject.href,
ae9bbed4 637 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
594d0c6a 638 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
09209296
C
639 videoId: video.id
640 }
641
4157cdb1
C
642 attributes.push(attribute)
643 }
644
645 return attributes
646}