1 import * as Bluebird from 'bluebird'
2 import * as sequelize from 'sequelize'
3 import * as magnetUtil from 'magnet-uri'
4 import * as request from 'request'
7 ActivityMagnetUrlObject,
8 ActivityPlaylistSegmentHashesObject,
9 ActivityPlaylistUrlObject, ActivityTagObject,
11 ActivityVideoUrlObject,
13 } from '../../../shared/index'
14 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
15 import { VideoPrivacy } from '../../../shared/models/videos'
16 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
17 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
18 import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
19 import { logger } from '../../helpers/logger'
20 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
24 P2P_MEDIA_LOADER_PEER_VERSION,
28 } from '../../initializers/constants'
29 import { TagModel } from '../../models/video/tag'
30 import { VideoModel } from '../../models/video/video'
31 import { VideoFileModel } from '../../models/video/video-file'
32 import { getOrCreateActorAndServerAndModel } from './actor'
33 import { addVideoComments } from './video-comments'
34 import { crawlCollectionPage } from './crawl'
35 import { sendCreateVideo, sendUpdateVideo } from './send'
36 import { isArray } from '../../helpers/custom-validators/misc'
37 import { VideoCaptionModel } from '../../models/video/video-caption'
38 import { JobQueue } from '../job-queue'
39 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
40 import { createRates } from './video-rates'
41 import { addVideoShares, shareVideoByServerAndChannel } from './share'
42 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
43 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
44 import { Notifier } from '../notifier'
45 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
46 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
47 import { AccountVideoRateModel } from '../../models/account/account-video-rate'
48 import { VideoShareModel } from '../../models/video/video-share'
49 import { VideoCommentModel } from '../../models/video/video-comment'
50 import { sequelizeTypescript } from '../../initializers/database'
51 import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
52 import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
53 import { join } from 'path'
54 import { FilteredModelAttributes } from '../../typings/sequelize'
55 import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
56 import { ActorFollowScoreCache } from '../files-cache'
65 MVideoAccountLightBlacklistAllFiles,
67 MVideoAPWithoutCaption,
72 } from '../../typings/models'
73 import { MThumbnail } from '../../typings/models/video/thumbnail'
75 async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
76 const video = videoArg as MVideoAP
79 // Check this is not a blacklisted video, or unfederated blacklisted video
80 (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) &&
81 // Check the video is public/unlisted and published
82 video.hasPrivacyForFederation() && video.state === VideoState.PUBLISHED
84 // Fetch more attributes that we will need to serialize in AP object
85 if (isArray(video.VideoCaptions) === false) {
86 video.VideoCaptions = await video.$get('VideoCaptions', {
87 attributes: [ 'language' ],
89 }) as VideoCaptionModel[]
93 // Now we'll add the video's meta data to our followers
94 await sendCreateVideo(video, transaction)
95 await shareVideoByServerAndChannel(video, transaction)
97 await sendUpdateVideo(video, transaction)
102 async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
110 logger.info('Fetching remote video %s.', videoUrl)
112 const { response, body } = await doRequest(options)
114 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
115 logger.debug('Remote video JSON is not valid.', { body })
116 return { response, videoObject: undefined }
119 return { response, videoObject: body }
122 async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
123 const host = video.VideoChannel.Account.Actor.Server.host
124 const path = video.getDescriptionAPIPath()
126 uri: REMOTE_SCHEME.HTTP + '://' + host + path,
130 const { body } = await doRequest(options)
131 return body.description ? body.description : ''
134 function fetchRemoteVideoStaticFile (video: MVideoAccountLight, path: string, destPath: string) {
135 const url = buildRemoteBaseUrl(video, path)
137 // We need to provide a callback, if no we could have an uncaught exception
138 return doRequestAndSaveToFile({ uri: url }, destPath)
141 function buildRemoteBaseUrl (video: MVideoAccountLight, path: string) {
142 const host = video.VideoChannel.Account.Actor.Server.host
144 return REMOTE_SCHEME.HTTP + '://' + host + path
147 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
148 const channel = videoObject.attributedTo.find(a => a.type === 'Group')
149 if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
151 if (checkUrlsSameHost(channel.id, videoObject.id) !== true) {
152 throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`)
155 return getOrCreateActorAndServerAndModel(channel.id, 'all')
164 refreshVideo?: boolean
166 async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) {
167 logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid)
169 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
171 if (syncParam.likes === true) {
172 const handler = items => createRates(items, video, 'like')
173 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
175 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
176 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
178 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
181 if (syncParam.dislikes === true) {
182 const handler = items => createRates(items, video, 'dislike')
183 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
185 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
186 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
188 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
191 if (syncParam.shares === true) {
192 const handler = items => addVideoShares(items, video)
193 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
195 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
196 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
198 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
201 if (syncParam.comments === true) {
202 const handler = items => addVideoComments(items)
203 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
205 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
206 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
208 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
211 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
214 function getOrCreateVideoAndAccountAndChannel (options: {
215 videoObject: { id: string } | string,
216 syncParam?: SyncParam,
218 allowRefresh?: boolean
219 }): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }>
220 function getOrCreateVideoAndAccountAndChannel (options: {
221 videoObject: { id: string } | string,
222 syncParam?: SyncParam,
223 fetchType?: VideoFetchByUrlType,
224 allowRefresh?: boolean
225 }): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }>
226 async function getOrCreateVideoAndAccountAndChannel (options: {
227 videoObject: { id: string } | string,
228 syncParam?: SyncParam,
229 fetchType?: VideoFetchByUrlType,
230 allowRefresh?: boolean // true by default
231 }): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> {
233 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
234 const fetchType = options.fetchType || 'all'
235 const allowRefresh = options.allowRefresh !== false
238 const videoUrl = getAPId(options.videoObject)
240 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
241 if (videoFromDatabase) {
242 if (videoFromDatabase.isOutdated() && allowRefresh === true) {
243 const refreshOptions = {
244 video: videoFromDatabase,
245 fetchedType: fetchType,
249 if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
250 else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } })
253 return { video: videoFromDatabase, created: false }
256 const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
257 if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
259 const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo)
260 const videoChannel = actor.VideoChannel
261 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail)
263 await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam)
265 return { video: videoCreated, created: true, autoBlacklisted }
268 async function updateVideoFromAP (options: {
269 video: MVideoAccountLightBlacklistAllFiles,
270 videoObject: VideoTorrentObject,
271 account: MAccountIdActor,
272 channel: MChannelDefault,
273 overrideTo?: string[]
275 const { video, videoObject, account, channel, overrideTo } = options
277 logger.debug('Updating remote video "%s".', options.videoObject.uuid, { account, channel })
279 let videoFieldsSave: any
280 const wasPrivateVideo = video.privacy === VideoPrivacy.PRIVATE
281 const wasUnlistedVideo = video.privacy === VideoPrivacy.UNLISTED
284 let thumbnailModel: MThumbnail
287 thumbnailModel = await createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
289 logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })
292 const videoUpdated = await sequelizeTypescript.transaction(async t => {
293 const sequelizeOptions = { transaction: t }
295 videoFieldsSave = video.toJSON()
297 // Check actor has the right to update the video
298 const videoChannel = video.VideoChannel
299 if (videoChannel.Account.id !== account.id) {
300 throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
303 const to = overrideTo ? overrideTo : videoObject.to
304 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
305 video.name = videoData.name
306 video.uuid = videoData.uuid
307 video.url = videoData.url
308 video.category = videoData.category
309 video.licence = videoData.licence
310 video.language = videoData.language
311 video.description = videoData.description
312 video.support = videoData.support
313 video.nsfw = videoData.nsfw
314 video.commentsEnabled = videoData.commentsEnabled
315 video.downloadEnabled = videoData.downloadEnabled
316 video.waitTranscoding = videoData.waitTranscoding
317 video.state = videoData.state
318 video.duration = videoData.duration
319 video.createdAt = videoData.createdAt
320 video.publishedAt = videoData.publishedAt
321 video.originallyPublishedAt = videoData.originallyPublishedAt
322 video.privacy = videoData.privacy
323 video.channelId = videoData.channelId
324 video.views = videoData.views
326 const videoUpdated = await video.save(sequelizeOptions) as MVideoFullLight
328 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t)
330 // FIXME: use icon URL instead
331 const previewUrl = buildRemoteBaseUrl(videoUpdated, join(STATIC_PATHS.PREVIEWS, videoUpdated.getPreview().filename))
332 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
333 await videoUpdated.addAndSaveThumbnail(previewModel, t)
336 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
337 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
339 // Remove video files that do not exist anymore
340 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
341 await Promise.all(destroyTasks)
343 // Update or add other one
344 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
345 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
349 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
350 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
352 // Remove video playlists that do not exist anymore
353 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
354 await Promise.all(destroyTasks)
356 let oldStreamingPlaylistFiles: MVideoFile[] = []
357 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
358 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
361 videoUpdated.VideoStreamingPlaylists = []
363 for (const playlistAttributes of streamingPlaylistAttributes) {
364 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
365 .then(([ streamingPlaylist ]) => streamingPlaylist)
367 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
368 .map(a => new VideoFileModel(a))
369 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
370 await Promise.all(destroyTasks)
372 // Update or add other one
373 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
374 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
376 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
382 const tags = videoObject.tag
383 .filter(isAPHashTagObject)
384 .map(tag => tag.name)
385 const tagInstances = await TagModel.findOrCreateTags(tags, t)
386 await videoUpdated.$set('Tags', tagInstances, sequelizeOptions)
391 await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t)
393 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
394 return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, t)
396 await Promise.all(videoCaptionsPromises)
402 await autoBlacklistVideoIfNeeded({
407 transaction: undefined
410 if (wasPrivateVideo || wasUnlistedVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) // Notify our users?
412 logger.info('Remote video with uuid %s updated', videoObject.uuid)
416 if (video !== undefined && videoFieldsSave !== undefined) {
417 resetSequelizeInstance(video, videoFieldsSave)
420 // This is just a debug because we will retry the insert
421 logger.debug('Cannot update the remote video.', { err })
426 async function refreshVideoIfNeeded (options: {
427 video: MVideoThumbnail,
428 fetchedType: VideoFetchByUrlType,
430 }): Promise<MVideoThumbnail> {
431 if (!options.video.isOutdated()) return options.video
433 // We need more attributes if the argument video was fetched with not enough joints
434 const video = options.fetchedType === 'all'
435 ? options.video as MVideoAccountLightBlacklistAllFiles
436 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
439 const { response, videoObject } = await fetchRemoteVideo(video.url)
440 if (response.statusCode === 404) {
441 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
443 // Video does not exist anymore
444 await video.destroy()
448 if (videoObject === undefined) {
449 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
451 await video.setAsRefreshed()
455 const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
457 const updateOptions = {
460 account: channelActor.VideoChannel.Account,
461 channel: channelActor.VideoChannel
463 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
464 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
466 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
470 logger.warn('Cannot refresh video %s.', options.video.url, { err })
472 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
474 // Don't refresh in loop
475 await video.setAsRefreshed()
482 refreshVideoIfNeeded,
483 federateVideoIfNeeded,
485 getOrCreateVideoAndAccountAndChannel,
486 fetchRemoteVideoStaticFile,
487 fetchRemoteVideoDescription,
488 getOrCreateVideoChannelFromVideoObject
491 // ---------------------------------------------------------------------------
493 function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
494 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
496 const urlMediaType = url.mediaType
497 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
500 function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
501 return url && url.mediaType === 'application/x-mpegURL'
504 function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
505 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
508 function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
509 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
512 function isAPHashTagObject (url: any): url is ActivityHashTagObject {
513 return url && url.type === 'Hashtag'
516 async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
517 logger.debug('Adding remote video %s.', videoObject.id)
519 const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to)
520 const video = VideoModel.build(videoData) as MVideoThumbnail
522 const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE)
524 let thumbnailModel: MThumbnail
525 if (waitThumbnail === true) {
526 thumbnailModel = await promiseThumbnail
529 const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
530 const sequelizeOptions = { transaction: t }
532 const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
533 videoCreated.VideoChannel = channel
535 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
537 // FIXME: use icon URL instead
538 const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
539 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
540 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
543 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
545 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
546 const videoFiles = await Promise.all(videoFilePromises)
548 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
549 videoCreated.VideoStreamingPlaylists = []
551 for (const playlistAttributes of streamingPlaylistsAttributes) {
552 const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t })
554 const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject)
555 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
556 playlistModel.VideoFiles = await Promise.all(videoFilePromises)
558 videoCreated.VideoStreamingPlaylists.push(playlistModel)
562 const tags = videoObject.tag
563 .filter(isAPHashTagObject)
565 const tagInstances = await TagModel.findOrCreateTags(tags, t)
566 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
569 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
570 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
572 await Promise.all(videoCaptionsPromises)
574 videoCreated.VideoFiles = videoFiles
575 videoCreated.Tags = tagInstances
577 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
585 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
587 return { autoBlacklisted, videoCreated }
590 if (waitThumbnail === false) {
591 promiseThumbnail.then(thumbnailModel => {
592 thumbnailModel = videoCreated.id
594 return thumbnailModel.save()
598 return { autoBlacklisted, videoCreated }
601 async function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) {
602 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED
603 const duration = videoObject.duration.replace(/[^\d]+/, '')
605 let language: string | undefined
606 if (videoObject.language) {
607 language = videoObject.language.identifier
610 let category: number | undefined
611 if (videoObject.category) {
612 category = parseInt(videoObject.category.identifier, 10)
615 let licence: number | undefined
616 if (videoObject.licence) {
617 licence = parseInt(videoObject.licence.identifier, 10)
620 const description = videoObject.content || null
621 const support = videoObject.support || null
624 name: videoObject.name,
625 uuid: videoObject.uuid,
632 nsfw: videoObject.sensitive,
633 commentsEnabled: videoObject.commentsEnabled,
634 downloadEnabled: videoObject.downloadEnabled,
635 waitTranscoding: videoObject.waitTranscoding,
636 state: videoObject.state,
637 channelId: videoChannel.id,
638 duration: parseInt(duration, 10),
639 createdAt: new Date(videoObject.published),
640 publishedAt: new Date(videoObject.published),
641 originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null,
642 // FIXME: updatedAt does not seems to be considered by Sequelize
643 updatedAt: new Date(videoObject.updated),
644 views: videoObject.views,
652 function videoFileActivityUrlToDBAttributes (
653 videoOrPlaylist: MVideo | MStreamingPlaylist,
654 urls: (ActivityTagObject | ActivityUrlObject)[]
656 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
658 if (fileUrls.length === 0) return []
660 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
661 for (const fileUrl of fileUrls) {
662 // Fetch associated magnet uri
663 const magnet = urls.filter(isAPMagnetUrlObject)
664 .find(u => u.height === fileUrl.height)
666 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
668 const parsed = magnetUtil.decode(magnet.href)
669 if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) {
670 throw new Error('Cannot parse magnet URI ' + magnet.href)
673 const mediaType = fileUrl.mediaType
675 extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
676 infoHash: parsed.infoHash,
677 resolution: fileUrl.height,
679 fps: fileUrl.fps || -1,
681 // This is a video file owned by a video or by a streaming playlist
682 videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,
683 videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
686 attributes.push(attribute)
692 function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObject: VideoTorrentObject, videoFiles: MVideoFile[]) {
693 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
694 if (playlistUrls.length === 0) return []
696 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
697 for (const playlistUrlObject of playlistUrls) {
698 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
700 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
702 // FIXME: backward compatibility introduced in v2.1.0
703 if (files.length === 0) files = videoFiles
705 if (!segmentsSha256UrlObject) {
706 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
711 type: VideoStreamingPlaylistType.HLS,
712 playlistUrl: playlistUrlObject.href,
713 segmentsSha256Url: segmentsSha256UrlObject.href,
714 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
715 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
717 tagAPObject: playlistUrlObject.tag
720 attributes.push(attribute)