X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Factivitypub%2Fvideos.ts;h=9e43caa204d26d3d10345c1196a191579a9a2240;hb=7eba5e1fa81c8e54cb8fe298a96e8070afa50921;hp=7a9d5168bd994d2c585c26a1c1ae943df2bc9736;hpb=cf59a2a0c367683ba35758419499bf6087c192ec;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 7a9d5168b..9e43caa20 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -6,7 +6,8 @@ import { ActivityHashTagObject, ActivityMagnetUrlObject, ActivityPlaylistSegmentHashesObject, - ActivityPlaylistUrlObject, ActivityTagObject, + ActivityPlaylistUrlObject, + ActivityTagObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState @@ -17,14 +18,14 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validat import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' -import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' +import { doRequest } from '../../helpers/requests' import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, REMOTE_SCHEME, - STATIC_PATHS + STATIC_PATHS, THUMBNAILS_SIZE } from '../../initializers/constants' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' @@ -40,7 +41,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' -import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' +import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { Notifier } from '../notifier' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' @@ -71,6 +72,7 @@ import { MVideoThumbnail } from '../../typings/models' import { MThumbnail } from '../../typings/models/video/thumbnail' +import { maxBy, minBy } from 'lodash' async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) { const video = videoArg as MVideoAP @@ -131,19 +133,6 @@ async function fetchRemoteVideoDescription (video: MVideoAccountLight) { return body.description ? body.description : '' } -function fetchRemoteVideoStaticFile (video: MVideoAccountLight, path: string, destPath: string) { - const url = buildRemoteBaseUrl(video, path) - - // We need to provide a callback, if no we could have an uncaught exception - return doRequestAndSaveToFile({ uri: url }, destPath) -} - -function buildRemoteBaseUrl (video: MVideoAccountLight, path: string) { - const host = video.VideoChannel.Account.Actor.Server.host - - return REMOTE_SCHEME.HTTP + '://' + host + path -} - function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { const channel = videoObject.attributedTo.find(a => a.type === 'Group') if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) @@ -173,7 +162,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) await crawlCollectionPage(fetchedVideo.likes, handler, cleaner) - .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) + .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.likes })) } else { jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) } @@ -183,7 +172,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) await crawlCollectionPage(fetchedVideo.dislikes, handler, cleaner) - .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) + .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err, rootUrl: fetchedVideo.dislikes })) } else { jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) } @@ -193,7 +182,7 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) await crawlCollectionPage(fetchedVideo.shares, handler, cleaner) - .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) + .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares })) } else { jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) } @@ -203,30 +192,30 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) await crawlCollectionPage(fetchedVideo.comments, handler, cleaner) - .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) + .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments })) } else { jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) } - await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) + await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })) } function getOrCreateVideoAndAccountAndChannel (options: { - videoObject: { id: string } | string, - syncParam?: SyncParam, - fetchType?: 'all', + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: 'all' allowRefresh?: boolean }): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }> function getOrCreateVideoAndAccountAndChannel (options: { - videoObject: { id: string } | string, - syncParam?: SyncParam, - fetchType?: VideoFetchByUrlType, + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: VideoFetchByUrlType allowRefresh?: boolean }): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> async function getOrCreateVideoAndAccountAndChannel (options: { - videoObject: { id: string } | string, - syncParam?: SyncParam, - fetchType?: VideoFetchByUrlType, + videoObject: { id: string } | string + syncParam?: SyncParam + fetchType?: VideoFetchByUrlType allowRefresh?: boolean // true by default }): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> { // Default params @@ -246,8 +235,14 @@ async function getOrCreateVideoAndAccountAndChannel (options: { syncParam } - if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) + if (syncParam.refreshVideo === true) { + videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) + } else { + await JobQueue.Instance.createJobWithPromise({ + type: 'activitypub-refresher', + payload: { type: 'video', url: videoFromDatabase.url } + }) + } } return { video: videoFromDatabase, created: false } @@ -266,10 +261,10 @@ async function getOrCreateVideoAndAccountAndChannel (options: { } async function updateVideoFromAP (options: { - video: MVideoAccountLightBlacklistAllFiles, - videoObject: VideoTorrentObject, - account: MAccountIdActor, - channel: MChannelDefault, + video: MVideoAccountLightBlacklistAllFiles + videoObject: VideoTorrentObject + account: MAccountIdActor + channel: MChannelDefault overrideTo?: string[] }) { const { video, videoObject, account, channel, overrideTo } = options @@ -284,7 +279,7 @@ async function updateVideoFromAP (options: { let thumbnailModel: MThumbnail try { - thumbnailModel = await createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) + thumbnailModel = await createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE) } catch (err) { logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }) } @@ -300,7 +295,7 @@ async function updateVideoFromAP (options: { throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) } - const to = overrideTo ? overrideTo : videoObject.to + const to = overrideTo || videoObject.to const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) video.name = videoData.name video.uuid = videoData.uuid @@ -327,8 +322,7 @@ async function updateVideoFromAP (options: { if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) - // FIXME: use icon URL instead - const previewUrl = buildRemoteBaseUrl(videoUpdated, join(STATIC_PATHS.PREVIEWS, videoUpdated.getPreview().filename)) + const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated) const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) await videoUpdated.addAndSaveThumbnail(previewModel, t) @@ -391,7 +385,7 @@ async function updateVideoFromAP (options: { await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t) const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, t) + return VideoCaptionModel.insertOrReplaceLanguage(videoUpdated.id, c.identifier, c.url, t) }) await Promise.all(videoCaptionsPromises) } @@ -424,8 +418,8 @@ async function updateVideoFromAP (options: { } async function refreshVideoIfNeeded (options: { - video: MVideoThumbnail, - fetchedType: VideoFetchByUrlType, + video: MVideoThumbnail + fetchedType: VideoFetchByUrlType syncParam: SyncParam }): Promise { if (!options.video.isOutdated()) return options.video @@ -483,7 +477,6 @@ export { federateVideoIfNeeded, fetchRemoteVideo, getOrCreateVideoAndAccountAndChannel, - fetchRemoteVideoStaticFile, fetchRemoteVideoDescription, getOrCreateVideoChannelFromVideoObject } @@ -519,7 +512,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, videoObject.to) const video = VideoModel.build(videoData) as MVideoThumbnail - const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) + const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE) let thumbnailModel: MThumbnail if (waitThumbnail === true) { @@ -534,9 +527,12 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - // FIXME: use icon URL instead - const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) - const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) + const previewIcon = getPreviewFromIcons(videoObject) + const previewUrl = previewIcon + ? previewIcon.url + : buildRemoteVideoBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) + const previewModel = createPlaceholderThumbnail(previewUrl, videoCreated, ThumbnailType.PREVIEW, PREVIEWS_SIZE) + if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) // Process files @@ -567,7 +563,7 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc // Process captions const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) + return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, c.url, t) }) await Promise.all(videoCaptionsPromises) @@ -592,13 +588,13 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc thumbnailModel = videoCreated.id return thumbnailModel.save() - }) + }).catch(err => logger.error('Cannot create miniature from url.', { err })) } return { autoBlacklisted, videoCreated } } -async function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) { +function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) { const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED const duration = videoObject.duration.replace(/[^\d]+/, '') @@ -671,7 +667,7 @@ function videoFileActivityUrlToDBAttributes ( const mediaType = fileUrl.mediaType const attribute = { - extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ], + extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType], infoHash: parsed.infoHash, resolution: fileUrl.height, size: fileUrl.size, @@ -721,3 +717,19 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec return attributes } + +function getThumbnailFromIcons (videoObject: VideoTorrentObject) { + let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) + // Fallback if there are not valid icons + if (validIcons.length === 0) validIcons = videoObject.icon + + return minBy(validIcons, 'width') +} + +function getPreviewFromIcons (videoObject: VideoTorrentObject) { + const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) + + // FIXME: don't put a fallback here for compatibility with PeerTube <2.2 + + return maxBy(validIcons, 'width') +}