X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Factivitypub%2Fvideos.ts;h=6d20e0e654437c5ffa7f186978ea332b0fdcee89;hb=5fb2e2888ce032c638e4b75d07458642f0833e52;hp=9e43caa204d26d3d10345c1196a191579a9a2240;hpb=a15871560f80e07386c1dabb8370cd2664ecfd1f;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 9e43caa20..6d20e0e65 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -6,7 +6,7 @@ import { ActivityHashTagObject, ActivityMagnetUrlObject, ActivityPlaylistSegmentHashesObject, - ActivityPlaylistUrlObject, + ActivityPlaylistUrlObject, ActivitypubHttpFetcherPayload, ActivityTagObject, ActivityUrlObject, ActivityVideoUrlObject, @@ -14,7 +14,7 @@ import { } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' -import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' +import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' @@ -25,7 +25,8 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, REMOTE_SCHEME, - STATIC_PATHS, THUMBNAILS_SIZE + STATIC_PATHS, + THUMBNAILS_SIZE } from '../../initializers/constants' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' @@ -37,7 +38,6 @@ import { sendCreateVideo, sendUpdateVideo } from './send' import { isArray } from '../../helpers/custom-validators/misc' import { VideoCaptionModel } from '../../models/video/video-caption' import { JobQueue } from '../job-queue' -import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' @@ -69,6 +69,7 @@ import { MVideoFile, MVideoFullLight, MVideoId, + MVideoImmutable, MVideoThumbnail } from '../../typings/models' import { MThumbnail } from '../../typings/models/video/thumbnail' @@ -111,7 +112,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request. logger.info('Fetching remote video %s.', videoUrl) - const { response, body } = await doRequest(options) + const { response, body } = await doRequest(options) if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { logger.debug('Remote video JSON is not valid.', { body }) @@ -129,7 +130,7 @@ async function fetchRemoteVideoDescription (video: MVideoAccountLight) { json: true } - const { body } = await doRequest(options) + const { body } = await doRequest(options) return body.description ? body.description : '' } @@ -200,24 +201,41 @@ async function syncVideoExternalAttributes (video: MVideo, fetchedVideo: VideoTo await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload })) } -function getOrCreateVideoAndAccountAndChannel (options: { +type GetVideoResult = Promise<{ + video: T + created: boolean + autoBlacklisted?: boolean +}> + +type GetVideoParamAll = { videoObject: { id: string } | string syncParam?: SyncParam fetchType?: 'all' allowRefresh?: boolean -}): Promise<{ video: MVideoAccountLightBlacklistAllFiles, created: boolean, autoBlacklisted?: boolean }> -function getOrCreateVideoAndAccountAndChannel (options: { +} + +type GetVideoParamImmutable = { videoObject: { id: string } | string syncParam?: SyncParam - fetchType?: VideoFetchByUrlType - allowRefresh?: boolean -}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> -async function getOrCreateVideoAndAccountAndChannel (options: { + fetchType: 'only-immutable-attributes' + allowRefresh: false +} + +type GetVideoParamOther = { videoObject: { id: string } | string syncParam?: SyncParam - fetchType?: VideoFetchByUrlType - allowRefresh?: boolean // true by default -}): Promise<{ video: MVideoAccountLightBlacklistAllFiles | MVideoThumbnail, created: boolean, autoBlacklisted?: boolean }> { + fetchType?: 'all' | 'only-video' + allowRefresh?: boolean +} + +function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult +function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult +function getOrCreateVideoAndAccountAndChannel ( + options: GetVideoParamOther +): GetVideoResult +async function getOrCreateVideoAndAccountAndChannel ( + options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther +): GetVideoResult { // Default params const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } const fetchType = options.fetchType || 'all' @@ -225,12 +243,13 @@ async function getOrCreateVideoAndAccountAndChannel (options: { // Get video url const videoUrl = getAPId(options.videoObject) - let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) + if (videoFromDatabase) { - if (videoFromDatabase.isOutdated() && allowRefresh === true) { + // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type + if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) { const refreshOptions = { - video: videoFromDatabase, + video: videoFromDatabase as MVideoThumbnail, fetchedType: fetchType, syncParam } @@ -253,11 +272,22 @@ async function getOrCreateVideoAndAccountAndChannel (options: { const actor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) const videoChannel = actor.VideoChannel - const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) - await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) + try { + const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(createVideo, fetchedVideo, videoChannel, syncParam.thumbnail) + + await syncVideoExternalAttributes(videoCreated, fetchedVideo, syncParam) + + return { video: videoCreated, created: true, autoBlacklisted } + } catch (err) { + // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video + if (err.name === 'SequelizeUniqueConstraintError') { + const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType) + if (fallbackVideo) return { video: fallbackVideo, created: false } + } - return { video: videoCreated, created: true, autoBlacklisted } + throw err + } } async function updateVideoFromAP (options: { @@ -322,9 +352,11 @@ async function updateVideoFromAP (options: { if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) - const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated) - const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) - await videoUpdated.addAndSaveThumbnail(previewModel, t) + if (videoUpdated.getPreview()) { + const previewUrl = videoUpdated.getPreview().getFileUrl(videoUpdated) + const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) + await videoUpdated.addAndSaveThumbnail(previewModel, t) + } { const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url) @@ -487,7 +519,7 @@ function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) const urlMediaType = url.mediaType - return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') + return mimeTypes.includes(urlMediaType) && urlMediaType.startsWith('video/') } function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { @@ -513,6 +545,10 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc const video = VideoModel.build(videoData) as MVideoThumbnail const promiseThumbnail = createVideoMiniatureFromUrl(getThumbnailFromIcons(videoObject).url, video, ThumbnailType.MINIATURE) + .catch(err => { + logger.error('Cannot create miniature from url.', { err }) + return undefined + }) let thumbnailModel: MThumbnail if (waitThumbnail === true) { @@ -584,34 +620,35 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc }) if (waitThumbnail === false) { + // Error is already caught above + // eslint-disable-next-line @typescript-eslint/no-floating-promises promiseThumbnail.then(thumbnailModel => { + if (!thumbnailModel) return + thumbnailModel = videoCreated.id return thumbnailModel.save() - }).catch(err => logger.error('Cannot create miniature from url.', { err })) + }) } return { autoBlacklisted, videoCreated } } function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObject: VideoTorrentObject, to: string[] = []) { - const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED - const duration = videoObject.duration.replace(/[^\d]+/, '') + const privacy = to.includes(ACTIVITY_PUB.PUBLIC) + ? VideoPrivacy.PUBLIC + : VideoPrivacy.UNLISTED - let language: string | undefined - if (videoObject.language) { - language = videoObject.language.identifier - } + const duration = videoObject.duration.replace(/[^\d]+/, '') + const language = videoObject.language?.identifier - let category: number | undefined - if (videoObject.category) { - category = parseInt(videoObject.category.identifier, 10) - } + const category = videoObject.category + ? parseInt(videoObject.category.identifier, 10) + : undefined - let licence: number | undefined - if (videoObject.licence) { - licence = parseInt(videoObject.licence.identifier, 10) - } + const licence = videoObject.licence + ? parseInt(videoObject.licence.identifier, 10) + : undefined const description = videoObject.content || null const support = videoObject.support || null @@ -634,7 +671,11 @@ function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObjec duration: parseInt(duration, 10), createdAt: new Date(videoObject.published), publishedAt: new Date(videoObject.published), - originallyPublishedAt: videoObject.originallyPublishedAt ? new Date(videoObject.originallyPublishedAt) : null, + + originallyPublishedAt: videoObject.originallyPublishedAt + ? new Date(videoObject.originallyPublishedAt) + : null, + updatedAt: new Date(videoObject.updated), views: videoObject.views, likes: 0, @@ -665,6 +706,14 @@ function videoFileActivityUrlToDBAttributes ( throw new Error('Cannot parse magnet URI ' + magnet.href) } + // Fetch associated metadata url, if any + const metadata = urls.filter(isAPVideoFileMetadataObject) + .find(u => { + return u.height === fileUrl.height && + u.fps === fileUrl.fps && + u.rel.includes(fileUrl.mediaType) + }) + const mediaType = fileUrl.mediaType const attribute = { extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType], @@ -672,6 +721,7 @@ function videoFileActivityUrlToDBAttributes ( resolution: fileUrl.height, size: fileUrl.size, fps: fileUrl.fps || -1, + metadataUrl: metadata?.href, // This is a video file owned by a video or by a streaming playlist videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,