X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Factivitypub%2Fvideos.ts;h=4cecf9345912a6d08eb59b73d13a6de183b20036;hb=361805c48b14c5402c9984485c67c45a1a3113cc;hp=48c0e0a5ca9c7217e19713daeb55b1531125ef0e;hpb=0491173a61aed66205c017e0d7e0503ea316c144;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 48c0e0a5c..4cecf9345 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -10,8 +10,8 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validat import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' -import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' +import { doRequest, downloadImage } from '../../helpers/requests' +import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_MIMETYPE_EXT } from '../../initializers' import { ActorModel } from '../../models/activitypub/actor' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' @@ -29,6 +29,7 @@ import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' +import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -63,7 +64,7 @@ async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request. const { response, body } = await doRequest(options) - if (sanitizeAndCheckVideoTorrentObject(body) === false) { + if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { logger.debug('Remote video JSON is not valid.', { body }) return { response, videoObject: undefined } } @@ -96,17 +97,17 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) const thumbnailName = video.getThumbnailName() const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) - const options = { - method: 'GET', - uri: icon.url - } - return doRequestAndSaveToFile(options, thumbnailPath) + return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE) } function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { const channel = videoObject.attributedTo.find(a => a.type === 'Group') if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) + if (checkUrlsSameHost(channel.id, videoObject.id) !== true) { + throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${videoObject.id}`) + } + return getOrCreateActorAndServerAndModel(channel.id, 'all') } @@ -166,7 +167,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { const refreshViews = options.refreshViews || false // Get video url - const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id + const videoUrl = getAPUrl(options.videoObject) let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) if (videoFromDatabase) { @@ -176,7 +177,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { syncParam, refreshViews } - const p = retryTransactionWrapper(refreshVideoIfNeeded, refreshOptions) + const p = refreshVideoIfNeeded(refreshOptions) if (syncParam.refreshVideo === true) videoFromDatabase = await p return { video: videoFromDatabase } @@ -205,7 +206,7 @@ async function updateVideoFromAP (options: { let videoFieldsSave: any try { - const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => { + await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } @@ -241,38 +242,44 @@ async function updateVideoFromAP (options: { if (options.updateViews === true) options.video.set('views', videoData.views) await options.video.save(sequelizeOptions) - // Don't block on request - generateThumbnailFromUrl(options.video, options.videoObject.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })) + { + const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) + const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) - // Remove old video files - const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of options.video.VideoFiles) { - videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) - } - await Promise.all(videoFileDestroyTasks) + // Remove video files that do not exist anymore + const destroyTasks = options.video.VideoFiles + .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f))) + .map(f => f.destroy(sequelizeOptions)) + await Promise.all(destroyTasks) + + // Update or add other one + const upsertTasks = videoFileAttributes.map(a => { + return VideoFileModel.upsert(a, { returning: true, transaction: t }) + .then(([ file ]) => file) + }) - const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) - const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) - await Promise.all(tasks) + options.video.VideoFiles = await Promise.all(upsertTasks) + } - // Update Tags - const tags = options.videoObject.tag.map(tag => tag.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await options.video.$set('Tags', tagInstances, sequelizeOptions) + { + // Update Tags + const tags = options.videoObject.tag.map(tag => tag.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await options.video.$set('Tags', tagInstances, sequelizeOptions) + } - // Update captions - await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) + { + // Update captions + await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(options.video.id, t) - const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { - return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) - }) - await Promise.all(videoCaptionsPromises) + const videoCaptionsPromises = options.videoObject.subtitleLanguage.map(c => { + return VideoCaptionModel.insertOrReplaceLanguage(options.video.id, c.identifier, t) + }) + options.video.VideoCaptions = await Promise.all(videoCaptionsPromises) + } }) logger.info('Remote video with uuid %s updated', options.videoObject.uuid) - - return updatedVideo } catch (err) { if (options.video !== undefined && videoFieldsSave !== undefined) { resetSequelizeInstance(options.video, videoFieldsSave) @@ -282,6 +289,12 @@ async function updateVideoFromAP (options: { logger.debug('Cannot update the remote video.', { err }) throw err } + + try { + await generateThumbnailFromUrl(options.video, options.videoObject.icon) + } catch (err) { + logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }) + } } export { @@ -300,7 +313,8 @@ export { function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) - return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') + const urlMediaType = url.mediaType || url.mimeType + return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') } async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { @@ -362,13 +376,15 @@ async function refreshVideoIfNeeded (options: { try { const { response, videoObject } = await fetchRemoteVideo(video.url) if (response.statusCode === 404) { + logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) + // Video does not exist anymore await video.destroy() return undefined } if (videoObject === undefined) { - logger.warn('Cannot refresh remote video: invalid body.') + logger.warn('Cannot refresh remote video %s: invalid body.', video.url) return video } @@ -382,10 +398,12 @@ async function refreshVideoIfNeeded (options: { channel: channelActor.VideoChannel, updateViews: options.refreshViews } - await updateVideoFromAP(updateOptions) + await retryTransactionWrapper(updateVideoFromAP, updateOptions) await syncVideoExternalAttributes(video, videoObject, options.syncParam) + + return video } catch (err) { - logger.warn('Cannot refresh video.', { err }) + logger.warn('Cannot refresh video %s.', options.video.url, { err }) return video } } @@ -443,18 +461,19 @@ async function videoActivityObjectToDBAttributes ( } } -function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { +function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] if (fileUrls.length === 0) { - throw new Error('Cannot find video files for ' + videoCreated.url) + throw new Error('Cannot find video files for ' + video.url) } const attributes: VideoFileModel[] = [] for (const fileUrl of fileUrls) { // Fetch associated magnet uri const magnet = videoObject.url.find(u => { - return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height + const mediaType = u.mediaType || u.mimeType + return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height }) if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) @@ -464,13 +483,14 @@ function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObje throw new Error('Cannot parse magnet URI ' + magnet.href) } + const mediaType = fileUrl.mediaType || fileUrl.mimeType const attribute = { - extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], + extname: VIDEO_MIMETYPE_EXT[ mediaType ], infoHash: parsed.infoHash, resolution: fileUrl.height, size: fileUrl.size, - videoId: videoCreated.id, - fps: fileUrl.fps + videoId: video.id, + fps: fileUrl.fps || -1 } as VideoFileModel attributes.push(attribute) }