From 49af5ac8c2653cb0ef23479c9d3256c5b724d49d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 3 Jun 2021 14:30:09 +0200 Subject: [PATCH] Refactor AP playlists --- server/lib/activitypub/cache-file.ts | 91 ++++---- server/lib/activitypub/playlist.ts | 204 ------------------ .../activitypub/playlists/create-update.ts | 146 +++++++++++++ server/lib/activitypub/playlists/index.ts | 2 + server/lib/activitypub/playlists/refresh.ts | 44 ++++ .../lib/activitypub/playlists/shared/index.ts | 2 + .../shared/object-to-model-attributes.ts | 40 ++++ .../playlists/shared/url-to-object.ts | 47 ++++ .../lib/activitypub/process/process-create.ts | 4 +- .../lib/activitypub/process/process-update.ts | 2 +- server/lib/activitypub/share.ts | 38 ++-- server/lib/activitypub/video-comments.ts | 33 ++- server/lib/activitypub/video-rates.ts | 56 ++--- .../handlers/activitypub-http-fetcher.ts | 2 +- .../handlers/activitypub-refresher.ts | 2 +- server/models/video/video-playlist.ts | 5 +- shared/models/activitypub/objects/index.ts | 4 + 17 files changed, 407 insertions(+), 315 deletions(-) delete mode 100644 server/lib/activitypub/playlist.ts create mode 100644 server/lib/activitypub/playlists/create-update.ts create mode 100644 server/lib/activitypub/playlists/index.ts create mode 100644 server/lib/activitypub/playlists/refresh.ts create mode 100644 server/lib/activitypub/playlists/shared/index.ts create mode 100644 server/lib/activitypub/playlists/shared/object-to-model-attributes.ts create mode 100644 server/lib/activitypub/playlists/shared/url-to-object.ts diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 2e6dd34e0..a16d2cd93 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts @@ -1,54 +1,27 @@ -import { CacheFileObject } from '../../../shared/index' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { Transaction } from 'sequelize' -import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' +import { CacheFileObject } from '../../../shared/index' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { - - if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { - const url = cacheFileObject.url - - const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) - if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) +async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { + const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoStreamingPlaylistId: playlist.id, - actorId: byActor.id - } + if (redundancyModel) { + return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) } - const url = cacheFileObject.url - const videoFile = video.VideoFiles.find(f => { - return f.resolution === url.height && f.fps === url.fps - }) - - if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) - - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoFileId: videoFile.id, - actorId: byActor.id - } + return createCacheFile(cacheFileObject, video, byActor, t) } -async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { - const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) +// --------------------------------------------------------------------------- - if (!redundancyModel) { - await createCacheFile(cacheFileObject, video, byActor, t) - } else { - await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) - } +export { + createOrUpdateCacheFile } +// --------------------------------------------------------------------------- + function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) @@ -74,9 +47,37 @@ function updateCacheFile ( return redundancyModel.save({ transaction: t }) } -export { - createOrUpdateCacheFile, - createCacheFile, - updateCacheFile, - cacheFileActivityObjectToDBAttributes +function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { + + if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { + const url = cacheFileObject.url + + const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) + if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) + + return { + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, + url: cacheFileObject.id, + fileUrl: url.href, + strategy: null, + videoStreamingPlaylistId: playlist.id, + actorId: byActor.id + } + } + + const url = cacheFileObject.url + const videoFile = video.VideoFiles.find(f => { + return f.resolution === url.height && f.fps === url.fps + }) + + if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) + + return { + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, + url: cacheFileObject.id, + fileUrl: url.href, + strategy: null, + videoFileId: videoFile.id, + actorId: byActor.id + } } diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts deleted file mode 100644 index 8fe6e79f2..000000000 --- a/server/lib/activitypub/playlist.ts +++ /dev/null @@ -1,204 +0,0 @@ -import * as Bluebird from 'bluebird' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' -import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' -import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' -import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { checkUrlsSameHost } from '../../helpers/activitypub' -import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' -import { isArray } from '../../helpers/custom-validators/misc' -import { logger } from '../../helpers/logger' -import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests' -import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { sequelizeTypescript } from '../../initializers/database' -import { VideoPlaylistModel } from '../../models/video/video-playlist' -import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' -import { MAccountDefault, MAccountId, MVideoId } from '../../types/models' -import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist' -import { FilteredModelAttributes } from '../../types/sequelize' -import { createPlaylistMiniatureFromUrl } from '../thumbnail' -import { getOrCreateActorAndServerAndModel } from './actor' -import { crawlCollectionPage } from './crawl' -import { getOrCreateAPVideo } from './videos' - -function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { - const privacy = to.includes(ACTIVITY_PUB.PUBLIC) - ? VideoPlaylistPrivacy.PUBLIC - : VideoPlaylistPrivacy.UNLISTED - - return { - name: playlistObject.name, - description: playlistObject.content, - privacy, - url: playlistObject.id, - uuid: playlistObject.uuid, - ownerAccountId: byAccount.id, - videoChannelId: null, - createdAt: new Date(playlistObject.published), - updatedAt: new Date(playlistObject.updated) - } -} - -function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { - return { - position: elementObject.position, - url: elementObject.id, - startTimestamp: elementObject.startTimestamp || null, - stopTimestamp: elementObject.stopTimestamp || null, - videoPlaylistId: videoPlaylist.id, - videoId: video.id - } -} - -async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) { - await Bluebird.map(playlistUrls, async playlistUrl => { - try { - const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) - if (exists === true) return - - // Fetch url - const { body } = await doJSONRequest(playlistUrl, { activityPub: true }) - - if (!isPlaylistObjectValid(body)) { - throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) - } - - if (!isArray(body.to)) { - throw new Error('Playlist does not have an audience.') - } - - return createOrUpdateVideoPlaylist(body, account, body.to) - } catch (err) { - logger.warn('Cannot add playlist element %s.', playlistUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { - const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to) - - if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) { - const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0]) - - if (actor.VideoChannel) { - playlistAttributes.videoChannelId = actor.VideoChannel.id - } else { - logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject }) - } - } - - const [ playlist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) - - let accItems: string[] = [] - await crawlCollectionPage(playlistObject.id, items => { - accItems = accItems.concat(items) - - return Promise.resolve() - }) - - const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null) - - if (playlistObject.icon) { - try { - const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist }) - await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined) - } catch (err) { - logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err }) - } - } else if (refreshedPlaylist.hasThumbnail()) { - await refreshedPlaylist.Thumbnail.destroy() - refreshedPlaylist.Thumbnail = null - } - - return resetVideoPlaylistElements(accItems, refreshedPlaylist) -} - -async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise { - if (!videoPlaylist.isOutdated()) return videoPlaylist - - try { - const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) - - if (playlistObject === undefined) { - logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) - - await videoPlaylist.setAsRefreshed() - return videoPlaylist - } - - const byAccount = videoPlaylist.OwnerAccount - await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) - - return videoPlaylist - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url) - - await videoPlaylist.destroy() - return undefined - } - - logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) - - await videoPlaylist.setAsRefreshed() - return videoPlaylist - } -} - -// --------------------------------------------------------------------------- - -export { - createAccountPlaylists, - playlistObjectToDBAttributes, - playlistElementObjectToDBAttributes, - createOrUpdateVideoPlaylist, - refreshVideoPlaylistIfNeeded -} - -// --------------------------------------------------------------------------- - -async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { - const elementsToCreate: FilteredModelAttributes[] = [] - - await Bluebird.map(elementUrls, async elementUrl => { - try { - const { body } = await doJSONRequest(elementUrl, { activityPub: true }) - - if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) - - if (checkUrlsSameHost(body.id, elementUrl) !== true) { - throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) - } - - const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' }) - - elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) - } catch (err) { - logger.warn('Cannot add playlist element %s.', elementUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) - - await sequelizeTypescript.transaction(async t => { - await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) - - for (const element of elementsToCreate) { - await VideoPlaylistElementModel.create(element, { transaction: t }) - } - }) - - logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length) - - return undefined -} - -async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { - logger.info('Fetching remote playlist %s.', playlistUrl) - - const { body, statusCode } = await doJSONRequest(playlistUrl, { activityPub: true }) - - if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { - logger.debug('Remote video playlist JSON is not valid.', { body }) - return { statusCode, playlistObject: undefined } - } - - return { statusCode, playlistObject: body } -} diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts new file mode 100644 index 000000000..886b1f288 --- /dev/null +++ b/server/lib/activitypub/playlists/create-update.ts @@ -0,0 +1,146 @@ +import { isArray } from '@server/helpers/custom-validators/misc' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' +import { sequelizeTypescript } from '@server/initializers/database' +import { createPlaylistMiniatureFromUrl } from '@server/lib/thumbnail' +import { VideoPlaylistModel } from '@server/models/video/video-playlist' +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' +import { FilteredModelAttributes } from '@server/types' +import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models' +import { AttributesOnly } from '@shared/core-utils' +import { PlaylistObject } from '@shared/models' +import { getOrCreateActorAndServerAndModel } from '../actor' +import { crawlCollectionPage } from '../crawl' +import { getOrCreateAPVideo } from '../videos' +import { + fetchRemotePlaylistElement, + fetchRemoteVideoPlaylist, + playlistElementObjectToDBAttributes, + playlistObjectToDBAttributes +} from './shared' + +import Bluebird = require('bluebird') + +const lTags = loggerTagsFactory('ap', 'video-playlist') + +async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) { + await Bluebird.map(playlistUrls, async playlistUrl => { + try { + const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) + if (exists === true) return + + const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) + + if (playlistObject === undefined) { + throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) + } + + return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to) + } catch (err) { + logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) +} + +async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { + const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to) + + await setVideoChannelIfNeeded(playlistObject, playlistAttributes) + + const [ upsertPlaylist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) + + const playlistElementUrls = await fetchElementUrls(playlistObject) + + // Refetch playlist from DB since elements fetching could be long in time + const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) + + try { + await updatePlaylistThumbnail(playlistObject, playlist) + } catch (err) { + logger.warn('Cannot update thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) + } + + return rebuildVideoPlaylistElements(playlistElementUrls, playlist) +} + +// --------------------------------------------------------------------------- + +export { + createAccountPlaylists, + createOrUpdateVideoPlaylist +} + +// --------------------------------------------------------------------------- + +async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { + if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return + + const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0]) + + if (!actor.VideoChannel) { + logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) + return + } + + playlistAttributes.videoChannelId = actor.VideoChannel.id +} + +async function fetchElementUrls (playlistObject: PlaylistObject) { + let accItems: string[] = [] + await crawlCollectionPage(playlistObject.id, items => { + accItems = accItems.concat(items) + + return Promise.resolve() + }) + + return accItems +} + +async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { + if (playlistObject.icon) { + const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) + await playlist.setAndSaveThumbnail(thumbnailModel, undefined) + + return + } + + // Playlist does not have an icon, destroy existing one + if (playlist.hasThumbnail()) { + await playlist.Thumbnail.destroy() + playlist.Thumbnail = null + } +} + +async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { + const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) + + await sequelizeTypescript.transaction(async t => { + await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) + + for (const element of elementsToCreate) { + await VideoPlaylistElementModel.create(element, { transaction: t }) + } + }) + + logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) + + return undefined +} + +async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { + const elementsToCreate: FilteredModelAttributes[] = [] + + await Bluebird.map(elementUrls, async elementUrl => { + try { + const { elementObject } = await fetchRemotePlaylistElement(elementUrl) + + const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) + + elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) + } catch (err) { + logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) + + return elementsToCreate +} diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts new file mode 100644 index 000000000..2885830b4 --- /dev/null +++ b/server/lib/activitypub/playlists/index.ts @@ -0,0 +1,2 @@ +export * from './create-update' +export * from './refresh' diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts new file mode 100644 index 000000000..ff9e5471a --- /dev/null +++ b/server/lib/activitypub/playlists/refresh.ts @@ -0,0 +1,44 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { PeerTubeRequestError } from '@server/helpers/requests' +import { MVideoPlaylistOwner } from '@server/types/models' +import { HttpStatusCode } from '@shared/core-utils' +import { createOrUpdateVideoPlaylist } from './create-update' +import { fetchRemoteVideoPlaylist } from './shared' + +async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise { + if (!videoPlaylist.isOutdated()) return videoPlaylist + + const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url) + + try { + const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) + + if (playlistObject === undefined) { + logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags()) + + await videoPlaylist.setAsRefreshed() + return videoPlaylist + } + + const byAccount = videoPlaylist.OwnerAccount + await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) + + return videoPlaylist + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags()) + + await videoPlaylist.destroy() + return undefined + } + + logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() }) + + await videoPlaylist.setAsRefreshed() + return videoPlaylist + } +} + +export { + refreshVideoPlaylistIfNeeded +} diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts new file mode 100644 index 000000000..a217f2291 --- /dev/null +++ b/server/lib/activitypub/playlists/shared/index.ts @@ -0,0 +1,2 @@ +export * from './object-to-model-attributes' +export * from './url-to-object' diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..6ec44485e --- /dev/null +++ b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts @@ -0,0 +1,40 @@ +import { ACTIVITY_PUB } from '@server/initializers/constants' +import { VideoPlaylistModel } from '@server/models/video/video-playlist' +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' +import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models' +import { AttributesOnly } from '@shared/core-utils' +import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' + +function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { + const privacy = to.includes(ACTIVITY_PUB.PUBLIC) + ? VideoPlaylistPrivacy.PUBLIC + : VideoPlaylistPrivacy.UNLISTED + + return { + name: playlistObject.name, + description: playlistObject.content, + privacy, + url: playlistObject.id, + uuid: playlistObject.uuid, + ownerAccountId: byAccount.id, + videoChannelId: null, + createdAt: new Date(playlistObject.published), + updatedAt: new Date(playlistObject.updated) + } as AttributesOnly +} + +function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { + return { + position: elementObject.position, + url: elementObject.id, + startTimestamp: elementObject.startTimestamp || null, + stopTimestamp: elementObject.stopTimestamp || null, + videoPlaylistId: videoPlaylist.id, + videoId: video.id + } as AttributesOnly +} + +export { + playlistObjectToDBAttributes, + playlistElementObjectToDBAttributes +} diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts new file mode 100644 index 000000000..ec8c01255 --- /dev/null +++ b/server/lib/activitypub/playlists/shared/url-to-object.ts @@ -0,0 +1,47 @@ +import { isArray } from 'lodash' +import { checkUrlsSameHost } from '@server/helpers/activitypub' +import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { doJSONRequest } from '@server/helpers/requests' +import { PlaylistElementObject, PlaylistObject } from '@shared/models' + +async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { + const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl) + + logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) + + const { body, statusCode } = await doJSONRequest(playlistUrl, { activityPub: true }) + + if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { + logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) + return { statusCode, playlistObject: undefined } + } + + if (!isArray(body.to)) { + logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() }) + return { statusCode, playlistObject: undefined } + } + + return { statusCode, playlistObject: body } +} + +async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> { + const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl) + + logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) + + const { body, statusCode } = await doJSONRequest(elementUrl, { activityPub: true }) + + if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) + + if (checkUrlsSameHost(body.id, elementUrl) !== true) { + throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) + } + + return { statusCode, elementObject: body } +} + +export { + fetchRemoteVideoPlaylist, + fetchRemotePlaylistElement +} diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index ef5a3100e..6b7f5aae8 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -1,3 +1,4 @@ +import { isBlockedByServerOrAccount } from '@server/lib/blocklist' import { isRedundancyAccepted } from '@server/lib/redundancy' import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared' import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' @@ -9,11 +10,10 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' import { Notifier } from '../../notifier' import { createOrUpdateCacheFile } from '../cache-file' -import { createOrUpdateVideoPlaylist } from '../playlist' +import { createOrUpdateVideoPlaylist } from '../playlists' import { forwardVideoRelatedActivity } from '../send/utils' import { resolveThread } from '../video-comments' import { getOrCreateAPVideo } from '../videos' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' async function processCreateActivity (options: APProcessorOptions) { const { activity, byActor } = options diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index be3f6acac..d2b63c901 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -15,7 +15,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature } from '../../../types/models' import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor' import { createOrUpdateCacheFile } from '../cache-file' -import { createOrUpdateVideoPlaylist } from '../playlist' +import { createOrUpdateVideoPlaylist } from '../playlists' import { forwardVideoRelatedActivity } from '../send/utils' import { APVideoUpdater, getOrCreateAPVideo } from '../videos' diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index c22fa0893..327955dd2 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -40,23 +40,7 @@ async function changeVideoChannelShare ( async function addVideoShares (shareUrls: string[], video: MVideoId) { await Bluebird.map(shareUrls, async shareUrl => { try { - const { body } = await doJSONRequest(shareUrl, { activityPub: true }) - if (!body || !body.actor) throw new Error('Body or body actor is invalid') - - const actorUrl = getAPId(body.actor) - if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { - throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) - } - - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - - const entry = { - actorId: actor.id, - videoId: video.id, - url: shareUrl - } - - await VideoShareModel.upsert(entry) + await addVideoShare(shareUrl, video) } catch (err) { logger.warn('Cannot add share %s.', shareUrl, { err }) } @@ -71,6 +55,26 @@ export { // --------------------------------------------------------------------------- +async function addVideoShare (shareUrl: string, video: MVideoId) { + const { body } = await doJSONRequest(shareUrl, { activityPub: true }) + if (!body || !body.actor) throw new Error('Body or body actor is invalid') + + const actorUrl = getAPId(body.actor) + if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { + throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) + } + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + + const entry = { + actorId: actor.id, + videoId: video.id, + url: shareUrl + } + + await VideoShareModel.upsert(entry) +} + async function shareByServer (video: MVideo, t: Transaction) { const serverActor = await getServerActor() diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 722147b69..760da719d 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -29,10 +29,11 @@ async function addVideoComments (commentUrls: string[]) { async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { const { url, isVideo } = params + if (params.commentCreated === undefined) params.commentCreated = false if (params.comments === undefined) params.comments = [] - // If it is not a video, or if we don't know if it's a video + // If it is not a video, or if we don't know if it's a video, try to get the thread from DB if (isVideo === false || isVideo === undefined) { const result = await resolveCommentFromDB(params) if (result) return result @@ -42,7 +43,7 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult // If it is a video, or if we don't know if it's a video if (isVideo === true || isVideo === undefined) { // Keep await so we catch the exception - return await tryResolveThreadFromVideo(params) + return await tryToResolveThreadFromVideo(params) } } catch (err) { logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) @@ -62,28 +63,26 @@ async function resolveCommentFromDB (params: ResolveThreadParams) { const { url, comments, commentCreated } = params const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) - if (commentFromDatabase) { - let parentComments = comments.concat([ commentFromDatabase ]) + if (!commentFromDatabase) return undefined - // Speed up things and resolve directly the thread - if (commentFromDatabase.InReplyToVideoComment) { - const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') + let parentComments = comments.concat([ commentFromDatabase ]) - parentComments = parentComments.concat(data) - } + // Speed up things and resolve directly the thread + if (commentFromDatabase.InReplyToVideoComment) { + const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') - return resolveThread({ - url: commentFromDatabase.Video.url, - comments: parentComments, - isVideo: true, - commentCreated - }) + parentComments = parentComments.concat(data) } - return undefined + return resolveThread({ + url: commentFromDatabase.Video.url, + comments: parentComments, + isVideo: true, + commentCreated + }) } -async function tryResolveThreadFromVideo (params: ResolveThreadParams) { +async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { const { url, comments, commentCreated } = params // Maybe it's a reply to a video? diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index f40c07fea..091f4ec23 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -15,30 +15,7 @@ import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlBy async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { await Bluebird.map(ratesUrl, async rateUrl => { try { - // Fetch url - const { body } = await doJSONRequest(rateUrl, { activityPub: true }) - if (!body || !body.actor) throw new Error('Body or body actor is invalid') - - const actorUrl = getAPId(body.actor) - if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { - throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) - } - - if (checkUrlsSameHost(body.id, rateUrl) !== true) { - throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) - } - - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - - const entry = { - videoId: video.id, - accountId: actor.Account.id, - type: rate, - url: body.id - } - - // Video "likes"/"dislikes" will be updated by the caller - await AccountVideoRateModel.upsert(entry) + await createRate(rateUrl, video, rate) } catch (err) { logger.warn('Cannot add rate %s.', rateUrl, { err }) } @@ -73,8 +50,39 @@ function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVid : getVideoDislikeActivityPubUrlByLocalActor(actor, video) } +// --------------------------------------------------------------------------- + export { getLocalRateUrl, createRates, sendVideoRateChange } + +// --------------------------------------------------------------------------- + +async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) { + // Fetch url + const { body } = await doJSONRequest(rateUrl, { activityPub: true }) + if (!body || !body.actor) throw new Error('Body or body actor is invalid') + + const actorUrl = getAPId(body.actor) + if (checkUrlsSameHost(actorUrl, rateUrl) !== true) { + throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`) + } + + if (checkUrlsSameHost(body.id, rateUrl) !== true) { + throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) + } + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + + const entry = { + videoId: video.id, + accountId: actor.Account.id, + type: rate, + url: body.id + } + + // Video "likes"/"dislikes" will be updated by the caller + await AccountVideoRateModel.upsert(entry) +} diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index e210ac3ef..04b25f955 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -8,7 +8,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment' import { VideoShareModel } from '../../../models/video/video-share' import { MAccountDefault, MVideoFullLight } from '../../../types/models' import { crawlCollectionPage } from '../../activitypub/crawl' -import { createAccountPlaylists } from '../../activitypub/playlist' +import { createAccountPlaylists } from '../../activitypub/playlists' import { processActivities } from '../../activitypub/process' import { addVideoShares } from '../../activitypub/share' import { addVideoComments } from '../../activitypub/video-comments' diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index a120e4ea8..10e6895da 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -1,5 +1,5 @@ import * as Bull from 'bull' -import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist' +import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists' import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos' import { RefreshPayload } from '@shared/models' import { logger } from '../../../helpers/logger' diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 98cea1b64..1a05f8d42 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -18,6 +18,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { v4 as uuidv4 } from 'uuid' +import { setAsUpdated } from '@server/helpers/database-utils' import { MAccountId, MChannelId } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { ActivityIconObject } from '../../../shared/models/activitypub/objects' @@ -531,9 +532,7 @@ export class VideoPlaylistModel extends Model