aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/controllers/api/search.ts4
-rw-r--r--server/controllers/api/videos/index.ts17
-rw-r--r--server/lib/activitypub/playlist.ts4
-rw-r--r--server/lib/activitypub/process/process-announce.ts4
-rw-r--r--server/lib/activitypub/process/process-create.ts6
-rw-r--r--server/lib/activitypub/process/process-dislike.ts4
-rw-r--r--server/lib/activitypub/process/process-like.ts4
-rw-r--r--server/lib/activitypub/process/process-undo.ts8
-rw-r--r--server/lib/activitypub/process/process-update.ts6
-rw-r--r--server/lib/activitypub/process/process-view.ts4
-rw-r--r--server/lib/activitypub/video-comments.ts4
-rw-r--r--server/lib/activitypub/videos/fetch.ts180
-rw-r--r--server/lib/activitypub/videos/get.ts109
-rw-r--r--server/lib/activitypub/videos/index.ts3
-rw-r--r--server/lib/activitypub/videos/refresh.ts64
-rw-r--r--server/lib/activitypub/videos/shared/index.ts1
-rw-r--r--server/lib/activitypub/videos/shared/url-to-object.ts22
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts4
-rw-r--r--server/models/utils.ts4
19 files changed, 243 insertions, 209 deletions
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts
index 77e3a024d..0cb5674c2 100644
--- a/server/controllers/api/search.ts
+++ b/server/controllers/api/search.ts
@@ -2,7 +2,7 @@ import * as express from 'express'
2import { sanitizeUrl } from '@server/helpers/core-utils' 2import { sanitizeUrl } from '@server/helpers/core-utils'
3import { doJSONRequest } from '@server/helpers/requests' 3import { doJSONRequest } from '@server/helpers/requests'
4import { CONFIG } from '@server/initializers/config' 4import { CONFIG } from '@server/initializers/config'
5import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' 5import { getOrCreateAPVideo } from '@server/lib/activitypub/videos'
6import { Hooks } from '@server/lib/plugins/hooks' 6import { Hooks } from '@server/lib/plugins/hooks'
7import { AccountBlocklistModel } from '@server/models/account/account-blocklist' 7import { AccountBlocklistModel } from '@server/models/account/account-blocklist'
8import { getServerActor } from '@server/models/application/application' 8import { getServerActor } from '@server/models/application/application'
@@ -244,7 +244,7 @@ async function searchVideoURI (url: string, res: express.Response) {
244 refreshVideo: false 244 refreshVideo: false
245 } 245 }
246 246
247 const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) 247 const result = await getOrCreateAPVideo({ videoObject: url, syncParam })
248 video = result ? result.video : undefined 248 video = result ? result.video : undefined
249 } catch (err) { 249 } catch (err) {
250 logger.info('Cannot search remote video %s.', url, { err }) 250 logger.info('Cannot search remote video %s.', url, { err })
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 47ab098ef..db23e5630 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -1,17 +1,18 @@
1import * as express from 'express' 1import * as express from 'express'
2import toInt from 'validator/lib/toInt' 2import toInt from 'validator/lib/toInt'
3import { doJSONRequest } from '@server/helpers/requests'
3import { LiveManager } from '@server/lib/live-manager' 4import { LiveManager } from '@server/lib/live-manager'
4import { getServerActor } from '@server/models/application/application' 5import { getServerActor } from '@server/models/application/application'
6import { MVideoAccountLight } from '@server/types/models'
5import { VideosCommonQuery } from '../../../../shared' 7import { VideosCommonQuery } from '../../../../shared'
6import { HttpStatusCode } from '../../../../shared/core-utils/miscs' 8import { HttpStatusCode } from '../../../../shared/core-utils/miscs'
7import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' 9import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
8import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils' 10import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils'
9import { logger } from '../../../helpers/logger' 11import { logger } from '../../../helpers/logger'
10import { getFormattedObjects } from '../../../helpers/utils' 12import { getFormattedObjects } from '../../../helpers/utils'
11import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' 13import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants'
12import { sequelizeTypescript } from '../../../initializers/database' 14import { sequelizeTypescript } from '../../../initializers/database'
13import { sendView } from '../../../lib/activitypub/send/send-view' 15import { sendView } from '../../../lib/activitypub/send/send-view'
14import { fetchRemoteVideoDescription } from '../../../lib/activitypub/videos'
15import { JobQueue } from '../../../lib/job-queue' 16import { JobQueue } from '../../../lib/job-queue'
16import { Hooks } from '../../../lib/plugins/hooks' 17import { Hooks } from '../../../lib/plugins/hooks'
17import { Redis } from '../../../lib/redis' 18import { Redis } from '../../../lib/redis'
@@ -245,3 +246,15 @@ async function removeVideo (_req: express.Request, res: express.Response) {
245 .status(HttpStatusCode.NO_CONTENT_204) 246 .status(HttpStatusCode.NO_CONTENT_204)
246 .end() 247 .end()
247} 248}
249
250// ---------------------------------------------------------------------------
251
252// FIXME: Should not exist, we rely on specific API
253async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
254 const host = video.VideoChannel.Account.Actor.Server.host
255 const path = video.getDescriptionAPIPath()
256 const url = REMOTE_SCHEME.HTTP + '://' + host + path
257
258 const { body } = await doJSONRequest<any>(url)
259 return body.description || ''
260}
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
index 7166c68a6..8fe6e79f2 100644
--- a/server/lib/activitypub/playlist.ts
+++ b/server/lib/activitypub/playlist.ts
@@ -18,7 +18,7 @@ import { FilteredModelAttributes } from '../../types/sequelize'
18import { createPlaylistMiniatureFromUrl } from '../thumbnail' 18import { createPlaylistMiniatureFromUrl } from '../thumbnail'
19import { getOrCreateActorAndServerAndModel } from './actor' 19import { getOrCreateActorAndServerAndModel } from './actor'
20import { crawlCollectionPage } from './crawl' 20import { crawlCollectionPage } from './crawl'
21import { getOrCreateVideoAndAccountAndChannel } from './videos' 21import { getOrCreateAPVideo } from './videos'
22 22
23function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { 23function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
24 const privacy = to.includes(ACTIVITY_PUB.PUBLIC) 24 const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
@@ -169,7 +169,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid
169 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) 169 throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
170 } 170 }
171 171
172 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' }) 172 const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' })
173 173
174 elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video)) 174 elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
175 } catch (err) { 175 } catch (err) {
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts
index 63082466e..ec23c705e 100644
--- a/server/lib/activitypub/process/process-announce.ts
+++ b/server/lib/activitypub/process/process-announce.ts
@@ -3,7 +3,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
3import { sequelizeTypescript } from '../../../initializers/database' 3import { sequelizeTypescript } from '../../../initializers/database'
4import { VideoShareModel } from '../../../models/video/video-share' 4import { VideoShareModel } from '../../../models/video/video-share'
5import { forwardVideoRelatedActivity } from '../send/utils' 5import { forwardVideoRelatedActivity } from '../send/utils'
6import { getOrCreateVideoAndAccountAndChannel } from '../videos' 6import { getOrCreateAPVideo } from '../videos'
7import { Notifier } from '../../notifier' 7import { Notifier } from '../../notifier'
8import { logger } from '../../../helpers/logger' 8import { logger } from '../../../helpers/logger'
9import { APProcessorOptions } from '../../../types/activitypub-processor.model' 9import { APProcessorOptions } from '../../../types/activitypub-processor.model'
@@ -32,7 +32,7 @@ async function processVideoShare (actorAnnouncer: MActorSignature, activity: Act
32 let videoCreated: boolean 32 let videoCreated: boolean
33 33
34 try { 34 try {
35 const result = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) 35 const result = await getOrCreateAPVideo({ videoObject: objectUri })
36 video = result.video 36 video = result.video
37 videoCreated = result.created 37 videoCreated = result.created
38 } catch (err) { 38 } catch (err) {
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 9cded4dec..ef5a3100e 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -12,7 +12,7 @@ import { createOrUpdateCacheFile } from '../cache-file'
12import { createOrUpdateVideoPlaylist } from '../playlist' 12import { createOrUpdateVideoPlaylist } from '../playlist'
13import { forwardVideoRelatedActivity } from '../send/utils' 13import { forwardVideoRelatedActivity } from '../send/utils'
14import { resolveThread } from '../video-comments' 14import { resolveThread } from '../video-comments'
15import { getOrCreateVideoAndAccountAndChannel } from '../videos' 15import { getOrCreateAPVideo } from '../videos'
16import { isBlockedByServerOrAccount } from '@server/lib/blocklist' 16import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
17 17
18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { 18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
@@ -55,7 +55,7 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) {
55 const videoToCreateData = activity.object as VideoObject 55 const videoToCreateData = activity.object as VideoObject
56 56
57 const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } 57 const syncParam = { likes: false, dislikes: false, shares: false, comments: false, thumbnail: true, refreshVideo: false }
58 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData, syncParam }) 58 const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam })
59 59
60 if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) 60 if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video)
61 61
@@ -67,7 +67,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor
67 67
68 const cacheFile = activity.object as CacheFileObject 68 const cacheFile = activity.object as CacheFileObject
69 69
70 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) 70 const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object })
71 71
72 await sequelizeTypescript.transaction(async t => { 72 await sequelizeTypescript.transaction(async t => {
73 return createOrUpdateCacheFile(cacheFile, video, byActor, t) 73 return createOrUpdateCacheFile(cacheFile, video, byActor, t)
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts
index 089c7b881..ecc57cd10 100644
--- a/server/lib/activitypub/process/process-dislike.ts
+++ b/server/lib/activitypub/process/process-dislike.ts
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
6import { APProcessorOptions } from '../../../types/activitypub-processor.model' 6import { APProcessorOptions } from '../../../types/activitypub-processor.model'
7import { MActorSignature } from '../../../types/models' 7import { MActorSignature } from '../../../types/models'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 9import { getOrCreateAPVideo } from '../videos'
10 10
11async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { 11async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) {
12 const { activity, byActor } = options 12 const { activity, byActor } = options
@@ -30,7 +30,7 @@ async function processDislike (activity: ActivityCreate | ActivityDislike, byAct
30 30
31 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) 31 if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
32 32
33 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislikeObject }) 33 const { video } = await getOrCreateAPVideo({ videoObject: dislikeObject })
34 34
35 return sequelizeTypescript.transaction(async t => { 35 return sequelizeTypescript.transaction(async t => {
36 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) 36 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t)
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts
index 8688b3b47..cd4e86cbb 100644
--- a/server/lib/activitypub/process/process-like.ts
+++ b/server/lib/activitypub/process/process-like.ts
@@ -6,7 +6,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
6import { APProcessorOptions } from '../../../types/activitypub-processor.model' 6import { APProcessorOptions } from '../../../types/activitypub-processor.model'
7import { MActorSignature } from '../../../types/models' 7import { MActorSignature } from '../../../types/models'
8import { forwardVideoRelatedActivity } from '../send/utils' 8import { forwardVideoRelatedActivity } from '../send/utils'
9import { getOrCreateVideoAndAccountAndChannel } from '../videos' 9import { getOrCreateAPVideo } from '../videos'
10 10
11async function processLikeActivity (options: APProcessorOptions<ActivityLike>) { 11async function processLikeActivity (options: APProcessorOptions<ActivityLike>) {
12 const { activity, byActor } = options 12 const { activity, byActor } = options
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: MActorSignature, activity: ActivityLik
27 const byAccount = byActor.Account 27 const byAccount = byActor.Account
28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) 28 if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
29 29
30 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) 30 const { video } = await getOrCreateAPVideo({ videoObject: videoUrl })
31 31
32 return sequelizeTypescript.transaction(async t => { 32 return sequelizeTypescript.transaction(async t => {
33 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) 33 const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t)
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts
index 9f031b528..fdb8dac24 100644
--- a/server/lib/activitypub/process/process-undo.ts
+++ b/server/lib/activitypub/process/process-undo.ts
@@ -11,7 +11,7 @@ import { VideoShareModel } from '../../../models/video/video-share'
11import { APProcessorOptions } from '../../../types/activitypub-processor.model' 11import { APProcessorOptions } from '../../../types/activitypub-processor.model'
12import { MActorSignature } from '../../../types/models' 12import { MActorSignature } from '../../../types/models'
13import { forwardVideoRelatedActivity } from '../send/utils' 13import { forwardVideoRelatedActivity } from '../send/utils'
14import { getOrCreateVideoAndAccountAndChannel } from '../videos' 14import { getOrCreateAPVideo } from '../videos'
15 15
16async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { 16async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) {
17 const { activity, byActor } = options 17 const { activity, byActor } = options
@@ -55,7 +55,7 @@ export {
55async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { 55async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) {
56 const likeActivity = activity.object as ActivityLike 56 const likeActivity = activity.object as ActivityLike
57 57
58 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) 58 const { video } = await getOrCreateAPVideo({ videoObject: likeActivity.object })
59 59
60 return sequelizeTypescript.transaction(async t => { 60 return sequelizeTypescript.transaction(async t => {
61 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 61 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
@@ -80,7 +80,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU
80 ? activity.object 80 ? activity.object
81 : activity.object.object as DislikeObject 81 : activity.object.object as DislikeObject
82 82
83 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) 83 const { video } = await getOrCreateAPVideo({ videoObject: dislike.object })
84 84
85 return sequelizeTypescript.transaction(async t => { 85 return sequelizeTypescript.transaction(async t => {
86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) 86 if (!byActor.Account) throw new Error('Unknown account ' + byActor.url)
@@ -103,7 +103,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU
103async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { 103async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) {
104 const cacheFileObject = activity.object.object as CacheFileObject 104 const cacheFileObject = activity.object.object as CacheFileObject
105 105
106 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) 106 const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object })
107 107
108 return sequelizeTypescript.transaction(async t => { 108 return sequelizeTypescript.transaction(async t => {
109 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) 109 const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 516bd8d70..be3f6acac 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -17,7 +17,7 @@ import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } f
17import { createOrUpdateCacheFile } from '../cache-file' 17import { createOrUpdateCacheFile } from '../cache-file'
18import { createOrUpdateVideoPlaylist } from '../playlist' 18import { createOrUpdateVideoPlaylist } from '../playlist'
19import { forwardVideoRelatedActivity } from '../send/utils' 19import { forwardVideoRelatedActivity } from '../send/utils'
20import { APVideoUpdater, getOrCreateVideoAndAccountAndChannel } from '../videos' 20import { APVideoUpdater, getOrCreateAPVideo } from '../videos'
21 21
22async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { 22async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) {
23 const { activity, byActor } = options 23 const { activity, byActor } = options
@@ -63,7 +63,7 @@ async function processUpdateVideo (activity: ActivityUpdate) {
63 return undefined 63 return undefined
64 } 64 }
65 65
66 const { video, created } = await getOrCreateVideoAndAccountAndChannel({ 66 const { video, created } = await getOrCreateAPVideo({
67 videoObject: videoObject.id, 67 videoObject: videoObject.id,
68 allowRefresh: false, 68 allowRefresh: false,
69 fetchType: 'all' 69 fetchType: 'all'
@@ -85,7 +85,7 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ
85 return undefined 85 return undefined
86 } 86 }
87 87
88 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) 88 const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object })
89 89
90 await sequelizeTypescript.transaction(async t => { 90 await sequelizeTypescript.transaction(async t => {
91 await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) 91 await createOrUpdateCacheFile(cacheFileObject, video, byActor, t)
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index 84697673b..c2d41dd28 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -1,4 +1,4 @@
1import { getOrCreateVideoAndAccountAndChannel } from '../videos' 1import { getOrCreateAPVideo } from '../videos'
2import { forwardVideoRelatedActivity } from '../send/utils' 2import { forwardVideoRelatedActivity } from '../send/utils'
3import { Redis } from '../../redis' 3import { Redis } from '../../redis'
4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub' 4import { ActivityCreate, ActivityView, ViewObject } from '../../../../shared/models/activitypub'
@@ -29,7 +29,7 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
29 fetchType: 'only-video' as 'only-video', 29 fetchType: 'only-video' as 'only-video',
30 allowRefresh: false as false 30 allowRefresh: false as false
31 } 31 }
32 const { video } = await getOrCreateVideoAndAccountAndChannel(options) 32 const { video } = await getOrCreateAPVideo(options)
33 33
34 if (!video.isLive) { 34 if (!video.isLive) {
35 await Redis.Instance.addVideoView(video.id) 35 await Redis.Instance.addVideoView(video.id)
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index e23e0c0e7..722147b69 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -7,7 +7,7 @@ import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/cons
7import { VideoCommentModel } from '../../models/video/video-comment' 7import { VideoCommentModel } from '../../models/video/video-comment'
8import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' 8import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video'
9import { getOrCreateActorAndServerAndModel } from './actor' 9import { getOrCreateActorAndServerAndModel } from './actor'
10import { getOrCreateVideoAndAccountAndChannel } from './videos' 10import { getOrCreateAPVideo } from './videos'
11 11
12type ResolveThreadParams = { 12type ResolveThreadParams = {
13 url: string 13 url: string
@@ -89,7 +89,7 @@ async function tryResolveThreadFromVideo (params: ResolveThreadParams) {
89 // Maybe it's a reply to a video? 89 // Maybe it's a reply to a video?
90 // If yes, it's done: we resolved all the thread 90 // If yes, it's done: we resolved all the thread
91 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false } 91 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false }
92 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam }) 92 const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam })
93 93
94 if (video.isOwned() && !video.hasPrivacyForFederation()) { 94 if (video.isOwned() && !video.hasPrivacyForFederation()) {
95 throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') 95 throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation')
diff --git a/server/lib/activitypub/videos/fetch.ts b/server/lib/activitypub/videos/fetch.ts
deleted file mode 100644
index 5113c9d7e..000000000
--- a/server/lib/activitypub/videos/fetch.ts
+++ /dev/null
@@ -1,180 +0,0 @@
1import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub'
2import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
3import { retryTransactionWrapper } from '@server/helpers/database-utils'
4import { logger } from '@server/helpers/logger'
5import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
6import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video'
7import { REMOTE_SCHEME } from '@server/initializers/constants'
8import { ActorFollowScoreCache } from '@server/lib/files-cache'
9import { JobQueue } from '@server/lib/job-queue'
10import { VideoModel } from '@server/models/video/video'
11import { MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
12import { HttpStatusCode } from '@shared/core-utils'
13import { VideoObject } from '@shared/models'
14import { APVideoCreator, SyncParam, syncVideoExternalAttributes } from './shared'
15import { APVideoUpdater } from './updater'
16
17async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
18 logger.info('Fetching remote video %s.', videoUrl)
19
20 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
21
22 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
23 logger.debug('Remote video JSON is not valid.', { body })
24 return { statusCode, videoObject: undefined }
25 }
26
27 return { statusCode, videoObject: body }
28}
29
30async function fetchRemoteVideoDescription (video: MVideoAccountLight) {
31 const host = video.VideoChannel.Account.Actor.Server.host
32 const path = video.getDescriptionAPIPath()
33 const url = REMOTE_SCHEME.HTTP + '://' + host + path
34
35 const { body } = await doJSONRequest<any>(url)
36 return body.description || ''
37}
38
39type GetVideoResult <T> = Promise<{
40 video: T
41 created: boolean
42 autoBlacklisted?: boolean
43}>
44
45type GetVideoParamAll = {
46 videoObject: { id: string } | string
47 syncParam?: SyncParam
48 fetchType?: 'all'
49 allowRefresh?: boolean
50}
51
52type GetVideoParamImmutable = {
53 videoObject: { id: string } | string
54 syncParam?: SyncParam
55 fetchType: 'only-immutable-attributes'
56 allowRefresh: false
57}
58
59type GetVideoParamOther = {
60 videoObject: { id: string } | string
61 syncParam?: SyncParam
62 fetchType?: 'all' | 'only-video'
63 allowRefresh?: boolean
64}
65
66function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
67function getOrCreateVideoAndAccountAndChannel (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
68function getOrCreateVideoAndAccountAndChannel (
69 options: GetVideoParamOther
70): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
71async function getOrCreateVideoAndAccountAndChannel (
72 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
73): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
74 // Default params
75 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
76 const fetchType = options.fetchType || 'all'
77 const allowRefresh = options.allowRefresh !== false
78
79 // Get video url
80 const videoUrl = getAPId(options.videoObject)
81 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
82
83 if (videoFromDatabase) {
84 // If allowRefresh is true, we could not call this function using 'only-immutable-attributes' fetch type
85 if (allowRefresh === true && (videoFromDatabase as MVideoThumbnail).isOutdated()) {
86 const refreshOptions = {
87 video: videoFromDatabase as MVideoThumbnail,
88 fetchedType: fetchType,
89 syncParam
90 }
91
92 if (syncParam.refreshVideo === true) {
93 videoFromDatabase = await refreshVideoIfNeeded(refreshOptions)
94 } else {
95 await JobQueue.Instance.createJobWithPromise({
96 type: 'activitypub-refresher',
97 payload: { type: 'video', url: videoFromDatabase.url }
98 })
99 }
100 }
101
102 return { video: videoFromDatabase, created: false }
103 }
104
105 const { videoObject } = await fetchRemoteVideo(videoUrl)
106 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
107
108 try {
109 const creator = new APVideoCreator(videoObject)
110 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
111
112 await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
113
114 return { video: videoCreated, created: true, autoBlacklisted }
115 } catch (err) {
116 // Maybe a concurrent getOrCreateVideoAndAccountAndChannel call created this video
117 if (err.name === 'SequelizeUniqueConstraintError') {
118 const fallbackVideo = await fetchVideoByUrl(videoUrl, fetchType)
119 if (fallbackVideo) return { video: fallbackVideo, created: false }
120 }
121
122 throw err
123 }
124}
125
126async function refreshVideoIfNeeded (options: {
127 video: MVideoThumbnail
128 fetchedType: VideoFetchByUrlType
129 syncParam: SyncParam
130}): Promise<MVideoThumbnail> {
131 if (!options.video.isOutdated()) return options.video
132
133 // We need more attributes if the argument video was fetched with not enough joints
134 const video = options.fetchedType === 'all'
135 ? options.video as MVideoAccountLightBlacklistAllFiles
136 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
137
138 try {
139 const { videoObject } = await fetchRemoteVideo(video.url)
140
141 if (videoObject === undefined) {
142 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
143
144 await video.setAsRefreshed()
145 return video
146 }
147
148 const videoUpdater = new APVideoUpdater(videoObject, video)
149 await videoUpdater.update()
150
151 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
152
153 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
154
155 return video
156 } catch (err) {
157 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
158 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
159
160 // Video does not exist anymore
161 await video.destroy()
162 return undefined
163 }
164
165 logger.warn('Cannot refresh video %s.', options.video.url, { err })
166
167 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
168
169 // Don't refresh in loop
170 await video.setAsRefreshed()
171 return video
172 }
173}
174
175export {
176 fetchRemoteVideo,
177 fetchRemoteVideoDescription,
178 refreshVideoIfNeeded,
179 getOrCreateVideoAndAccountAndChannel
180}
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts
new file mode 100644
index 000000000..a8c41e178
--- /dev/null
+++ b/server/lib/activitypub/videos/get.ts
@@ -0,0 +1,109 @@
1import { getAPId } from '@server/helpers/activitypub'
2import { retryTransactionWrapper } from '@server/helpers/database-utils'
3import { fetchVideoByUrl, VideoFetchByUrlType } from '@server/helpers/video'
4import { JobQueue } from '@server/lib/job-queue'
5import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models'
6import { refreshVideoIfNeeded } from './refresh'
7import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
8
9type GetVideoResult <T> = Promise<{
10 video: T
11 created: boolean
12 autoBlacklisted?: boolean
13}>
14
15type GetVideoParamAll = {
16 videoObject: { id: string } | string
17 syncParam?: SyncParam
18 fetchType?: 'all'
19 allowRefresh?: boolean
20}
21
22type GetVideoParamImmutable = {
23 videoObject: { id: string } | string
24 syncParam?: SyncParam
25 fetchType: 'only-immutable-attributes'
26 allowRefresh: false
27}
28
29type GetVideoParamOther = {
30 videoObject: { id: string } | string
31 syncParam?: SyncParam
32 fetchType?: 'all' | 'only-video'
33 allowRefresh?: boolean
34}
35
36function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult<MVideoAccountLightBlacklistAllFiles>
37function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult<MVideoImmutable>
38function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail>
39
40async function getOrCreateAPVideo (
41 options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther
42): GetVideoResult<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
43 // Default params
44 const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
45 const fetchType = options.fetchType || 'all'
46 const allowRefresh = options.allowRefresh !== false
47
48 // Get video url
49 const videoUrl = getAPId(options.videoObject)
50 let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
51
52 if (videoFromDatabase) {
53 if (allowRefresh === true) {
54 // Typings ensure allowRefresh === false in only-immutable-attributes fetch type
55 videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam)
56 }
57
58 return { video: videoFromDatabase, created: false }
59 }
60
61 const { videoObject } = await fetchRemoteVideo(videoUrl)
62 if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
63
64 try {
65 const creator = new APVideoCreator(videoObject)
66 const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator), syncParam.thumbnail)
67
68 await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
69
70 return { video: videoCreated, created: true, autoBlacklisted }
71 } catch (err) {
72 // Maybe a concurrent getOrCreateAPVideo call created this video
73 if (err.name === 'SequelizeUniqueConstraintError') {
74 const alreadyCreatedVideo = await fetchVideoByUrl(videoUrl, fetchType)
75 if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false }
76 }
77
78 throw err
79 }
80}
81
82// ---------------------------------------------------------------------------
83
84export {
85 getOrCreateAPVideo
86}
87
88// ---------------------------------------------------------------------------
89
90async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoFetchByUrlType, syncParam: SyncParam) {
91 if (!video.isOutdated()) return video
92
93 const refreshOptions = {
94 video,
95 fetchedType: fetchType,
96 syncParam
97 }
98
99 if (syncParam.refreshVideo === true) {
100 return refreshVideoIfNeeded(refreshOptions)
101 }
102
103 await JobQueue.Instance.createJobWithPromise({
104 type: 'activitypub-refresher',
105 payload: { type: 'video', url: video.url }
106 })
107
108 return video
109}
diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts
index b560acb76..b22062598 100644
--- a/server/lib/activitypub/videos/index.ts
+++ b/server/lib/activitypub/videos/index.ts
@@ -1,3 +1,4 @@
1export * from './federate' 1export * from './federate'
2export * from './fetch' 2export * from './get'
3export * from './refresh'
3export * from './updater' 4export * from './updater'
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts
new file mode 100644
index 000000000..205a3ccb1
--- /dev/null
+++ b/server/lib/activitypub/videos/refresh.ts
@@ -0,0 +1,64 @@
1import { logger } from '@server/helpers/logger'
2import { PeerTubeRequestError } from '@server/helpers/requests'
3import { VideoFetchByUrlType } from '@server/helpers/video'
4import { ActorFollowScoreCache } from '@server/lib/files-cache'
5import { VideoModel } from '@server/models/video/video'
6import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models'
7import { HttpStatusCode } from '@shared/core-utils'
8import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
9import { APVideoUpdater } from './updater'
10
11async function refreshVideoIfNeeded (options: {
12 video: MVideoThumbnail
13 fetchedType: VideoFetchByUrlType
14 syncParam: SyncParam
15}): Promise<MVideoThumbnail> {
16 if (!options.video.isOutdated()) return options.video
17
18 // We need more attributes if the argument video was fetched with not enough joints
19 const video = options.fetchedType === 'all'
20 ? options.video as MVideoAccountLightBlacklistAllFiles
21 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
22
23 try {
24 const { videoObject } = await fetchRemoteVideo(video.url)
25
26 if (videoObject === undefined) {
27 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
28
29 await video.setAsRefreshed()
30 return video
31 }
32
33 const videoUpdater = new APVideoUpdater(videoObject, video)
34 await videoUpdater.update()
35
36 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
37
38 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
39
40 return video
41 } catch (err) {
42 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
43 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
44
45 // Video does not exist anymore
46 await video.destroy()
47 return undefined
48 }
49
50 logger.warn('Cannot refresh video %s.', options.video.url, { err })
51
52 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
53
54 // Don't refresh in loop
55 await video.setAsRefreshed()
56 return video
57 }
58}
59
60// ---------------------------------------------------------------------------
61
62export {
63 refreshVideoIfNeeded
64}
diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts
index 208a43705..951403493 100644
--- a/server/lib/activitypub/videos/shared/index.ts
+++ b/server/lib/activitypub/videos/shared/index.ts
@@ -2,4 +2,5 @@ export * from './abstract-builder'
2export * from './creator' 2export * from './creator'
3export * from './object-to-model-attributes' 3export * from './object-to-model-attributes'
4export * from './trackers' 4export * from './trackers'
5export * from './url-to-object'
5export * from './video-sync-attributes' 6export * from './video-sync-attributes'
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts
new file mode 100644
index 000000000..b1ecac8ca
--- /dev/null
+++ b/server/lib/activitypub/videos/shared/url-to-object.ts
@@ -0,0 +1,22 @@
1import { checkUrlsSameHost } from '@server/helpers/activitypub'
2import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
3import { logger } from '@server/helpers/logger'
4import { doJSONRequest } from '@server/helpers/requests'
5import { VideoObject } from '@shared/models'
6
7async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
8 logger.info('Fetching remote video %s.', videoUrl)
9
10 const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
11
12 if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
13 logger.debug('Remote video JSON is not valid.', { body })
14 return { statusCode, videoObject: undefined }
15 }
16
17 return { statusCode, videoObject: body }
18}
19
20export {
21 fetchRemoteVideo
22}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 59b55cccc..b5a5eb697 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -23,7 +23,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../.
23import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 23import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
24import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 24import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
25import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' 25import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
26import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' 26import { getOrCreateAPVideo } from '../activitypub/videos'
27import { downloadPlaylistSegments } from '../hls' 27import { downloadPlaylistSegments } from '../hls'
28import { removeVideoRedundancy } from '../redundancy' 28import { removeVideoRedundancy } from '../redundancy'
29import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' 29import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths'
@@ -351,7 +351,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
351 syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true }, 351 syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true },
352 fetchType: 'all' as 'all' 352 fetchType: 'all' as 'all'
353 } 353 }
354 const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions) 354 const { video } = await getOrCreateAPVideo(getVideoOptions)
355 355
356 return video 356 return video
357 } 357 }
diff --git a/server/models/utils.ts b/server/models/utils.ts
index e27625bc8..83b2b8f03 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -102,6 +102,10 @@ function getFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'ASC' ]):
102} 102}
103 103
104function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { 104function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
105 if (!model.createdAt || !model.updatedAt) {
106 throw new Error('Miss createdAt & updatedAt attribuets to model')
107 }
108
105 const now = Date.now() 109 const now = Date.now()
106 const createdAtTime = model.createdAt.getTime() 110 const createdAtTime = model.createdAt.getTime()
107 const updatedAtTime = model.updatedAt.getTime() 111 const updatedAtTime = model.updatedAt.getTime()