aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts22
-rw-r--r--server/lib/activitypub/playlist.ts17
-rw-r--r--server/lib/activitypub/videos.ts19
-rw-r--r--server/lib/job-queue/handlers/activitypub-cleaner.ts49
4 files changed, 57 insertions, 50 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 52b6c1f56..3c9a7ba02 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -14,7 +14,7 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
14import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' 14import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
15import { logger } from '../../helpers/logger' 15import { logger } from '../../helpers/logger'
16import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' 16import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
17import { doJSONRequest } from '../../helpers/requests' 17import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
18import { getUrlFromWebfinger } from '../../helpers/webfinger' 18import { getUrlFromWebfinger } from '../../helpers/webfinger'
19import { MIMETYPES, WEBSERVER } from '../../initializers/constants' 19import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
20import { sequelizeTypescript } from '../../initializers/database' 20import { sequelizeTypescript } from '../../initializers/database'
@@ -279,16 +279,7 @@ async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannel
279 actorUrl = actor.url 279 actorUrl = actor.url
280 } 280 }
281 281
282 const { result, statusCode } = await fetchRemoteActor(actorUrl) 282 const { result } = await fetchRemoteActor(actorUrl)
283
284 if (statusCode === HttpStatusCode.NOT_FOUND_404) {
285 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
286 actor.Account
287 ? await actor.Account.destroy()
288 : await actor.VideoChannel.destroy()
289
290 return { actor: undefined, refreshed: false }
291 }
292 283
293 if (result === undefined) { 284 if (result === undefined) {
294 logger.warn('Cannot fetch remote actor in refresh actor.') 285 logger.warn('Cannot fetch remote actor in refresh actor.')
@@ -328,6 +319,15 @@ async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannel
328 return { refreshed: true, actor } 319 return { refreshed: true, actor }
329 }) 320 })
330 } catch (err) { 321 } catch (err) {
322 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
323 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
324 actor.Account
325 ? await actor.Account.destroy()
326 : await actor.VideoChannel.destroy()
327
328 return { actor: undefined, refreshed: false }
329 }
330
331 logger.warn('Cannot refresh actor %s.', actor.url, { err }) 331 logger.warn('Cannot refresh actor %s.', actor.url, { err })
332 return { actor, refreshed: false } 332 return { actor, refreshed: false }
333 } 333 }
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
index 795be60d7..7166c68a6 100644
--- a/server/lib/activitypub/playlist.ts
+++ b/server/lib/activitypub/playlist.ts
@@ -7,7 +7,7 @@ import { checkUrlsSameHost } from '../../helpers/activitypub'
7import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' 7import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
8import { isArray } from '../../helpers/custom-validators/misc' 8import { isArray } from '../../helpers/custom-validators/misc'
9import { logger } from '../../helpers/logger' 9import { logger } from '../../helpers/logger'
10import { doJSONRequest } from '../../helpers/requests' 10import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
11import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 11import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
12import { sequelizeTypescript } from '../../initializers/database' 12import { sequelizeTypescript } from '../../initializers/database'
13import { VideoPlaylistModel } from '../../models/video/video-playlist' 13import { VideoPlaylistModel } from '../../models/video/video-playlist'
@@ -116,13 +116,7 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
116 if (!videoPlaylist.isOutdated()) return videoPlaylist 116 if (!videoPlaylist.isOutdated()) return videoPlaylist
117 117
118 try { 118 try {
119 const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) 119 const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
120 if (statusCode === HttpStatusCode.NOT_FOUND_404) {
121 logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
122
123 await videoPlaylist.destroy()
124 return undefined
125 }
126 120
127 if (playlistObject === undefined) { 121 if (playlistObject === undefined) {
128 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) 122 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
@@ -136,6 +130,13 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
136 130
137 return videoPlaylist 131 return videoPlaylist
138 } catch (err) { 132 } catch (err) {
133 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
134 logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
135
136 await videoPlaylist.destroy()
137 return undefined
138 }
139
139 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) 140 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
140 141
141 await videoPlaylist.setAsRefreshed() 142 await videoPlaylist.setAsRefreshed()
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index a5f58dd01..d484edd36 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -30,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc'
30import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 30import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
31import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 31import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
32import { logger } from '../../helpers/logger' 32import { logger } from '../../helpers/logger'
33import { doJSONRequest } from '../../helpers/requests' 33import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
34import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' 34import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
35import { 35import {
36 ACTIVITY_PUB, 36 ACTIVITY_PUB,
@@ -523,14 +523,7 @@ async function refreshVideoIfNeeded (options: {
523 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) 523 : await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
524 524
525 try { 525 try {
526 const { statusCode, videoObject } = await fetchRemoteVideo(video.url) 526 const { videoObject } = await fetchRemoteVideo(video.url)
527 if (statusCode === HttpStatusCode.NOT_FOUND_404) {
528 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
529
530 // Video does not exist anymore
531 await video.destroy()
532 return undefined
533 }
534 527
535 if (videoObject === undefined) { 528 if (videoObject === undefined) {
536 logger.warn('Cannot refresh remote video %s: invalid body.', video.url) 529 logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
@@ -554,6 +547,14 @@ async function refreshVideoIfNeeded (options: {
554 547
555 return video 548 return video
556 } catch (err) { 549 } catch (err) {
550 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
551 logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
552
553 // Video does not exist anymore
554 await video.destroy()
555 return undefined
556 }
557
557 logger.warn('Cannot refresh video %s.', options.video.url, { err }) 558 logger.warn('Cannot refresh video %s.', options.video.url, { err })
558 559
559 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) 560 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts
index 9dcc778fa..1caca1dcc 100644
--- a/server/lib/job-queue/handlers/activitypub-cleaner.ts
+++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts
@@ -7,7 +7,7 @@ import {
7 isLikeActivityValid 7 isLikeActivityValid
8} from '@server/helpers/custom-validators/activitypub/activity' 8} from '@server/helpers/custom-validators/activitypub/activity'
9import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' 9import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
10import { doJSONRequest } from '@server/helpers/requests' 10import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
11import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants' 11import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants'
12import { VideoModel } from '@server/models/video/video' 12import { VideoModel } from '@server/models/video/video'
13import { VideoCommentModel } from '@server/models/video/video-comment' 13import { VideoCommentModel } from '@server/models/video/video-comment'
@@ -81,39 +81,44 @@ async function updateObjectIfNeeded <T> (
81 updater: (url: string, newUrl: string) => Promise<T>, 81 updater: (url: string, newUrl: string) => Promise<T>,
82 deleter: (url: string) => Promise<T> 82 deleter: (url: string) => Promise<T>
83): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { 83): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
84 const { statusCode, body } = await doJSONRequest<any>(url, { activityPub: true }) 84 const on404OrTombstone = async () => {
85
86 // Does not exist anymore, remove entry
87 if (statusCode === HttpStatusCode.NOT_FOUND_404) {
88 logger.info('Removing remote AP object %s.', url) 85 logger.info('Removing remote AP object %s.', url)
89 const data = await deleter(url) 86 const data = await deleter(url)
90 87
91 return { status: 'deleted', data } 88 return { status: 'deleted' as 'deleted', data }
92 } 89 }
93 90
94 // If not same id, check same host and update 91 try {
95 if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) 92 const { body } = await doJSONRequest<any>(url, { activityPub: true })
96 93
97 if (body.type === 'Tombstone') { 94 // If not same id, check same host and update
98 logger.info('Removing remote AP object %s.', url) 95 if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
99 const data = await deleter(url)
100 96
101 return { status: 'deleted', data } 97 if (body.type === 'Tombstone') {
102 } 98 return on404OrTombstone()
99 }
103 100
104 const newUrl = body.id 101 const newUrl = body.id
105 if (newUrl !== url) { 102 if (newUrl !== url) {
106 if (checkUrlsSameHost(newUrl, url) !== true) { 103 if (checkUrlsSameHost(newUrl, url) !== true) {
107 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) 104 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
105 }
106
107 logger.info('Updating remote AP object %s.', url)
108 const data = await updater(url, newUrl)
109
110 return { status: 'updated', data }
108 } 111 }
109 112
110 logger.info('Updating remote AP object %s.', url) 113 return null
111 const data = await updater(url, newUrl) 114 } catch (err) {
115 // Does not exist anymore, remove entry
116 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
117 return on404OrTombstone()
118 }
112 119
113 return { status: 'updated', data } 120 throw err
114 } 121 }
115
116 return null
117} 122}
118 123
119function rateOptionsFactory () { 124function rateOptionsFactory () {