diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/helpers/logger.ts | 5 | ||||
-rw-r--r-- | server/lib/activitypub/videos/refresh.ts | 10 | ||||
-rw-r--r-- | server/lib/activitypub/videos/shared/abstract-builder.ts | 5 | ||||
-rw-r--r-- | server/lib/activitypub/videos/shared/creator.ts | 7 | ||||
-rw-r--r-- | server/lib/activitypub/videos/shared/url-to-object.ts | 9 | ||||
-rw-r--r-- | server/lib/activitypub/videos/shared/video-sync-attributes.ts | 22 | ||||
-rw-r--r-- | server/lib/activitypub/videos/updater.ts | 13 |
7 files changed, 46 insertions, 25 deletions
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index a112fd300..29e06860d 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts | |||
@@ -151,7 +151,8 @@ const bunyanLogger = { | |||
151 | fatal: bunyanLogFactory('error') | 151 | fatal: bunyanLogFactory('error') |
152 | } | 152 | } |
153 | 153 | ||
154 | function loggerTagsFactory (...defaultTags: string[]) { | 154 | type LoggerTagsFn = (...tags: string[]) => { tags: string[] } |
155 | function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { | ||
155 | return (...tags: string[]) => { | 156 | return (...tags: string[]) => { |
156 | return { tags: defaultTags.concat(tags) } | 157 | return { tags: defaultTags.concat(tags) } |
157 | } | 158 | } |
@@ -160,6 +161,8 @@ function loggerTagsFactory (...defaultTags: string[]) { | |||
160 | // --------------------------------------------------------------------------- | 161 | // --------------------------------------------------------------------------- |
161 | 162 | ||
162 | export { | 163 | export { |
164 | LoggerTagsFn, | ||
165 | |||
163 | buildLogger, | 166 | buildLogger, |
164 | timestampFormatter, | 167 | timestampFormatter, |
165 | labelFormatter, | 168 | labelFormatter, |
diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts index 205a3ccb1..da068cde9 100644 --- a/server/lib/activitypub/videos/refresh.ts +++ b/server/lib/activitypub/videos/refresh.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { logger } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { PeerTubeRequestError } from '@server/helpers/requests' | 2 | import { PeerTubeRequestError } from '@server/helpers/requests' |
3 | import { VideoFetchByUrlType } from '@server/helpers/video' | 3 | import { VideoFetchByUrlType } from '@server/helpers/video' |
4 | import { ActorFollowScoreCache } from '@server/lib/files-cache' | 4 | import { ActorFollowScoreCache } from '@server/lib/files-cache' |
@@ -8,6 +8,8 @@ import { HttpStatusCode } from '@shared/core-utils' | |||
8 | import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | 8 | import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' |
9 | import { APVideoUpdater } from './updater' | 9 | import { APVideoUpdater } from './updater' |
10 | 10 | ||
11 | const lTags = loggerTagsFactory('ap', 'video', 'refresh') | ||
12 | |||
11 | async function refreshVideoIfNeeded (options: { | 13 | async function refreshVideoIfNeeded (options: { |
12 | video: MVideoThumbnail | 14 | video: MVideoThumbnail |
13 | fetchedType: VideoFetchByUrlType | 15 | fetchedType: VideoFetchByUrlType |
@@ -24,7 +26,7 @@ async function refreshVideoIfNeeded (options: { | |||
24 | const { videoObject } = await fetchRemoteVideo(video.url) | 26 | const { videoObject } = await fetchRemoteVideo(video.url) |
25 | 27 | ||
26 | if (videoObject === undefined) { | 28 | if (videoObject === undefined) { |
27 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url) | 29 | logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags(video.uuid)) |
28 | 30 | ||
29 | await video.setAsRefreshed() | 31 | await video.setAsRefreshed() |
30 | return video | 32 | return video |
@@ -40,14 +42,14 @@ async function refreshVideoIfNeeded (options: { | |||
40 | return video | 42 | return video |
41 | } catch (err) { | 43 | } catch (err) { |
42 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { | 44 | if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { |
43 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) | 45 | logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags(video.uuid)) |
44 | 46 | ||
45 | // Video does not exist anymore | 47 | // Video does not exist anymore |
46 | await video.destroy() | 48 | await video.destroy() |
47 | return undefined | 49 | return undefined |
48 | } | 50 | } |
49 | 51 | ||
50 | logger.warn('Cannot refresh video %s.', options.video.url, { err }) | 52 | logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags(video.uuid) }) |
51 | 53 | ||
52 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) | 54 | ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) |
53 | 55 | ||
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index c7631cd45..953710f6c 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { Transaction } from 'sequelize/types' | 1 | import { Transaction } from 'sequelize/types' |
2 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | 2 | import { checkUrlsSameHost } from '@server/helpers/activitypub' |
3 | import { deleteNonExistingModels } from '@server/helpers/database-utils' | 3 | import { deleteNonExistingModels } from '@server/helpers/database-utils' |
4 | import { logger } from '@server/helpers/logger' | 4 | import { logger, LoggerTagsFn } from '@server/helpers/logger' |
5 | import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail' | 5 | import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail' |
6 | import { setVideoTags } from '@server/lib/video' | 6 | import { setVideoTags } from '@server/lib/video' |
7 | import { VideoCaptionModel } from '@server/models/video/video-caption' | 7 | import { VideoCaptionModel } from '@server/models/video/video-caption' |
@@ -24,6 +24,7 @@ import { getTrackerUrls, setVideoTrackers } from './trackers' | |||
24 | 24 | ||
25 | export abstract class APVideoAbstractBuilder { | 25 | export abstract class APVideoAbstractBuilder { |
26 | protected abstract videoObject: VideoObject | 26 | protected abstract videoObject: VideoObject |
27 | protected abstract lTags: LoggerTagsFn | ||
27 | 28 | ||
28 | protected async getOrCreateVideoChannelFromVideoObject () { | 29 | protected async getOrCreateVideoChannelFromVideoObject () { |
29 | const channel = this.videoObject.attributedTo.find(a => a.type === 'Group') | 30 | const channel = this.videoObject.attributedTo.find(a => a.type === 'Group') |
@@ -42,7 +43,7 @@ export abstract class APVideoAbstractBuilder { | |||
42 | video, | 43 | video, |
43 | type: ThumbnailType.MINIATURE | 44 | type: ThumbnailType.MINIATURE |
44 | }).catch(err => { | 45 | }).catch(err => { |
45 | logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err }) | 46 | logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags(video.uuid) }) |
46 | 47 | ||
47 | return undefined | 48 | return undefined |
48 | }) | 49 | }) |
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts index dd9bfb508..ba3f6f77f 100644 --- a/server/lib/activitypub/videos/shared/creator.ts +++ b/server/lib/activitypub/videos/shared/creator.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | 1 | ||
2 | import { logger } from '@server/helpers/logger' | 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
3 | import { sequelizeTypescript } from '@server/initializers/database' | 3 | import { sequelizeTypescript } from '@server/initializers/database' |
4 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | 4 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' |
5 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
@@ -9,13 +9,14 @@ import { APVideoAbstractBuilder } from './abstract-builder' | |||
9 | import { getVideoAttributesFromObject } from './object-to-model-attributes' | 9 | import { getVideoAttributesFromObject } from './object-to-model-attributes' |
10 | 10 | ||
11 | export class APVideoCreator extends APVideoAbstractBuilder { | 11 | export class APVideoCreator extends APVideoAbstractBuilder { |
12 | protected lTags = loggerTagsFactory('ap', 'video', 'create') | ||
12 | 13 | ||
13 | constructor (protected readonly videoObject: VideoObject) { | 14 | constructor (protected readonly videoObject: VideoObject) { |
14 | super() | 15 | super() |
15 | } | 16 | } |
16 | 17 | ||
17 | async create (waitThumbnail = false) { | 18 | async create (waitThumbnail = false) { |
18 | logger.debug('Adding remote video %s.', this.videoObject.id) | 19 | logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags(this.videoObject.uuid)) |
19 | 20 | ||
20 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | 21 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() |
21 | const channel = channelActor.VideoChannel | 22 | const channel = channelActor.VideoChannel |
@@ -56,7 +57,7 @@ export class APVideoCreator extends APVideoAbstractBuilder { | |||
56 | transaction: t | 57 | transaction: t |
57 | }) | 58 | }) |
58 | 59 | ||
59 | logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid) | 60 | logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags(videoCreated.uuid)) |
60 | 61 | ||
61 | return { autoBlacklisted, videoCreated } | 62 | return { autoBlacklisted, videoCreated } |
62 | } catch (err) { | 63 | } catch (err) { |
diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts index b1ecac8ca..dba3e9480 100644 --- a/server/lib/activitypub/videos/shared/url-to-object.ts +++ b/server/lib/activitypub/videos/shared/url-to-object.ts | |||
@@ -1,16 +1,19 @@ | |||
1 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | 1 | import { checkUrlsSameHost } from '@server/helpers/activitypub' |
2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' | 2 | import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' |
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
4 | import { doJSONRequest } from '@server/helpers/requests' | 4 | import { doJSONRequest } from '@server/helpers/requests' |
5 | import { VideoObject } from '@shared/models' | 5 | import { VideoObject } from '@shared/models' |
6 | 6 | ||
7 | const lTags = loggerTagsFactory('ap', 'video') | ||
8 | |||
7 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { | 9 | async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { |
8 | logger.info('Fetching remote video %s.', videoUrl) | 10 | logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) |
9 | 11 | ||
10 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) | 12 | const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true }) |
11 | 13 | ||
12 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { | 14 | if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { |
13 | logger.debug('Remote video JSON is not valid.', { body }) | 15 | logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) |
16 | |||
14 | return { statusCode, videoObject: undefined } | 17 | return { statusCode, videoObject: undefined } |
15 | } | 18 | } |
16 | 19 | ||
diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts index c1318abf7..102d40621 100644 --- a/server/lib/activitypub/videos/shared/video-sync-attributes.ts +++ b/server/lib/activitypub/videos/shared/video-sync-attributes.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { logger } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { JobQueue } from '@server/lib/job-queue' | 2 | import { JobQueue } from '@server/lib/job-queue' |
3 | import { AccountVideoRateModel } from '@server/models/account/account-video-rate' | 3 | import { AccountVideoRateModel } from '@server/models/account/account-video-rate' |
4 | import { VideoCommentModel } from '@server/models/video/video-comment' | 4 | import { VideoCommentModel } from '@server/models/video/video-comment' |
@@ -10,6 +10,8 @@ import { addVideoShares } from '../../share' | |||
10 | import { addVideoComments } from '../../video-comments' | 10 | import { addVideoComments } from '../../video-comments' |
11 | import { createRates } from '../../video-rates' | 11 | import { createRates } from '../../video-rates' |
12 | 12 | ||
13 | const lTags = loggerTagsFactory('ap', 'video') | ||
14 | |||
13 | type SyncParam = { | 15 | type SyncParam = { |
14 | likes: boolean | 16 | likes: boolean |
15 | dislikes: boolean | 17 | dislikes: boolean |
@@ -60,29 +62,33 @@ function syncRates (type: 'like' | 'dislike', video: MVideo, fetchedVideo: Video | |||
60 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, type, crawlStartDate) | 62 | const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, type, crawlStartDate) |
61 | 63 | ||
62 | return crawlCollectionPage<string>(uri, handler, cleaner) | 64 | return crawlCollectionPage<string>(uri, handler, cleaner) |
63 | .catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri })) | 65 | .catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid) })) |
64 | } | 66 | } |
65 | 67 | ||
66 | function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | 68 | function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { |
69 | const uri = fetchedVideo.shares | ||
70 | |||
67 | if (!isSync) { | 71 | if (!isSync) { |
68 | return createJob({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' }) | 72 | return createJob({ uri, videoId: video.id, type: 'video-shares' }) |
69 | } | 73 | } |
70 | 74 | ||
71 | const handler = items => addVideoShares(items, video) | 75 | const handler = items => addVideoShares(items, video) |
72 | const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) | 76 | const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) |
73 | 77 | ||
74 | return crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner) | 78 | return crawlCollectionPage<string>(uri, handler, cleaner) |
75 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares })) | 79 | .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid) })) |
76 | } | 80 | } |
77 | 81 | ||
78 | function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { | 82 | function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { |
83 | const uri = fetchedVideo.comments | ||
84 | |||
79 | if (!isSync) { | 85 | if (!isSync) { |
80 | return createJob({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' }) | 86 | return createJob({ uri, videoId: video.id, type: 'video-comments' }) |
81 | } | 87 | } |
82 | 88 | ||
83 | const handler = items => addVideoComments(items) | 89 | const handler = items => addVideoComments(items) |
84 | const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) | 90 | const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) |
85 | 91 | ||
86 | return crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner) | 92 | return crawlCollectionPage<string>(uri, handler, cleaner) |
87 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments })) | 93 | .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid) })) |
88 | } | 94 | } |
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 11c177a68..9e1c74969 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { Transaction } from 'sequelize/types' | 1 | import { Transaction } from 'sequelize/types' |
2 | import { resetSequelizeInstance } from '@server/helpers/database-utils' | 2 | import { resetSequelizeInstance } from '@server/helpers/database-utils' |
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
4 | import { sequelizeTypescript } from '@server/initializers/database' | 4 | import { sequelizeTypescript } from '@server/initializers/database' |
5 | import { Notifier } from '@server/lib/notifier' | 5 | import { Notifier } from '@server/lib/notifier' |
6 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | 6 | import { PeerTubeSocket } from '@server/lib/peertube-socket' |
@@ -19,6 +19,8 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
19 | 19 | ||
20 | private readonly oldVideoChannel: MChannelAccountLight | 20 | private readonly oldVideoChannel: MChannelAccountLight |
21 | 21 | ||
22 | protected lTags = loggerTagsFactory('ap', 'video', 'update') | ||
23 | |||
22 | constructor ( | 24 | constructor ( |
23 | protected readonly videoObject: VideoObject, | 25 | protected readonly videoObject: VideoObject, |
24 | private readonly video: MVideoAccountLightBlacklistAllFiles | 26 | private readonly video: MVideoAccountLightBlacklistAllFiles |
@@ -34,7 +36,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
34 | } | 36 | } |
35 | 37 | ||
36 | async update (overrideTo?: string[]) { | 38 | async update (overrideTo?: string[]) { |
37 | logger.debug('Updating remote video "%s".', this.videoObject.uuid, { videoObject: this.videoObject }) | 39 | logger.debug( |
40 | 'Updating remote video "%s".', this.videoObject.uuid, | ||
41 | { videoObject: this.videoObject, ...this.lTags(this.videoObject.uuid) } | ||
42 | ) | ||
38 | 43 | ||
39 | try { | 44 | try { |
40 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | 45 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() |
@@ -77,7 +82,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
77 | PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated) | 82 | PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated) |
78 | } | 83 | } |
79 | 84 | ||
80 | logger.info('Remote video with uuid %s updated', this.videoObject.uuid) | 85 | logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags(this.videoObject.uuid)) |
81 | 86 | ||
82 | return videoUpdated | 87 | return videoUpdated |
83 | } catch (err) { | 88 | } catch (err) { |
@@ -153,7 +158,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
153 | } | 158 | } |
154 | 159 | ||
155 | // This is just a debug because we will retry the insert | 160 | // This is just a debug because we will retry the insert |
156 | logger.debug('Cannot update the remote video.', { err }) | 161 | logger.debug('Cannot update the remote video.', { err, ...this.lTags(this.videoObject.uuid) }) |
157 | throw err | 162 | throw err |
158 | } | 163 | } |
159 | } | 164 | } |