aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-06-08 17:29:45 +0200
committerChocobozzz <me@florianbigard.com>2021-06-08 17:41:48 +0200
commit28dfb44b145c537aba07ae73cb1287f25532022a (patch)
tree4aee64a049b396c8689f29e5ca8a8094dd3baf54
parent75e12406e271e3aaf1f7c394a63ce570091db480 (diff)
downloadPeerTube-28dfb44b145c537aba07ae73cb1287f25532022a.tar.gz
PeerTube-28dfb44b145c537aba07ae73cb1287f25532022a.tar.zst
PeerTube-28dfb44b145c537aba07ae73cb1287f25532022a.zip
Try to speed up AP update transaction
-rw-r--r--server/helpers/database-utils.ts25
-rw-r--r--server/lib/activitypub/actors/get.ts2
-rw-r--r--server/lib/activitypub/actors/updater.ts24
-rw-r--r--server/lib/activitypub/process/process-view.ts9
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts2
-rw-r--r--server/lib/activitypub/videos/shared/trackers.ts2
-rw-r--r--server/lib/activitypub/videos/updater.ts33
-rw-r--r--server/models/video/video.ts2
8 files changed, 55 insertions, 44 deletions
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
index 7befa2c49..240b18033 100644
--- a/server/helpers/database-utils.ts
+++ b/server/helpers/database-utils.ts
@@ -58,7 +58,7 @@ function transactionRetryer <T> (func: (err: any, data: T) => any) {
58 58
59 errorFilter: err => { 59 errorFilter: err => {
60 const willRetry = (err.name === 'SequelizeDatabaseError') 60 const willRetry = (err.name === 'SequelizeDatabaseError')
61 logger.debug('Maybe retrying the transaction function.', { willRetry, err }) 61 logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
62 return willRetry 62 return willRetry
63 } 63 }
64 }, 64 },
@@ -68,6 +68,8 @@ function transactionRetryer <T> (func: (err: any, data: T) => any) {
68 }) 68 })
69} 69}
70 70
71// ---------------------------------------------------------------------------
72
71function updateInstanceWithAnother <M, T extends U, U extends Model<M>> (instanceToUpdate: T, baseInstance: U) { 73function updateInstanceWithAnother <M, T extends U, U extends Model<M>> (instanceToUpdate: T, baseInstance: U) {
72 const obj = baseInstance.toJSON() 74 const obj = baseInstance.toJSON()
73 75
@@ -82,12 +84,6 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
82 }) 84 })
83} 85}
84 86
85function afterCommitIfTransaction (t: Transaction, fn: Function) {
86 if (t) return t.afterCommit(() => fn())
87
88 return fn()
89}
90
91function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> ( 87function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> (
92 fromDatabase: T[], 88 fromDatabase: T[],
93 newModels: T[], 89 newModels: T[],
@@ -111,6 +107,18 @@ function setAsUpdated (table: string, id: number, transaction?: Transaction) {
111 107
112// --------------------------------------------------------------------------- 108// ---------------------------------------------------------------------------
113 109
110function runInReadCommittedTransaction <T> (fn: (t: Transaction) => Promise<T>) {
111 return sequelizeTypescript.transaction(t => fn(t))
112}
113
114function afterCommitIfTransaction (t: Transaction, fn: Function) {
115 if (t) return t.afterCommit(() => fn())
116
117 return fn()
118}
119
120// ---------------------------------------------------------------------------
121
114export { 122export {
115 resetSequelizeInstance, 123 resetSequelizeInstance,
116 retryTransactionWrapper, 124 retryTransactionWrapper,
@@ -118,5 +126,6 @@ export {
118 updateInstanceWithAnother, 126 updateInstanceWithAnother,
119 afterCommitIfTransaction, 127 afterCommitIfTransaction,
120 deleteNonExistingModels, 128 deleteNonExistingModels,
121 setAsUpdated 129 setAsUpdated,
130 runInReadCommittedTransaction
122} 131}
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts
index c7b49d6e4..de93aa964 100644
--- a/server/lib/activitypub/actors/get.ts
+++ b/server/lib/activitypub/actors/get.ts
@@ -56,7 +56,7 @@ async function getOrCreateAPActor (
56 if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor 56 if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor
57 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor 57 if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
58 58
59 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) 59 const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded(actor, fetchType)
60 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') 60 if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
61 61
62 await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) 62 await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections)
diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts
index 471688f11..fb880a767 100644
--- a/server/lib/activitypub/actors/updater.ts
+++ b/server/lib/activitypub/actors/updater.ts
@@ -1,6 +1,5 @@
1import { resetSequelizeInstance } from '@server/helpers/database-utils' 1import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
2import { logger } from '@server/helpers/logger' 2import { logger } from '@server/helpers/logger'
3import { sequelizeTypescript } from '@server/initializers/database'
4import { VideoChannelModel } from '@server/models/video/video-channel' 3import { VideoChannelModel } from '@server/models/video/video-channel'
5import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' 4import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models'
6import { ActivityPubActor, ActorImageType } from '@shared/models' 5import { ActivityPubActor, ActorImageType } from '@shared/models'
@@ -32,22 +31,23 @@ export class APActorUpdater {
32 const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER) 31 const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER)
33 32
34 try { 33 try {
35 await sequelizeTypescript.transaction(async t => { 34 await this.updateActorInstance(this.actor, this.actorObject)
36 await this.updateActorInstance(this.actor, this.actorObject)
37 35
38 await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t) 36 this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername
39 await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t) 37 this.accountOrChannel.description = this.actorObject.summary
40
41 await this.actor.save({ transaction: t })
42
43 this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername
44 this.accountOrChannel.description = this.actorObject.summary
45 38
46 if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support 39 if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support
47 40
41 await runInReadCommittedTransaction(async t => {
42 await this.actor.save({ transaction: t })
48 await this.accountOrChannel.save({ transaction: t }) 43 await this.accountOrChannel.save({ transaction: t })
49 }) 44 })
50 45
46 await runInReadCommittedTransaction(async t => {
47 await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t)
48 await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t)
49 })
50
51 logger.info('Remote account %s updated', this.actorObject.url) 51 logger.info('Remote account %s updated', this.actorObject.url)
52 } catch (err) { 52 } catch (err) {
53 if (this.actor !== undefined && this.actorFieldsSave !== undefined) { 53 if (this.actor !== undefined && this.actorFieldsSave !== undefined) {
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index c2d41dd28..0a0231a3a 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -24,12 +24,11 @@ async function processCreateView (activity: ActivityView | ActivityCreate, byAct
24 ? activity.object 24 ? activity.object
25 : (activity.object as ViewObject).object 25 : (activity.object as ViewObject).object
26 26
27 const options = { 27 const { video } = await getOrCreateAPVideo({
28 videoObject, 28 videoObject,
29 fetchType: 'only-video' as 'only-video', 29 fetchType: 'only-video',
30 allowRefresh: false as false 30 allowRefresh: false
31 } 31 })
32 const { video } = await getOrCreateAPVideo(options)
33 32
34 if (!video.isLive) { 33 if (!video.isLive) {
35 await Redis.Instance.addVideoView(video.id) 34 await Redis.Instance.addVideoView(video.id)
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
index 0b58ddb33..22280fce1 100644
--- a/server/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -49,7 +49,7 @@ export abstract class APVideoAbstractBuilder {
49 }) 49 })
50 } 50 }
51 51
52 protected async setPreview (video: MVideoFullLight, t: Transaction) { 52 protected async setPreview (video: MVideoFullLight, t?: Transaction) {
53 // Don't fetch the preview that could be big, create a placeholder instead 53 // Don't fetch the preview that could be big, create a placeholder instead
54 const previewIcon = getPreviewFromIcons(this.videoObject) 54 const previewIcon = getPreviewFromIcons(this.videoObject)
55 if (!previewIcon) return 55 if (!previewIcon) return
diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts
index fcb2a5091..1c5fc4f84 100644
--- a/server/lib/activitypub/videos/shared/trackers.ts
+++ b/server/lib/activitypub/videos/shared/trackers.ts
@@ -28,7 +28,7 @@ function getTrackerUrls (object: VideoObject, video: MVideoWithHost) {
28async function setVideoTrackers (options: { 28async function setVideoTrackers (options: {
29 video: MVideo 29 video: MVideo
30 trackers: string[] 30 trackers: string[]
31 transaction?: Transaction 31 transaction: Transaction
32}) { 32}) {
33 const { video, trackers, transaction } = options 33 const { video, trackers, transaction } = options
34 34
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts
index 3339611fc..e17e5fdc2 100644
--- a/server/lib/activitypub/videos/updater.ts
+++ b/server/lib/activitypub/videos/updater.ts
@@ -1,7 +1,6 @@
1import { Transaction } from 'sequelize/types' 1import { Transaction } from 'sequelize/types'
2import { resetSequelizeInstance } from '@server/helpers/database-utils' 2import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils'
3import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' 3import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { Notifier } from '@server/lib/notifier' 4import { Notifier } from '@server/lib/notifier'
6import { PeerTubeSocket } from '@server/lib/peertube-socket' 5import { PeerTubeSocket } from '@server/lib/peertube-socket'
7import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' 6import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
@@ -48,24 +47,26 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
48 47
49 const thumbnailModel = await this.tryToGenerateThumbnail(this.video) 48 const thumbnailModel = await this.tryToGenerateThumbnail(this.video)
50 49
51 const videoUpdated = await sequelizeTypescript.transaction(async t => { 50 this.checkChannelUpdateOrThrow(channelActor)
52 this.checkChannelUpdateOrThrow(channelActor)
53 51
54 const videoUpdated = await this.updateVideo(channelActor.VideoChannel, t, overrideTo) 52 const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo)
55 53
56 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel, t) 54 if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel)
57 55
58 await this.setPreview(videoUpdated, t) 56 await runInReadCommittedTransaction(async t => {
59 await this.setWebTorrentFiles(videoUpdated, t) 57 await this.setWebTorrentFiles(videoUpdated, t)
60 await this.setStreamingPlaylists(videoUpdated, t) 58 await this.setStreamingPlaylists(videoUpdated, t)
61 await this.setTags(videoUpdated, t)
62 await this.setTrackers(videoUpdated, t)
63 await this.setCaptions(videoUpdated, t)
64 await this.setOrDeleteLive(videoUpdated, t)
65
66 return videoUpdated
67 }) 59 })
68 60
61 await Promise.all([
62 runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)),
63 runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)),
64 this.setOrDeleteLive(videoUpdated),
65 this.setPreview(videoUpdated)
66 ])
67
68 await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t))
69
69 await autoBlacklistVideoIfNeeded({ 70 await autoBlacklistVideoIfNeeded({
70 video: videoUpdated, 71 video: videoUpdated,
71 user: undefined, 72 user: undefined,
@@ -103,7 +104,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
103 } 104 }
104 } 105 }
105 106
106 private updateVideo (channel: MChannelId, transaction: Transaction, overrideTo?: string[]) { 107 private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) {
107 const to = overrideTo || this.videoObject.to 108 const to = overrideTo || this.videoObject.to
108 const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) 109 const videoData = getVideoAttributesFromObject(channel, this.videoObject, to)
109 this.video.name = videoData.name 110 this.video.name = videoData.name
@@ -140,7 +141,9 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
140 await this.insertOrReplaceCaptions(videoUpdated, t) 141 await this.insertOrReplaceCaptions(videoUpdated, t)
141 } 142 }
142 143
143 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction: Transaction) { 144 private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) {
145 if (!this.video.isLive) return
146
144 if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) 147 if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction)
145 148
146 // Delete existing live if it exists 149 // Delete existing live if it exists
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 8f561116b..44aaa24ef 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1886,7 +1886,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1886 return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0 1886 return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0
1887 } 1887 }
1888 1888
1889 async addAndSaveThumbnail (thumbnail: MThumbnail, transaction: Transaction) { 1889 async addAndSaveThumbnail (thumbnail: MThumbnail, transaction?: Transaction) {
1890 thumbnail.videoId = this.id 1890 thumbnail.videoId = this.id
1891 1891
1892 const savedThumbnail = await thumbnail.save({ transaction }) 1892 const savedThumbnail = await thumbnail.save({ transaction })