aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/actor.ts67
-rw-r--r--server/lib/activitypub/process/process-update.ts10
-rw-r--r--server/lib/avatar.ts38
-rw-r--r--server/lib/oauth-model.ts34
4 files changed, 97 insertions, 52 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 04296864b..9f5d12eb4 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
10import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' 10import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' 12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13import { doRequest, downloadImage } from '../../helpers/requests' 13import { doRequest } from '../../helpers/requests'
14import { getUrlFromWebfinger } from '../../helpers/webfinger' 14import { getUrlFromWebfinger } from '../../helpers/webfinger'
15import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants' 15import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
16import { AccountModel } from '../../models/account/account' 16import { AccountModel } from '../../models/account/account'
17import { ActorModel } from '../../models/activitypub/actor' 17import { ActorModel } from '../../models/activitypub/actor'
18import { AvatarModel } from '../../models/avatar/avatar' 18import { AvatarModel } from '../../models/avatar/avatar'
@@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel'
21import { JobQueue } from '../job-queue' 21import { JobQueue } from '../job-queue'
22import { getServerActor } from '../../helpers/utils' 22import { getServerActor } from '../../helpers/utils'
23import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' 23import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24import { CONFIG } from '../../initializers/config'
25import { sequelizeTypescript } from '../../initializers/database' 24import { sequelizeTypescript } from '../../initializers/database'
26 25
27// Set account keys, this could be long so process after the account creation and do not block the client 26// Set account keys, this could be long so process after the account creation and do not block the client
@@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ
141 actorInstance.followingUrl = attributes.following 140 actorInstance.followingUrl = attributes.following
142} 141}
143 142
144async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) { 143async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) {
145 if (avatarName !== undefined) { 144 if (info.name !== undefined) {
146 if (actorInstance.avatarId) { 145 if (actor.avatarId) {
147 try { 146 try {
148 await actorInstance.Avatar.destroy({ transaction: t }) 147 await actor.Avatar.destroy({ transaction: t })
149 } catch (err) { 148 } catch (err) {
150 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err }) 149 logger.error('Cannot remove old avatar of actor %s.', actor.url, { err })
151 } 150 }
152 } 151 }
153 152
154 const avatar = await AvatarModel.create({ 153 const avatar = await AvatarModel.create({
155 filename: avatarName 154 filename: info.name,
155 onDisk: info.onDisk,
156 fileUrl: info.fileUrl
156 }, { transaction: t }) 157 }, { transaction: t })
157 158
158 actorInstance.set('avatarId', avatar.id) 159 actor.avatarId = avatar.id
159 actorInstance.Avatar = avatar 160 actor.Avatar = avatar
160 } 161 }
161 162
162 return actorInstance 163 return actor
163} 164}
164 165
165async function fetchActorTotalItems (url: string) { 166async function fetchActorTotalItems (url: string) {
@@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) {
179 } 180 }
180} 181}
181 182
182async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { 183async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) {
183 if ( 184 if (
184 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && 185 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
185 isActivityPubUrlValid(actorJSON.icon.url) 186 isActivityPubUrlValid(actorJSON.icon.url)
186 ) { 187 ) {
187 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] 188 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
188 189
189 const avatarName = uuidv4() + extension 190 return {
190 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) 191 name: uuidv4() + extension,
191 192 fileUrl: actorJSON.icon.url
192 return avatarName 193 }
193 } 194 }
194 195
195 return undefined 196 return undefined
@@ -245,8 +246,14 @@ async function refreshActorIfNeeded (
245 return sequelizeTypescript.transaction(async t => { 246 return sequelizeTypescript.transaction(async t => {
246 updateInstanceWithAnother(actor, result.actor) 247 updateInstanceWithAnother(actor, result.actor)
247 248
248 if (result.avatarName !== undefined) { 249 if (result.avatar !== undefined) {
249 await updateActorAvatarInstance(actor, result.avatarName, t) 250 const avatarInfo = {
251 name: result.avatar.name,
252 fileUrl: result.avatar.fileUrl,
253 onDisk: false
254 }
255
256 await updateActorAvatarInstance(actor, avatarInfo, t)
250 } 257 }
251 258
252 // Force update 259 // Force update
@@ -279,7 +286,7 @@ export {
279 buildActorInstance, 286 buildActorInstance,
280 setAsyncActorKeys, 287 setAsyncActorKeys,
281 fetchActorTotalItems, 288 fetchActorTotalItems,
282 fetchAvatarIfExists, 289 getAvatarInfoIfExists,
283 updateActorInstance, 290 updateActorInstance,
284 refreshActorIfNeeded, 291 refreshActorIfNeeded,
285 updateActorAvatarInstance, 292 updateActorAvatarInstance,
@@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist (
314 const [ server ] = await ServerModel.findOrCreate(serverOptions) 321 const [ server ] = await ServerModel.findOrCreate(serverOptions)
315 322
316 // Save our new account in database 323 // Save our new account in database
317 actor.set('serverId', server.id) 324 actor.serverId = server.id
318 325
319 // Avatar? 326 // Avatar?
320 if (result.avatarName) { 327 if (result.avatar) {
321 const avatar = await AvatarModel.create({ 328 const avatar = await AvatarModel.create({
322 filename: result.avatarName 329 filename: result.avatar.name,
330 fileUrl: result.avatar.fileUrl,
331 onDisk: false
323 }, { transaction: t }) 332 }, { transaction: t })
324 actor.set('avatarId', avatar.id) 333
334 actor.avatarId = avatar.id
325 } 335 }
326 336
327 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists 337 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
@@ -355,7 +365,10 @@ type FetchRemoteActorResult = {
355 summary: string 365 summary: string
356 support?: string 366 support?: string
357 playlists?: string 367 playlists?: string
358 avatarName?: string 368 avatar?: {
369 name: string,
370 fileUrl: string
371 }
359 attributedTo: ActivityPubAttributedTo[] 372 attributedTo: ActivityPubAttributedTo[]
360} 373}
361async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { 374async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
@@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
399 followingUrl: actorJSON.following 412 followingUrl: actorJSON.following
400 }) 413 })
401 414
402 const avatarName = await fetchAvatarIfExists(actorJSON) 415 const avatarInfo = await getAvatarInfoIfExists(actorJSON)
403 416
404 const name = actorJSON.name || actorJSON.preferredUsername 417 const name = actorJSON.name || actorJSON.preferredUsername
405 return { 418 return {
@@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
407 result: { 420 result: {
408 actor, 421 actor,
409 name, 422 name,
410 avatarName, 423 avatar: avatarInfo,
411 summary: actorJSON.summary, 424 summary: actorJSON.summary,
412 support: actorJSON.support, 425 support: actorJSON.support,
413 playlists: actorJSON.playlists, 426 playlists: actorJSON.playlists,
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index e3c862221..414f9e375 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers'
6import { AccountModel } from '../../../models/account/account' 6import { AccountModel } from '../../../models/account/account'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' 9import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' 10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' 11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
@@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
105 let accountOrChannelFieldsSave: object 105 let accountOrChannelFieldsSave: object
106 106
107 // Fetch icon? 107 // Fetch icon?
108 const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate) 108 const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate)
109 109
110 try { 110 try {
111 await sequelizeTypescript.transaction(async t => { 111 await sequelizeTypescript.transaction(async t => {
@@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
118 118
119 await updateActorInstance(actor, actorAttributesToUpdate) 119 await updateActorInstance(actor, actorAttributesToUpdate)
120 120
121 if (avatarName !== undefined) { 121 if (avatarInfo !== undefined) {
122 await updateActorAvatarInstance(actor, avatarName, t) 122 const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false })
123
124 await updateActorAvatarInstance(actor, avatarOptions, t)
123 } 125 }
124 126
125 await actor.save({ transaction: t }) 127 await actor.save({ transaction: t })
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 09b4e38ca..1b38e6cb5 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -1,6 +1,6 @@
1import 'multer' 1import 'multer'
2import { sendUpdateActor } from './activitypub/send' 2import { sendUpdateActor } from './activitypub/send'
3import { AVATARS_SIZE } from '../initializers/constants' 3import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants'
4import { updateActorAvatarInstance } from './activitypub' 4import { updateActorAvatarInstance } from './activitypub'
5import { processImage } from '../helpers/image-utils' 5import { processImage } from '../helpers/image-utils'
6import { AccountModel } from '../models/account/account' 6import { AccountModel } from '../models/account/account'
@@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils'
10import * as uuidv4 from 'uuid/v4' 10import * as uuidv4 from 'uuid/v4'
11import { CONFIG } from '../initializers/config' 11import { CONFIG } from '../initializers/config'
12import { sequelizeTypescript } from '../initializers/database' 12import { sequelizeTypescript } from '../initializers/database'
13import * as LRUCache from 'lru-cache'
14import { queue } from 'async'
15import { downloadImage } from '../helpers/requests'
13 16
14async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { 17async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
15 const extension = extname(avatarPhysicalFile.filename) 18 const extension = extname(avatarPhysicalFile.filename)
@@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
19 22
20 return retryTransactionWrapper(() => { 23 return retryTransactionWrapper(() => {
21 return sequelizeTypescript.transaction(async t => { 24 return sequelizeTypescript.transaction(async t => {
22 const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t) 25 const avatarInfo = {
26 name: avatarName,
27 fileUrl: null,
28 onDisk: true
29 }
30
31 const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t)
23 await updatedActor.save({ transaction: t }) 32 await updatedActor.save({ transaction: t })
24 33
25 await sendUpdateActor(accountOrChannel, t) 34 await sendUpdateActor(accountOrChannel, t)
@@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
29 }) 38 })
30} 39}
31 40
41type DownloadImageQueueTask = { fileUrl: string, filename: string }
42
43const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => {
44 downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE)
45 .then(() => cb())
46 .catch(err => cb(err))
47}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE)
48
49function pushAvatarProcessInQueue (task: DownloadImageQueueTask) {
50 return new Promise((res, rej) => {
51 downloadImageQueue.push(task, err => {
52 if (err) return rej(err)
53
54 return res()
55 })
56 })
57}
58
59// Unsafe so could returns paths that does not exist anymore
60const avatarPathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE })
61
32export { 62export {
33 updateActorAvatarFile 63 avatarPathUnsafeCache,
64 updateActorAvatarFile,
65 pushAvatarProcessInQueue
34} 66}
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 45ac3e7c4..a1153e88a 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,13 +4,15 @@ import { logger } from '../helpers/logger'
4import { UserModel } from '../models/account/user' 4import { UserModel } from '../models/account/user'
5import { OAuthClientModel } from '../models/oauth/oauth-client' 5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { OAuthTokenModel } from '../models/oauth/oauth-token' 6import { OAuthTokenModel } from '../models/oauth/oauth-token'
7import { CACHE } from '../initializers/constants' 7import { LRU_CACHE } from '../initializers/constants'
8import { Transaction } from 'sequelize' 8import { Transaction } from 'sequelize'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import * as LRUCache from 'lru-cache'
10 11
11type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } 12type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
12let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} 13
13let userHavingToken: { [ userId: number ]: string } = {} 14const accessTokenCache = new LRUCache<string, OAuthTokenModel>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
15const userHavingToken = new LRUCache<number, string>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
14 16
15// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
16 18
@@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) {
21} 23}
22 24
23function clearCacheByUserId (userId: number) { 25function clearCacheByUserId (userId: number) {
24 const token = userHavingToken[userId] 26 const token = userHavingToken.get(userId)
27
25 if (token !== undefined) { 28 if (token !== undefined) {
26 accessTokenCache[ token ] = undefined 29 accessTokenCache.del(token)
27 userHavingToken[ userId ] = undefined 30 userHavingToken.del(userId)
28 } 31 }
29} 32}
30 33
31function clearCacheByToken (token: string) { 34function clearCacheByToken (token: string) {
32 const tokenModel = accessTokenCache[ token ] 35 const tokenModel = accessTokenCache.get(token)
36
33 if (tokenModel !== undefined) { 37 if (tokenModel !== undefined) {
34 userHavingToken[tokenModel.userId] = undefined 38 userHavingToken.del(tokenModel.userId)
35 accessTokenCache[ token ] = undefined 39 accessTokenCache.del(token)
36 } 40 }
37} 41}
38 42
@@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) {
41 45
42 if (!bearerToken) return Bluebird.resolve(undefined) 46 if (!bearerToken) return Bluebird.resolve(undefined)
43 47
44 if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) 48 if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken))
45 49
46 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 50 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
47 .then(tokenModel => { 51 .then(tokenModel => {
48 if (tokenModel) { 52 if (tokenModel) {
49 // Reinit our cache 53 accessTokenCache.set(bearerToken, tokenModel)
50 if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) { 54 userHavingToken.set(tokenModel.userId, tokenModel.accessToken)
51 accessTokenCache = {}
52 userHavingToken = {}
53 }
54
55 accessTokenCache[ bearerToken ] = tokenModel
56 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
57 } 55 }
58 56
59 return tokenModel 57 return tokenModel