From 557b13ae24019d9ab214bbea7eaa0f892c8f4b05 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 9 Aug 2019 11:32:40 +0200 Subject: Lazy load avatars --- server/lib/activitypub/actor.ts | 67 ++++++++++++++---------- server/lib/activitypub/process/process-update.ts | 10 ++-- server/lib/avatar.ts | 38 ++++++++++++-- server/lib/oauth-model.ts | 34 ++++++------ 4 files changed, 97 insertions(+), 52 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 04296864b..9f5d12eb4 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest, downloadImage } from '../../helpers/requests' +import { doRequest } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { MIMETYPES, WEBSERVER } from '../../initializers/constants' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' @@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel' import { JobQueue } from '../job-queue' import { getServerActor } from '../../helpers/utils' import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' -import { CONFIG } from '../../initializers/config' import { sequelizeTypescript } from '../../initializers/database' // Set account keys, this could be long so process after the account creation and do not block the client @@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ actorInstance.followingUrl = attributes.following } -async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) { - if (avatarName !== undefined) { - if (actorInstance.avatarId) { +async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) { + if (info.name !== undefined) { + if (actor.avatarId) { try { - await actorInstance.Avatar.destroy({ transaction: t }) + await actor.Avatar.destroy({ transaction: t }) } catch (err) { - logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err }) + logger.error('Cannot remove old avatar of actor %s.', actor.url, { err }) } } const avatar = await AvatarModel.create({ - filename: avatarName + filename: info.name, + onDisk: info.onDisk, + fileUrl: info.fileUrl }, { transaction: t }) - actorInstance.set('avatarId', avatar.id) - actorInstance.Avatar = avatar + actor.avatarId = avatar.id + actor.Avatar = avatar } - return actorInstance + return actor } async function fetchActorTotalItems (url: string) { @@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) { } } -async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { +async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) { if ( actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && isActivityPubUrlValid(actorJSON.icon.url) ) { const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] - const avatarName = uuidv4() + extension - await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) - - return avatarName + return { + name: uuidv4() + extension, + fileUrl: actorJSON.icon.url + } } return undefined @@ -245,8 +246,14 @@ async function refreshActorIfNeeded ( return sequelizeTypescript.transaction(async t => { updateInstanceWithAnother(actor, result.actor) - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) + if (result.avatar !== undefined) { + const avatarInfo = { + name: result.avatar.name, + fileUrl: result.avatar.fileUrl, + onDisk: false + } + + await updateActorAvatarInstance(actor, avatarInfo, t) } // Force update @@ -279,7 +286,7 @@ export { buildActorInstance, setAsyncActorKeys, fetchActorTotalItems, - fetchAvatarIfExists, + getAvatarInfoIfExists, updateActorInstance, refreshActorIfNeeded, updateActorAvatarInstance, @@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist ( const [ server ] = await ServerModel.findOrCreate(serverOptions) // Save our new account in database - actor.set('serverId', server.id) + actor.serverId = server.id // Avatar? - if (result.avatarName) { + if (result.avatar) { const avatar = await AvatarModel.create({ - filename: result.avatarName + filename: result.avatar.name, + fileUrl: result.avatar.fileUrl, + onDisk: false }, { transaction: t }) - actor.set('avatarId', avatar.id) + + actor.avatarId = avatar.id } // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists @@ -355,7 +365,10 @@ type FetchRemoteActorResult = { summary: string support?: string playlists?: string - avatarName?: string + avatar?: { + name: string, + fileUrl: string + } attributedTo: ActivityPubAttributedTo[] } async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { @@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe followingUrl: actorJSON.following }) - const avatarName = await fetchAvatarIfExists(actorJSON) + const avatarInfo = await getAvatarInfoIfExists(actorJSON) const name = actorJSON.name || actorJSON.preferredUsername return { @@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe result: { actor, name, - avatarName, + avatar: avatarInfo, summary: actorJSON.summary, support: actorJSON.support, playlists: actorJSON.playlists, diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index e3c862221..414f9e375 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' import { VideoChannelModel } from '../../../models/video/video-channel' -import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' +import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' @@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) let accountOrChannelFieldsSave: object // Fetch icon? - const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate) + const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate) try { await sequelizeTypescript.transaction(async t => { @@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) await updateActorInstance(actor, actorAttributesToUpdate) - if (avatarName !== undefined) { - await updateActorAvatarInstance(actor, avatarName, t) + if (avatarInfo !== undefined) { + const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false }) + + await updateActorAvatarInstance(actor, avatarOptions, t) } await actor.save({ transaction: t }) diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts index 09b4e38ca..1b38e6cb5 100644 --- a/server/lib/avatar.ts +++ b/server/lib/avatar.ts @@ -1,6 +1,6 @@ import 'multer' import { sendUpdateActor } from './activitypub/send' -import { AVATARS_SIZE } from '../initializers/constants' +import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants' import { updateActorAvatarInstance } from './activitypub' import { processImage } from '../helpers/image-utils' import { AccountModel } from '../models/account/account' @@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils' import * as uuidv4 from 'uuid/v4' import { CONFIG } from '../initializers/config' import { sequelizeTypescript } from '../initializers/database' +import * as LRUCache from 'lru-cache' +import { queue } from 'async' +import { downloadImage } from '../helpers/requests' async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { const extension = extname(avatarPhysicalFile.filename) @@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a return retryTransactionWrapper(() => { return sequelizeTypescript.transaction(async t => { - const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t) + const avatarInfo = { + name: avatarName, + fileUrl: null, + onDisk: true + } + + const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t) await updatedActor.save({ transaction: t }) await sendUpdateActor(accountOrChannel, t) @@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a }) } +type DownloadImageQueueTask = { fileUrl: string, filename: string } + +const downloadImageQueue = queue((task, cb) => { + downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE) + .then(() => cb()) + .catch(err => cb(err)) +}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE) + +function pushAvatarProcessInQueue (task: DownloadImageQueueTask) { + return new Promise((res, rej) => { + downloadImageQueue.push(task, err => { + if (err) return rej(err) + + return res() + }) + }) +} + +// Unsafe so could returns paths that does not exist anymore +const avatarPathUnsafeCache = new LRUCache({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE }) + export { - updateActorAvatarFile + avatarPathUnsafeCache, + updateActorAvatarFile, + pushAvatarProcessInQueue } diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 45ac3e7c4..a1153e88a 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts @@ -4,13 +4,15 @@ import { logger } from '../helpers/logger' import { UserModel } from '../models/account/user' import { OAuthClientModel } from '../models/oauth/oauth-client' import { OAuthTokenModel } from '../models/oauth/oauth-token' -import { CACHE } from '../initializers/constants' +import { LRU_CACHE } from '../initializers/constants' import { Transaction } from 'sequelize' import { CONFIG } from '../initializers/config' +import * as LRUCache from 'lru-cache' type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } -let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} -let userHavingToken: { [ userId: number ]: string } = {} + +const accessTokenCache = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) +const userHavingToken = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) // --------------------------------------------------------------------------- @@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) { } function clearCacheByUserId (userId: number) { - const token = userHavingToken[userId] + const token = userHavingToken.get(userId) + if (token !== undefined) { - accessTokenCache[ token ] = undefined - userHavingToken[ userId ] = undefined + accessTokenCache.del(token) + userHavingToken.del(userId) } } function clearCacheByToken (token: string) { - const tokenModel = accessTokenCache[ token ] + const tokenModel = accessTokenCache.get(token) + if (tokenModel !== undefined) { - userHavingToken[tokenModel.userId] = undefined - accessTokenCache[ token ] = undefined + userHavingToken.del(tokenModel.userId) + accessTokenCache.del(token) } } @@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) { if (!bearerToken) return Bluebird.resolve(undefined) - if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) + if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken)) return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) .then(tokenModel => { if (tokenModel) { - // Reinit our cache - if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) { - accessTokenCache = {} - userHavingToken = {} - } - - accessTokenCache[ bearerToken ] = tokenModel - userHavingToken[ tokenModel.userId ] = tokenModel.accessToken + accessTokenCache.set(bearerToken, tokenModel) + userHavingToken.set(tokenModel.userId, tokenModel.accessToken) } return tokenModel -- cgit v1.2.3