Lazy load avatars

4 files changed, 97 insertions, 52 deletions

diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 04296864b..9f5d12eb4 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
-import { doRequest, downloadImage } from '../../helpers/requests'
+import { doRequest } from '../../helpers/requests'
 import { getUrlFromWebfinger } from '../../helpers/webfinger'
-import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants'
+import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
 import { AccountModel } from '../../models/account/account'
 import { ActorModel } from '../../models/activitypub/actor'
 import { AvatarModel } from '../../models/avatar/avatar'
@@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel'
 import { JobQueue } from '../job-queue'
 import { getServerActor } from '../../helpers/utils'
 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
-import { CONFIG } from '../../initializers/config'
 import { sequelizeTypescript } from '../../initializers/database'
 
 // Set account keys, this could be long so process after the account creation and do not block the client
@@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ
   actorInstance.followingUrl = attributes.following
 }
 
-async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
-  if (avatarName !== undefined) {
-    if (actorInstance.avatarId) {
+async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) {
+  if (info.name !== undefined) {
+    if (actor.avatarId) {
       try {
-        await actorInstance.Avatar.destroy({ transaction: t })
+        await actor.Avatar.destroy({ transaction: t })
       } catch (err) {
-        logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
+        logger.error('Cannot remove old avatar of actor %s.', actor.url, { err })
       }
     }
 
     const avatar = await AvatarModel.create({
-      filename: avatarName
+      filename: info.name,
+      onDisk: info.onDisk,
+      fileUrl: info.fileUrl
     }, { transaction: t })
 
-    actorInstance.set('avatarId', avatar.id)
-    actorInstance.Avatar = avatar
+    actor.avatarId = avatar.id
+    actor.Avatar = avatar
   }
 
-  return actorInstance
+  return actor
 }
 
 async function fetchActorTotalItems (url: string) {
@@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) {
   }
 }
 
-async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
+async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) {
   if (
     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
     isActivityPubUrlValid(actorJSON.icon.url)
   ) {
     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 
-    const avatarName = uuidv4() + extension
-    await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
-
-    return avatarName
+    return {
+      name: uuidv4() + extension,
+      fileUrl: actorJSON.icon.url
+    }
   }
 
   return undefined
@@ -245,8 +246,14 @@ async function refreshActorIfNeeded (
     return sequelizeTypescript.transaction(async t => {
       updateInstanceWithAnother(actor, result.actor)
 
-      if (result.avatarName !== undefined) {
-        await updateActorAvatarInstance(actor, result.avatarName, t)
+      if (result.avatar !== undefined) {
+        const avatarInfo = {
+          name: result.avatar.name,
+          fileUrl: result.avatar.fileUrl,
+          onDisk: false
+        }
+
+        await updateActorAvatarInstance(actor, avatarInfo, t)
       }
 
       // Force update
@@ -279,7 +286,7 @@ export {
   buildActorInstance,
   setAsyncActorKeys,
   fetchActorTotalItems,
-  fetchAvatarIfExists,
+  getAvatarInfoIfExists,
   updateActorInstance,
   refreshActorIfNeeded,
   updateActorAvatarInstance,
@@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist (
     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 
     // Save our new account in database
-    actor.set('serverId', server.id)
+    actor.serverId = server.id
 
     // Avatar?
-    if (result.avatarName) {
+    if (result.avatar) {
       const avatar = await AvatarModel.create({
-        filename: result.avatarName
+        filename: result.avatar.name,
+        fileUrl: result.avatar.fileUrl,
+        onDisk: false
       }, { transaction: t })
-      actor.set('avatarId', avatar.id)
+
+      actor.avatarId = avatar.id
     }
 
     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
@@ -355,7 +365,10 @@ type FetchRemoteActorResult = {
   summary: string
   support?: string
   playlists?: string
-  avatarName?: string
+  avatar?: {
+    name: string,
+    fileUrl: string
+  }
   attributedTo: ActivityPubAttributedTo[]
 }
 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
@@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
     followingUrl: actorJSON.following
   })
 
-  const avatarName = await fetchAvatarIfExists(actorJSON)
+  const avatarInfo = await getAvatarInfoIfExists(actorJSON)
 
   const name = actorJSON.name || actorJSON.preferredUsername
   return {
@@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
     result: {
       actor,
       name,
-      avatarName,
+      avatar: avatarInfo,
       summary: actorJSON.summary,
       support: actorJSON.support,
       playlists: actorJSON.playlists,
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index e3c862221..414f9e375 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers'
 import { AccountModel } from '../../../models/account/account'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoChannelModel } from '../../../models/video/video-channel'
-import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
+import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
 import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
 import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
@@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
   let accountOrChannelFieldsSave: object
 
   // Fetch icon?
-  const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate)
+  const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate)
 
   try {
     await sequelizeTypescript.transaction(async t => {
@@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
 
       await updateActorInstance(actor, actorAttributesToUpdate)
 
-      if (avatarName !== undefined) {
-        await updateActorAvatarInstance(actor, avatarName, t)
+      if (avatarInfo !== undefined) {
+        const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false })
+
+        await updateActorAvatarInstance(actor, avatarOptions, t)
       }
 
       await actor.save({ transaction: t })
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 09b4e38ca..1b38e6cb5 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -1,6 +1,6 @@
 import 'multer'
 import { sendUpdateActor } from './activitypub/send'
-import { AVATARS_SIZE } from '../initializers/constants'
+import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants'
 import { updateActorAvatarInstance } from './activitypub'
 import { processImage } from '../helpers/image-utils'
 import { AccountModel } from '../models/account/account'
@@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils'
 import * as uuidv4 from 'uuid/v4'
 import { CONFIG } from '../initializers/config'
 import { sequelizeTypescript } from '../initializers/database'
+import * as LRUCache from 'lru-cache'
+import { queue } from 'async'
+import { downloadImage } from '../helpers/requests'
 
 async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
   const extension = extname(avatarPhysicalFile.filename)
@@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
 
   return retryTransactionWrapper(() => {
     return sequelizeTypescript.transaction(async t => {
-      const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t)
+      const avatarInfo = {
+        name: avatarName,
+        fileUrl: null,
+        onDisk: true
+      }
+
+      const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t)
       await updatedActor.save({ transaction: t })
 
       await sendUpdateActor(accountOrChannel, t)
@@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
   })
 }
 
+type DownloadImageQueueTask = { fileUrl: string, filename: string }
+
+const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => {
+  downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE)
+    .then(() => cb())
+    .catch(err => cb(err))
+}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE)
+
+function pushAvatarProcessInQueue (task: DownloadImageQueueTask) {
+  return new Promise((res, rej) => {
+    downloadImageQueue.push(task, err => {
+      if (err) return rej(err)
+
+      return res()
+    })
+  })
+}
+
+// Unsafe so could returns paths that does not exist anymore
+const avatarPathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE })
+
 export {
-  updateActorAvatarFile
+  avatarPathUnsafeCache,
+  updateActorAvatarFile,
+  pushAvatarProcessInQueue
 }
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 45ac3e7c4..a1153e88a 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,13 +4,15 @@ import { logger } from '../helpers/logger'
 import { UserModel } from '../models/account/user'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { OAuthTokenModel } from '../models/oauth/oauth-token'
-import { CACHE } from '../initializers/constants'
+import { LRU_CACHE } from '../initializers/constants'
 import { Transaction } from 'sequelize'
 import { CONFIG } from '../initializers/config'
+import * as LRUCache from 'lru-cache'
 
 type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
-let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
-let userHavingToken: { [ userId: number ]: string } = {}
+
+const accessTokenCache = new LRUCache<string, OAuthTokenModel>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
+const userHavingToken = new LRUCache<number, string>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
 
 // ---------------------------------------------------------------------------
 
@@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) {
 }
 
 function clearCacheByUserId (userId: number) {
-  const token = userHavingToken[userId]
+  const token = userHavingToken.get(userId)
+
   if (token !== undefined) {
-    accessTokenCache[ token ] = undefined
-    userHavingToken[ userId ] = undefined
+    accessTokenCache.del(token)
+    userHavingToken.del(userId)
   }
 }
 
 function clearCacheByToken (token: string) {
-  const tokenModel = accessTokenCache[ token ]
+  const tokenModel = accessTokenCache.get(token)
+
   if (tokenModel !== undefined) {
-    userHavingToken[tokenModel.userId] = undefined
-    accessTokenCache[ token ] = undefined
+    userHavingToken.del(tokenModel.userId)
+    accessTokenCache.del(token)
   }
 }
 
@@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) {
 
   if (!bearerToken) return Bluebird.resolve(undefined)
 
-  if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken])
+  if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken))
 
   return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
     .then(tokenModel => {
       if (tokenModel) {
-        // Reinit our cache
-        if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
-          accessTokenCache = {}
-          userHavingToken = {}
-        }
-
-        accessTokenCache[ bearerToken ] = tokenModel
-        userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
+        accessTokenCache.set(bearerToken, tokenModel)
+        userHavingToken.set(tokenModel.userId, tokenModel.accessToken)
       }
 
       return tokenModel