aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-08-09 11:32:40 +0200
committerChocobozzz <me@florianbigard.com>2019-08-09 11:32:40 +0200
commit557b13ae24019d9ab214bbea7eaa0f892c8f4b05 (patch)
treeaa32396531acf93e3dfdb29880177813039ed77f /server
parentc5407d7046168abb4098df1408e7aa84519cb61a (diff)
downloadPeerTube-557b13ae24019d9ab214bbea7eaa0f892c8f4b05.tar.gz
PeerTube-557b13ae24019d9ab214bbea7eaa0f892c8f4b05.tar.zst
PeerTube-557b13ae24019d9ab214bbea7eaa0f892c8f4b05.zip
Lazy load avatars
Diffstat (limited to 'server')
-rw-r--r--server/controllers/index.ts1
-rw-r--r--server/controllers/lazy-static.ts80
-rw-r--r--server/controllers/static.ts24
-rw-r--r--server/initializers/constants.ts22
-rw-r--r--server/initializers/migrations/0420-avatar-lazy.ts60
-rw-r--r--server/lib/activitypub/actor.ts67
-rw-r--r--server/lib/activitypub/process/process-update.ts10
-rw-r--r--server/lib/avatar.ts38
-rw-r--r--server/lib/oauth-model.ts34
-rw-r--r--server/models/account/user-notification.ts4
-rw-r--r--server/models/activitypub/actor.ts2
-rw-r--r--server/models/avatar/avatar.ts43
-rw-r--r--server/models/video/thumbnail.ts4
-rw-r--r--server/models/video/video-caption.ts4
-rw-r--r--server/models/video/video.ts3
15 files changed, 306 insertions, 90 deletions
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index 8b3501712..0d64b33bb 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -4,6 +4,7 @@ export * from './client'
4export * from './feeds' 4export * from './feeds'
5export * from './services' 5export * from './services'
6export * from './static' 6export * from './static'
7export * from './lazy-static'
7export * from './webfinger' 8export * from './webfinger'
8export * from './tracker' 9export * from './tracker'
9export * from './bots' 10export * from './bots'
diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts
new file mode 100644
index 000000000..4285fd727
--- /dev/null
+++ b/server/controllers/lazy-static.ts
@@ -0,0 +1,80 @@
1import * as cors from 'cors'
2import * as express from 'express'
3import { LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
4import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
5import { asyncMiddleware } from '../middlewares'
6import { AvatarModel } from '../models/avatar/avatar'
7import { logger } from '../helpers/logger'
8import { avatarPathUnsafeCache, pushAvatarProcessInQueue } from '../lib/avatar'
9
10const lazyStaticRouter = express.Router()
11
12lazyStaticRouter.use(cors())
13
14lazyStaticRouter.use(
15 LAZY_STATIC_PATHS.AVATARS + ':filename',
16 asyncMiddleware(getAvatar)
17)
18
19lazyStaticRouter.use(
20 LAZY_STATIC_PATHS.PREVIEWS + ':uuid.jpg',
21 asyncMiddleware(getPreview)
22)
23
24lazyStaticRouter.use(
25 LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt',
26 asyncMiddleware(getVideoCaption)
27)
28
29// ---------------------------------------------------------------------------
30
31export {
32 lazyStaticRouter,
33 getPreview,
34 getVideoCaption
35}
36
37// ---------------------------------------------------------------------------
38
39async function getAvatar (req: express.Request, res: express.Response) {
40 const filename = req.params.filename
41
42 if (avatarPathUnsafeCache.has(filename)) {
43 return res.sendFile(avatarPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
44 }
45
46 const avatar = await AvatarModel.loadByName(filename)
47 if (avatar.onDisk === false) {
48 if (!avatar.fileUrl) return res.sendStatus(404)
49
50 logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
51
52 await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
53
54 avatar.onDisk = true
55 avatar.save()
56 .catch(err => logger.error('Cannot save new avatar disk state.', { err }))
57 }
58
59 const path = avatar.getPath()
60
61 avatarPathUnsafeCache.set(filename, path)
62 return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER })
63}
64
65async function getPreview (req: express.Request, res: express.Response) {
66 const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
67 if (!result) return res.sendStatus(404)
68
69 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })
70}
71
72async function getVideoCaption (req: express.Request, res: express.Response) {
73 const result = await VideosCaptionCache.Instance.getFilePath({
74 videoId: req.params.videoId,
75 language: req.params.captionLanguage
76 })
77 if (!result) return res.sendStatus(404)
78
79 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })
80}
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 110d25031..8979ef5f3 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -9,7 +9,6 @@ import {
9 STATIC_PATHS, 9 STATIC_PATHS,
10 WEBSERVER 10 WEBSERVER
11} from '../initializers/constants' 11} from '../initializers/constants'
12import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
13import { cacheRoute } from '../middlewares/cache' 12import { cacheRoute } from '../middlewares/cache'
14import { asyncMiddleware, videosGetValidator } from '../middlewares' 13import { asyncMiddleware, videosGetValidator } from '../middlewares'
15import { VideoModel } from '../models/video/video' 14import { VideoModel } from '../models/video/video'
@@ -19,6 +18,7 @@ import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/node
19import { join } from 'path' 18import { join } from 'path'
20import { root } from '../helpers/core-utils' 19import { root } from '../helpers/core-utils'
21import { CONFIG } from '../initializers/config' 20import { CONFIG } from '../initializers/config'
21import { getPreview, getVideoCaption } from './lazy-static'
22 22
23const staticRouter = express.Router() 23const staticRouter = express.Router()
24 24
@@ -72,19 +72,20 @@ staticRouter.use(
72 express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist 72 express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist
73) 73)
74 74
75// DEPRECATED: use lazy-static route instead
75const avatarsPhysicalPath = CONFIG.STORAGE.AVATARS_DIR 76const avatarsPhysicalPath = CONFIG.STORAGE.AVATARS_DIR
76staticRouter.use( 77staticRouter.use(
77 STATIC_PATHS.AVATARS, 78 STATIC_PATHS.AVATARS,
78 express.static(avatarsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist 79 express.static(avatarsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist
79) 80)
80 81
81// We don't have video previews, fetch them from the origin instance 82// DEPRECATED: use lazy-static route instead
82staticRouter.use( 83staticRouter.use(
83 STATIC_PATHS.PREVIEWS + ':uuid.jpg', 84 STATIC_PATHS.PREVIEWS + ':uuid.jpg',
84 asyncMiddleware(getPreview) 85 asyncMiddleware(getPreview)
85) 86)
86 87
87// We don't have video captions, fetch them from the origin instance 88// DEPRECATED: use lazy-static route instead
88staticRouter.use( 89staticRouter.use(
89 STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt', 90 STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt',
90 asyncMiddleware(getVideoCaption) 91 asyncMiddleware(getVideoCaption)
@@ -177,23 +178,6 @@ export {
177 178
178// --------------------------------------------------------------------------- 179// ---------------------------------------------------------------------------
179 180
180async function getPreview (req: express.Request, res: express.Response) {
181 const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
182 if (!result) return res.sendStatus(404)
183
184 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE })
185}
186
187async function getVideoCaption (req: express.Request, res: express.Response) {
188 const result = await VideosCaptionCache.Instance.getFilePath({
189 videoId: req.params.videoId,
190 language: req.params.captionLanguage
191 })
192 if (!result) return res.sendStatus(404)
193
194 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE })
195}
196
197async function generateNodeinfo (req: express.Request, res: express.Response) { 181async function generateNodeinfo (req: express.Request, res: express.Response) {
198 const { totalVideos } = await VideoModel.getStats() 182 const { totalVideos } = await VideoModel.getStats()
199 const { totalLocalVideoComments } = await VideoCommentModel.getStats() 183 const { totalLocalVideoComments } = await VideoCommentModel.getStats()
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index b9d90b2bd..3dc178b11 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
14 14
15// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
16 16
17const LAST_MIGRATION_VERSION = 415 17const LAST_MIGRATION_VERSION = 420
18 18
19// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
20 20
@@ -498,6 +498,11 @@ const STATIC_DOWNLOAD_PATHS = {
498 TORRENTS: '/download/torrents/', 498 TORRENTS: '/download/torrents/',
499 VIDEOS: '/download/videos/' 499 VIDEOS: '/download/videos/'
500} 500}
501const LAZY_STATIC_PATHS = {
502 AVATARS: '/lazy-static/avatars/',
503 PREVIEWS: '/static/previews/',
504 VIDEO_CAPTIONS: '/static/video-captions/'
505}
501 506
502// Cache control 507// Cache control
503let STATIC_MAX_AGE = { 508let STATIC_MAX_AGE = {
@@ -536,9 +541,12 @@ const FILES_CACHE = {
536 } 541 }
537} 542}
538 543
539const CACHE = { 544const LRU_CACHE = {
540 USER_TOKENS: { 545 USER_TOKENS: {
541 MAX_SIZE: 10000 546 MAX_SIZE: 1000
547 },
548 AVATAR_STATIC: {
549 MAX_SIZE: 500
542 } 550 }
543} 551}
544 552
@@ -549,6 +557,10 @@ const MEMOIZE_TTL = {
549 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours 557 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
550} 558}
551 559
560const QUEUE_CONCURRENCY = {
561 AVATAR_PROCESS_IMAGE: 3
562}
563
552const REDUNDANCY = { 564const REDUNDANCY = {
553 VIDEOS: { 565 VIDEOS: {
554 RANDOMIZED_FACTOR: 5 566 RANDOMIZED_FACTOR: 5
@@ -649,6 +661,7 @@ export {
649 WEBSERVER, 661 WEBSERVER,
650 API_VERSION, 662 API_VERSION,
651 PEERTUBE_VERSION, 663 PEERTUBE_VERSION,
664 LAZY_STATIC_PATHS,
652 HLS_REDUNDANCY_DIRECTORY, 665 HLS_REDUNDANCY_DIRECTORY,
653 P2P_MEDIA_LOADER_PEER_VERSION, 666 P2P_MEDIA_LOADER_PEER_VERSION,
654 AVATARS_SIZE, 667 AVATARS_SIZE,
@@ -695,11 +708,12 @@ export {
695 VIDEO_PRIVACIES, 708 VIDEO_PRIVACIES,
696 VIDEO_LICENCES, 709 VIDEO_LICENCES,
697 VIDEO_STATES, 710 VIDEO_STATES,
711 QUEUE_CONCURRENCY,
698 VIDEO_RATE_TYPES, 712 VIDEO_RATE_TYPES,
699 VIDEO_TRANSCODING_FPS, 713 VIDEO_TRANSCODING_FPS,
700 FFMPEG_NICE, 714 FFMPEG_NICE,
701 VIDEO_ABUSE_STATES, 715 VIDEO_ABUSE_STATES,
702 CACHE, 716 LRU_CACHE,
703 JOB_REQUEST_TIMEOUT, 717 JOB_REQUEST_TIMEOUT,
704 USER_PASSWORD_RESET_LIFETIME, 718 USER_PASSWORD_RESET_LIFETIME,
705 MEMOIZE_TTL, 719 MEMOIZE_TTL,
diff --git a/server/initializers/migrations/0420-avatar-lazy.ts b/server/initializers/migrations/0420-avatar-lazy.ts
new file mode 100644
index 000000000..5fc57aac2
--- /dev/null
+++ b/server/initializers/migrations/0420-avatar-lazy.ts
@@ -0,0 +1,60 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize,
7 db: any
8}): Promise<void> {
9 {
10 // We'll add a unique index on filename, so delete duplicates or PeerTube won't start
11 const query = 'DELETE FROM "avatar" s1 ' +
12 'USING (SELECT MIN(id) as id, filename FROM "avatar" GROUP BY "filename" HAVING COUNT(*) > 1) s2 ' +
13 'WHERE s1."filename" = s2."filename" AND s1.id <> s2.id'
14 await utils.sequelize.query(query)
15 }
16
17 {
18 const data = {
19 type: Sequelize.STRING,
20 allowNull: true,
21 defaultValue: null
22 }
23
24 await utils.queryInterface.addColumn('avatar', 'fileUrl', data)
25 }
26
27 {
28 const data = {
29 type: Sequelize.BOOLEAN,
30 allowNull: true,
31 defaultValue: null
32 }
33
34 await utils.queryInterface.addColumn('avatar', 'onDisk', data)
35 }
36
37 {
38 const query = 'UPDATE "avatar" SET "onDisk" = true;'
39 await utils.sequelize.query(query)
40 }
41
42 {
43 const data = {
44 type: Sequelize.BOOLEAN,
45 allowNull: false,
46 defaultValue: null
47 }
48
49 await utils.queryInterface.changeColumn('avatar', 'onDisk', data)
50 }
51}
52
53function down (options) {
54 throw new Error('Not implemented.')
55}
56
57export {
58 up,
59 down
60}
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 04296864b..9f5d12eb4 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
10import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' 10import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' 12import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13import { doRequest, downloadImage } from '../../helpers/requests' 13import { doRequest } from '../../helpers/requests'
14import { getUrlFromWebfinger } from '../../helpers/webfinger' 14import { getUrlFromWebfinger } from '../../helpers/webfinger'
15import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants' 15import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
16import { AccountModel } from '../../models/account/account' 16import { AccountModel } from '../../models/account/account'
17import { ActorModel } from '../../models/activitypub/actor' 17import { ActorModel } from '../../models/activitypub/actor'
18import { AvatarModel } from '../../models/avatar/avatar' 18import { AvatarModel } from '../../models/avatar/avatar'
@@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel'
21import { JobQueue } from '../job-queue' 21import { JobQueue } from '../job-queue'
22import { getServerActor } from '../../helpers/utils' 22import { getServerActor } from '../../helpers/utils'
23import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' 23import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24import { CONFIG } from '../../initializers/config'
25import { sequelizeTypescript } from '../../initializers/database' 24import { sequelizeTypescript } from '../../initializers/database'
26 25
27// Set account keys, this could be long so process after the account creation and do not block the client 26// Set account keys, this could be long so process after the account creation and do not block the client
@@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ
141 actorInstance.followingUrl = attributes.following 140 actorInstance.followingUrl = attributes.following
142} 141}
143 142
144async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) { 143async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) {
145 if (avatarName !== undefined) { 144 if (info.name !== undefined) {
146 if (actorInstance.avatarId) { 145 if (actor.avatarId) {
147 try { 146 try {
148 await actorInstance.Avatar.destroy({ transaction: t }) 147 await actor.Avatar.destroy({ transaction: t })
149 } catch (err) { 148 } catch (err) {
150 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err }) 149 logger.error('Cannot remove old avatar of actor %s.', actor.url, { err })
151 } 150 }
152 } 151 }
153 152
154 const avatar = await AvatarModel.create({ 153 const avatar = await AvatarModel.create({
155 filename: avatarName 154 filename: info.name,
155 onDisk: info.onDisk,
156 fileUrl: info.fileUrl
156 }, { transaction: t }) 157 }, { transaction: t })
157 158
158 actorInstance.set('avatarId', avatar.id) 159 actor.avatarId = avatar.id
159 actorInstance.Avatar = avatar 160 actor.Avatar = avatar
160 } 161 }
161 162
162 return actorInstance 163 return actor
163} 164}
164 165
165async function fetchActorTotalItems (url: string) { 166async function fetchActorTotalItems (url: string) {
@@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) {
179 } 180 }
180} 181}
181 182
182async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { 183async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) {
183 if ( 184 if (
184 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && 185 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
185 isActivityPubUrlValid(actorJSON.icon.url) 186 isActivityPubUrlValid(actorJSON.icon.url)
186 ) { 187 ) {
187 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] 188 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
188 189
189 const avatarName = uuidv4() + extension 190 return {
190 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) 191 name: uuidv4() + extension,
191 192 fileUrl: actorJSON.icon.url
192 return avatarName 193 }
193 } 194 }
194 195
195 return undefined 196 return undefined
@@ -245,8 +246,14 @@ async function refreshActorIfNeeded (
245 return sequelizeTypescript.transaction(async t => { 246 return sequelizeTypescript.transaction(async t => {
246 updateInstanceWithAnother(actor, result.actor) 247 updateInstanceWithAnother(actor, result.actor)
247 248
248 if (result.avatarName !== undefined) { 249 if (result.avatar !== undefined) {
249 await updateActorAvatarInstance(actor, result.avatarName, t) 250 const avatarInfo = {
251 name: result.avatar.name,
252 fileUrl: result.avatar.fileUrl,
253 onDisk: false
254 }
255
256 await updateActorAvatarInstance(actor, avatarInfo, t)
250 } 257 }
251 258
252 // Force update 259 // Force update
@@ -279,7 +286,7 @@ export {
279 buildActorInstance, 286 buildActorInstance,
280 setAsyncActorKeys, 287 setAsyncActorKeys,
281 fetchActorTotalItems, 288 fetchActorTotalItems,
282 fetchAvatarIfExists, 289 getAvatarInfoIfExists,
283 updateActorInstance, 290 updateActorInstance,
284 refreshActorIfNeeded, 291 refreshActorIfNeeded,
285 updateActorAvatarInstance, 292 updateActorAvatarInstance,
@@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist (
314 const [ server ] = await ServerModel.findOrCreate(serverOptions) 321 const [ server ] = await ServerModel.findOrCreate(serverOptions)
315 322
316 // Save our new account in database 323 // Save our new account in database
317 actor.set('serverId', server.id) 324 actor.serverId = server.id
318 325
319 // Avatar? 326 // Avatar?
320 if (result.avatarName) { 327 if (result.avatar) {
321 const avatar = await AvatarModel.create({ 328 const avatar = await AvatarModel.create({
322 filename: result.avatarName 329 filename: result.avatar.name,
330 fileUrl: result.avatar.fileUrl,
331 onDisk: false
323 }, { transaction: t }) 332 }, { transaction: t })
324 actor.set('avatarId', avatar.id) 333
334 actor.avatarId = avatar.id
325 } 335 }
326 336
327 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists 337 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
@@ -355,7 +365,10 @@ type FetchRemoteActorResult = {
355 summary: string 365 summary: string
356 support?: string 366 support?: string
357 playlists?: string 367 playlists?: string
358 avatarName?: string 368 avatar?: {
369 name: string,
370 fileUrl: string
371 }
359 attributedTo: ActivityPubAttributedTo[] 372 attributedTo: ActivityPubAttributedTo[]
360} 373}
361async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { 374async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
@@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
399 followingUrl: actorJSON.following 412 followingUrl: actorJSON.following
400 }) 413 })
401 414
402 const avatarName = await fetchAvatarIfExists(actorJSON) 415 const avatarInfo = await getAvatarInfoIfExists(actorJSON)
403 416
404 const name = actorJSON.name || actorJSON.preferredUsername 417 const name = actorJSON.name || actorJSON.preferredUsername
405 return { 418 return {
@@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
407 result: { 420 result: {
408 actor, 421 actor,
409 name, 422 name,
410 avatarName, 423 avatar: avatarInfo,
411 summary: actorJSON.summary, 424 summary: actorJSON.summary,
412 support: actorJSON.support, 425 support: actorJSON.support,
413 playlists: actorJSON.playlists, 426 playlists: actorJSON.playlists,
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index e3c862221..414f9e375 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers'
6import { AccountModel } from '../../../models/account/account' 6import { AccountModel } from '../../../models/account/account'
7import { ActorModel } from '../../../models/activitypub/actor' 7import { ActorModel } from '../../../models/activitypub/actor'
8import { VideoChannelModel } from '../../../models/video/video-channel' 8import { VideoChannelModel } from '../../../models/video/video-channel'
9import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor' 9import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' 10import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' 11import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' 12import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
@@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
105 let accountOrChannelFieldsSave: object 105 let accountOrChannelFieldsSave: object
106 106
107 // Fetch icon? 107 // Fetch icon?
108 const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate) 108 const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate)
109 109
110 try { 110 try {
111 await sequelizeTypescript.transaction(async t => { 111 await sequelizeTypescript.transaction(async t => {
@@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
118 118
119 await updateActorInstance(actor, actorAttributesToUpdate) 119 await updateActorInstance(actor, actorAttributesToUpdate)
120 120
121 if (avatarName !== undefined) { 121 if (avatarInfo !== undefined) {
122 await updateActorAvatarInstance(actor, avatarName, t) 122 const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false })
123
124 await updateActorAvatarInstance(actor, avatarOptions, t)
123 } 125 }
124 126
125 await actor.save({ transaction: t }) 127 await actor.save({ transaction: t })
diff --git a/server/lib/avatar.ts b/server/lib/avatar.ts
index 09b4e38ca..1b38e6cb5 100644
--- a/server/lib/avatar.ts
+++ b/server/lib/avatar.ts
@@ -1,6 +1,6 @@
1import 'multer' 1import 'multer'
2import { sendUpdateActor } from './activitypub/send' 2import { sendUpdateActor } from './activitypub/send'
3import { AVATARS_SIZE } from '../initializers/constants' 3import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants'
4import { updateActorAvatarInstance } from './activitypub' 4import { updateActorAvatarInstance } from './activitypub'
5import { processImage } from '../helpers/image-utils' 5import { processImage } from '../helpers/image-utils'
6import { AccountModel } from '../models/account/account' 6import { AccountModel } from '../models/account/account'
@@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils'
10import * as uuidv4 from 'uuid/v4' 10import * as uuidv4 from 'uuid/v4'
11import { CONFIG } from '../initializers/config' 11import { CONFIG } from '../initializers/config'
12import { sequelizeTypescript } from '../initializers/database' 12import { sequelizeTypescript } from '../initializers/database'
13import * as LRUCache from 'lru-cache'
14import { queue } from 'async'
15import { downloadImage } from '../helpers/requests'
13 16
14async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) { 17async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
15 const extension = extname(avatarPhysicalFile.filename) 18 const extension = extname(avatarPhysicalFile.filename)
@@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
19 22
20 return retryTransactionWrapper(() => { 23 return retryTransactionWrapper(() => {
21 return sequelizeTypescript.transaction(async t => { 24 return sequelizeTypescript.transaction(async t => {
22 const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t) 25 const avatarInfo = {
26 name: avatarName,
27 fileUrl: null,
28 onDisk: true
29 }
30
31 const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t)
23 await updatedActor.save({ transaction: t }) 32 await updatedActor.save({ transaction: t })
24 33
25 await sendUpdateActor(accountOrChannel, t) 34 await sendUpdateActor(accountOrChannel, t)
@@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
29 }) 38 })
30} 39}
31 40
41type DownloadImageQueueTask = { fileUrl: string, filename: string }
42
43const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => {
44 downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE)
45 .then(() => cb())
46 .catch(err => cb(err))
47}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE)
48
49function pushAvatarProcessInQueue (task: DownloadImageQueueTask) {
50 return new Promise((res, rej) => {
51 downloadImageQueue.push(task, err => {
52 if (err) return rej(err)
53
54 return res()
55 })
56 })
57}
58
59// Unsafe so could returns paths that does not exist anymore
60const avatarPathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE })
61
32export { 62export {
33 updateActorAvatarFile 63 avatarPathUnsafeCache,
64 updateActorAvatarFile,
65 pushAvatarProcessInQueue
34} 66}
diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts
index 45ac3e7c4..a1153e88a 100644
--- a/server/lib/oauth-model.ts
+++ b/server/lib/oauth-model.ts
@@ -4,13 +4,15 @@ import { logger } from '../helpers/logger'
4import { UserModel } from '../models/account/user' 4import { UserModel } from '../models/account/user'
5import { OAuthClientModel } from '../models/oauth/oauth-client' 5import { OAuthClientModel } from '../models/oauth/oauth-client'
6import { OAuthTokenModel } from '../models/oauth/oauth-token' 6import { OAuthTokenModel } from '../models/oauth/oauth-token'
7import { CACHE } from '../initializers/constants' 7import { LRU_CACHE } from '../initializers/constants'
8import { Transaction } from 'sequelize' 8import { Transaction } from 'sequelize'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import * as LRUCache from 'lru-cache'
10 11
11type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } 12type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
12let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} 13
13let userHavingToken: { [ userId: number ]: string } = {} 14const accessTokenCache = new LRUCache<string, OAuthTokenModel>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
15const userHavingToken = new LRUCache<number, string>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
14 16
15// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
16 18
@@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) {
21} 23}
22 24
23function clearCacheByUserId (userId: number) { 25function clearCacheByUserId (userId: number) {
24 const token = userHavingToken[userId] 26 const token = userHavingToken.get(userId)
27
25 if (token !== undefined) { 28 if (token !== undefined) {
26 accessTokenCache[ token ] = undefined 29 accessTokenCache.del(token)
27 userHavingToken[ userId ] = undefined 30 userHavingToken.del(userId)
28 } 31 }
29} 32}
30 33
31function clearCacheByToken (token: string) { 34function clearCacheByToken (token: string) {
32 const tokenModel = accessTokenCache[ token ] 35 const tokenModel = accessTokenCache.get(token)
36
33 if (tokenModel !== undefined) { 37 if (tokenModel !== undefined) {
34 userHavingToken[tokenModel.userId] = undefined 38 userHavingToken.del(tokenModel.userId)
35 accessTokenCache[ token ] = undefined 39 accessTokenCache.del(token)
36 } 40 }
37} 41}
38 42
@@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) {
41 45
42 if (!bearerToken) return Bluebird.resolve(undefined) 46 if (!bearerToken) return Bluebird.resolve(undefined)
43 47
44 if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) 48 if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken))
45 49
46 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) 50 return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
47 .then(tokenModel => { 51 .then(tokenModel => {
48 if (tokenModel) { 52 if (tokenModel) {
49 // Reinit our cache 53 accessTokenCache.set(bearerToken, tokenModel)
50 if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) { 54 userHavingToken.set(tokenModel.userId, tokenModel.accessToken)
51 accessTokenCache = {}
52 userHavingToken = {}
53 }
54
55 accessTokenCache[ bearerToken ] = tokenModel
56 userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
57 } 55 }
58 56
59 return tokenModel 57 return tokenModel
diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts
index a4f97037b..f38cd7e78 100644
--- a/server/models/account/user-notification.ts
+++ b/server/models/account/user-notification.ts
@@ -410,7 +410,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
410 id: this.ActorFollow.ActorFollower.Account.id, 410 id: this.ActorFollow.ActorFollower.Account.id,
411 displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), 411 displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(),
412 name: this.ActorFollow.ActorFollower.preferredUsername, 412 name: this.ActorFollow.ActorFollower.preferredUsername,
413 avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined, 413 avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getStaticPath() } : undefined,
414 host: this.ActorFollow.ActorFollower.getHost() 414 host: this.ActorFollow.ActorFollower.getHost()
415 }, 415 },
416 following: { 416 following: {
@@ -446,7 +446,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
446 446
447 private formatActor (accountOrChannel: AccountModel | VideoChannelModel) { 447 private formatActor (accountOrChannel: AccountModel | VideoChannelModel) {
448 const avatar = accountOrChannel.Actor.Avatar 448 const avatar = accountOrChannel.Actor.Avatar
449 ? { path: accountOrChannel.Actor.Avatar.getWebserverPath() } 449 ? { path: accountOrChannel.Actor.Avatar.getStaticPath() }
450 : undefined 450 : undefined
451 451
452 return { 452 return {
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index bd6a2c8fd..9cc53f78a 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -513,7 +513,7 @@ export class ActorModel extends Model<ActorModel> {
513 getAvatarUrl () { 513 getAvatarUrl () {
514 if (!this.avatarId) return undefined 514 if (!this.avatarId) return undefined
515 515
516 return WEBSERVER.URL + this.Avatar.getWebserverPath() 516 return WEBSERVER.URL + this.Avatar.getStaticPath()
517 } 517 }
518 518
519 isOutdated () { 519 isOutdated () {
diff --git a/server/models/avatar/avatar.ts b/server/models/avatar/avatar.ts
index aaf1b8bd9..7a370bcd3 100644
--- a/server/models/avatar/avatar.ts
+++ b/server/models/avatar/avatar.ts
@@ -1,13 +1,21 @@
1import { join } from 'path' 1import { join } from 'path'
2import { AfterDestroy, AllowNull, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' 2import { AfterDestroy, AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
3import { Avatar } from '../../../shared/models/avatars/avatar.model' 3import { Avatar } from '../../../shared/models/avatars/avatar.model'
4import { STATIC_PATHS } from '../../initializers/constants' 4import { LAZY_STATIC_PATHS } from '../../initializers/constants'
5import { logger } from '../../helpers/logger' 5import { logger } from '../../helpers/logger'
6import { remove } from 'fs-extra' 6import { remove } from 'fs-extra'
7import { CONFIG } from '../../initializers/config' 7import { CONFIG } from '../../initializers/config'
8import { throwIfNotValid } from '../utils'
9import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
8 10
9@Table({ 11@Table({
10 tableName: 'avatar' 12 tableName: 'avatar',
13 indexes: [
14 {
15 fields: [ 'filename' ],
16 unique: true
17 }
18 ]
11}) 19})
12export class AvatarModel extends Model<AvatarModel> { 20export class AvatarModel extends Model<AvatarModel> {
13 21
@@ -15,6 +23,15 @@ export class AvatarModel extends Model<AvatarModel> {
15 @Column 23 @Column
16 filename: string 24 filename: string
17 25
26 @AllowNull(true)
27 @Is('AvatarFileUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'fileUrl'))
28 @Column
29 fileUrl: string
30
31 @AllowNull(false)
32 @Column
33 onDisk: boolean
34
18 @CreatedAt 35 @CreatedAt
19 createdAt: Date 36 createdAt: Date
20 37
@@ -30,16 +47,30 @@ export class AvatarModel extends Model<AvatarModel> {
30 .catch(err => logger.error('Cannot remove avatar file %s.', instance.filename, err)) 47 .catch(err => logger.error('Cannot remove avatar file %s.', instance.filename, err))
31 } 48 }
32 49
50 static loadByName (filename: string) {
51 const query = {
52 where: {
53 filename
54 }
55 }
56
57 return AvatarModel.findOne(query)
58 }
59
33 toFormattedJSON (): Avatar { 60 toFormattedJSON (): Avatar {
34 return { 61 return {
35 path: this.getWebserverPath(), 62 path: this.getStaticPath(),
36 createdAt: this.createdAt, 63 createdAt: this.createdAt,
37 updatedAt: this.updatedAt 64 updatedAt: this.updatedAt
38 } 65 }
39 } 66 }
40 67
41 getWebserverPath () { 68 getStaticPath () {
42 return join(STATIC_PATHS.AVATARS, this.filename) 69 return join(LAZY_STATIC_PATHS.AVATARS, this.filename)
70 }
71
72 getPath () {
73 return join(CONFIG.STORAGE.AVATARS_DIR, this.filename)
43 } 74 }
44 75
45 removeAvatar () { 76 removeAvatar () {
diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts
index b767a6874..cf2040cbf 100644
--- a/server/models/video/thumbnail.ts
+++ b/server/models/video/thumbnail.ts
@@ -1,6 +1,6 @@
1import { join } from 'path' 1import { join } from 'path'
2import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' 2import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
3import { STATIC_PATHS, WEBSERVER } from '../../initializers/constants' 3import { LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
4import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
5import { remove } from 'fs-extra' 5import { remove } from 'fs-extra'
6import { CONFIG } from '../../initializers/config' 6import { CONFIG } from '../../initializers/config'
@@ -87,7 +87,7 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
87 [ThumbnailType.PREVIEW]: { 87 [ThumbnailType.PREVIEW]: {
88 label: 'preview', 88 label: 'preview',
89 directory: CONFIG.STORAGE.PREVIEWS_DIR, 89 directory: CONFIG.STORAGE.PREVIEWS_DIR,
90 staticPath: STATIC_PATHS.PREVIEWS 90 staticPath: LAZY_STATIC_PATHS.PREVIEWS
91 } 91 }
92 } 92 }
93 93
diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts
index 76243bf48..a01565851 100644
--- a/server/models/video/video-caption.ts
+++ b/server/models/video/video-caption.ts
@@ -16,7 +16,7 @@ import { buildWhereIdOrUUID, throwIfNotValid } from '../utils'
16import { VideoModel } from './video' 16import { VideoModel } from './video'
17import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' 17import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
18import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' 18import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
19import { STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants' 19import { LAZY_STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants'
20import { join } from 'path' 20import { join } from 'path'
21import { logger } from '../../helpers/logger' 21import { logger } from '../../helpers/logger'
22import { remove } from 'fs-extra' 22import { remove } from 'fs-extra'
@@ -163,7 +163,7 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
163 } 163 }
164 164
165 getCaptionStaticPath () { 165 getCaptionStaticPath () {
166 return join(STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName()) 166 return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName())
167 } 167 }
168 168
169 getCaptionName () { 169 getCaptionName () {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index ae29cf286..1321337ff 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -63,6 +63,7 @@ import {
63 CONSTRAINTS_FIELDS, 63 CONSTRAINTS_FIELDS,
64 HLS_REDUNDANCY_DIRECTORY, 64 HLS_REDUNDANCY_DIRECTORY,
65 HLS_STREAMING_PLAYLIST_DIRECTORY, 65 HLS_STREAMING_PLAYLIST_DIRECTORY,
66 LAZY_STATIC_PATHS,
66 REMOTE_SCHEME, 67 REMOTE_SCHEME,
67 STATIC_DOWNLOAD_PATHS, 68 STATIC_DOWNLOAD_PATHS,
68 STATIC_PATHS, 69 STATIC_PATHS,
@@ -1856,7 +1857,7 @@ export class VideoModel extends Model<VideoModel> {
1856 if (!preview) return null 1857 if (!preview) return null
1857 1858
1858 // We use a local cache, so specify our cache endpoint instead of potential remote URL 1859 // We use a local cache, so specify our cache endpoint instead of potential remote URL
1859 return join(STATIC_PATHS.PREVIEWS, preview.filename) 1860 return join(LAZY_STATIC_PATHS.PREVIEWS, preview.filename)
1860 } 1861 }
1861 1862
1862 toFormattedJSON (options?: VideoFormattingJSONOptions): Video { 1863 toFormattedJSON (options?: VideoFormattingJSONOptions): Video {