aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-06-06 15:59:51 +0200
committerChocobozzz <me@florianbigard.com>2023-06-29 10:19:33 +0200
commitf162d32da098aa55f6de2367142faa166edb7c08 (patch)
tree31c6a96972994171853cb6c4e0b88b63241f8979
parenta673d9e848e51186602548a621e05925663b98be (diff)
downloadPeerTube-f162d32da098aa55f6de2367142faa166edb7c08.tar.gz
PeerTube-f162d32da098aa55f6de2367142faa166edb7c08.tar.zst
PeerTube-f162d32da098aa55f6de2367142faa166edb7c08.zip
Support lazy download thumbnails
-rw-r--r--server.ts10
-rw-r--r--server/controllers/api/video-playlist.ts8
-rw-r--r--server/controllers/api/videos/import.ts6
-rw-r--r--server/controllers/api/videos/live.ts4
-rw-r--r--server/controllers/api/videos/upload.ts4
-rw-r--r--server/controllers/download.ts4
-rw-r--r--server/controllers/lazy-static.ts100
-rw-r--r--server/initializers/constants.ts4
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts8
-rw-r--r--server/lib/files-cache/avatar-permanent-file-cache.ts27
-rw-r--r--server/lib/files-cache/index.ts9
-rw-r--r--server/lib/files-cache/shared/abstract-permanent-file-cache.ts119
-rw-r--r--server/lib/files-cache/shared/abstract-simple-file-cache.ts (renamed from server/lib/files-cache/abstract-video-static-file-cache.ts)4
-rw-r--r--server/lib/files-cache/shared/index.ts2
-rw-r--r--server/lib/files-cache/video-captions-simple-file-cache.ts (renamed from server/lib/files-cache/videos-caption-cache.ts)12
-rw-r--r--server/lib/files-cache/video-previews-simple-file-cache.ts (renamed from server/lib/files-cache/videos-preview-cache.ts)8
-rw-r--r--server/lib/files-cache/video-storyboards-simple-file-cache.ts (renamed from server/lib/files-cache/videos-storyboard-cache.ts)8
-rw-r--r--server/lib/files-cache/video-torrents-simple-file-cache.ts (renamed from server/lib/files-cache/videos-torrent-cache.ts)8
-rw-r--r--server/lib/job-queue/handlers/video-import.ts4
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts8
-rw-r--r--server/lib/local-actor.ts44
-rw-r--r--server/lib/thumbnail.ts48
-rw-r--r--server/lib/video-pre-import.ts4
-rw-r--r--server/lib/video.ts4
-rw-r--r--server/models/video/sql/video/shared/video-table-attributes.ts1
-rw-r--r--server/models/video/thumbnail.ts4
-rw-r--r--support/nginx/peertube22
27 files changed, 272 insertions, 212 deletions
diff --git a/server.ts b/server.ts
index 5d3acb2cd..e25322b66 100644
--- a/server.ts
+++ b/server.ts
@@ -21,7 +21,7 @@ import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initi
21 21
22// Do not use barrels because we don't want to load all modules here (we need to initialize database first) 22// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
23import { CONFIG } from './server/initializers/config' 23import { CONFIG } from './server/initializers/config'
24import { API_VERSION, FILES_CACHE, WEBSERVER, loadLanguages } from './server/initializers/constants' 24import { API_VERSION, WEBSERVER, loadLanguages } from './server/initializers/constants'
25import { logger } from './server/helpers/logger' 25import { logger } from './server/helpers/logger'
26 26
27const missed = checkMissedConfig() 27const missed = checkMissedConfig()
@@ -101,7 +101,6 @@ loadLanguages()
101import { installApplication } from './server/initializers/installer' 101import { installApplication } from './server/initializers/installer'
102import { Emailer } from './server/lib/emailer' 102import { Emailer } from './server/lib/emailer'
103import { JobQueue } from './server/lib/job-queue' 103import { JobQueue } from './server/lib/job-queue'
104import { VideosPreviewCache, VideosCaptionCache, VideosStoryboardCache } from './server/lib/files-cache'
105import { 104import {
106 activityPubRouter, 105 activityPubRouter,
107 apiRouter, 106 apiRouter,
@@ -143,7 +142,6 @@ import { Hooks } from './server/lib/plugins/hooks'
143import { PluginManager } from './server/lib/plugins/plugin-manager' 142import { PluginManager } from './server/lib/plugins/plugin-manager'
144import { LiveManager } from './server/lib/live' 143import { LiveManager } from './server/lib/live'
145import { HttpStatusCode } from './shared/models/http/http-error-codes' 144import { HttpStatusCode } from './shared/models/http/http-error-codes'
146import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
147import { ServerConfigManager } from '@server/lib/server-config-manager' 145import { ServerConfigManager } from '@server/lib/server-config-manager'
148import { VideoViewsManager } from '@server/lib/views/video-views-manager' 146import { VideoViewsManager } from '@server/lib/views/video-views-manager'
149import { isTestOrDevInstance } from './server/helpers/core-utils' 147import { isTestOrDevInstance } from './server/helpers/core-utils'
@@ -312,12 +310,6 @@ async function startApplication () {
312 ServerConfigManager.Instance.init() 310 ServerConfigManager.Instance.init()
313 ]) 311 ])
314 312
315 // Caches initializations
316 VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
317 VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
318 VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
319 VideosStoryboardCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
320
321 // Enable Schedulers 313 // Enable Schedulers
322 ActorFollowScheduler.Instance.enable() 314 ActorFollowScheduler.Instance.enable()
323 RemoveOldJobsScheduler.Instance.enable() 315 RemoveOldJobsScheduler.Instance.enable()
diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts
index fe00034ed..1568ee597 100644
--- a/server/controllers/api/video-playlist.ts
+++ b/server/controllers/api/video-playlist.ts
@@ -23,7 +23,7 @@ import { MIMETYPES, VIDEO_PLAYLIST_PRIVACIES } from '../../initializers/constant
23import { sequelizeTypescript } from '../../initializers/database' 23import { sequelizeTypescript } from '../../initializers/database'
24import { sendCreateVideoPlaylist, sendDeleteVideoPlaylist, sendUpdateVideoPlaylist } from '../../lib/activitypub/send' 24import { sendCreateVideoPlaylist, sendDeleteVideoPlaylist, sendUpdateVideoPlaylist } from '../../lib/activitypub/send'
25import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '../../lib/activitypub/url' 25import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '../../lib/activitypub/url'
26import { updatePlaylistMiniatureFromExisting } from '../../lib/thumbnail' 26import { updateLocalPlaylistMiniatureFromExisting } from '../../lib/thumbnail'
27import { 27import {
28 apiRateLimiter, 28 apiRateLimiter,
29 asyncMiddleware, 29 asyncMiddleware,
@@ -178,7 +178,7 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) {
178 178
179 const thumbnailField = req.files['thumbnailfile'] 179 const thumbnailField = req.files['thumbnailfile']
180 const thumbnailModel = thumbnailField 180 const thumbnailModel = thumbnailField
181 ? await updatePlaylistMiniatureFromExisting({ 181 ? await updateLocalPlaylistMiniatureFromExisting({
182 inputPath: thumbnailField[0].path, 182 inputPath: thumbnailField[0].path,
183 playlist: videoPlaylist, 183 playlist: videoPlaylist,
184 automaticallyGenerated: false 184 automaticallyGenerated: false
@@ -220,7 +220,7 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response)
220 220
221 const thumbnailField = req.files['thumbnailfile'] 221 const thumbnailField = req.files['thumbnailfile']
222 const thumbnailModel = thumbnailField 222 const thumbnailModel = thumbnailField
223 ? await updatePlaylistMiniatureFromExisting({ 223 ? await updateLocalPlaylistMiniatureFromExisting({
224 inputPath: thumbnailField[0].path, 224 inputPath: thumbnailField[0].path,
225 playlist: videoPlaylistInstance, 225 playlist: videoPlaylistInstance,
226 automaticallyGenerated: false 226 automaticallyGenerated: false
@@ -497,7 +497,7 @@ async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbn
497 } 497 }
498 498
499 const inputPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoMiniature.filename) 499 const inputPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoMiniature.filename)
500 const thumbnailModel = await updatePlaylistMiniatureFromExisting({ 500 const thumbnailModel = await updateLocalPlaylistMiniatureFromExisting({
501 inputPath, 501 inputPath,
502 playlist: videoPlaylist, 502 playlist: videoPlaylist,
503 automaticallyGenerated: true, 503 automaticallyGenerated: true,
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index b8016140e..defe9efd4 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -14,7 +14,7 @@ import { getSecureTorrentName } from '../../../helpers/utils'
14import { CONFIG } from '../../../initializers/config' 14import { CONFIG } from '../../../initializers/config'
15import { MIMETYPES } from '../../../initializers/constants' 15import { MIMETYPES } from '../../../initializers/constants'
16import { JobQueue } from '../../../lib/job-queue/job-queue' 16import { JobQueue } from '../../../lib/job-queue/job-queue'
17import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail' 17import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail'
18import { 18import {
19 asyncMiddleware, 19 asyncMiddleware,
20 asyncRetryTransactionMiddleware, 20 asyncRetryTransactionMiddleware,
@@ -193,7 +193,7 @@ async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
193 if (thumbnailField) { 193 if (thumbnailField) {
194 const thumbnailPhysicalFile = thumbnailField[0] 194 const thumbnailPhysicalFile = thumbnailField[0]
195 195
196 return updateVideoMiniatureFromExisting({ 196 return updateLocalVideoMiniatureFromExisting({
197 inputPath: thumbnailPhysicalFile.path, 197 inputPath: thumbnailPhysicalFile.path,
198 video, 198 video,
199 type: ThumbnailType.MINIATURE, 199 type: ThumbnailType.MINIATURE,
@@ -209,7 +209,7 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
209 if (previewField) { 209 if (previewField) {
210 const previewPhysicalFile = previewField[0] 210 const previewPhysicalFile = previewField[0]
211 211
212 return updateVideoMiniatureFromExisting({ 212 return updateLocalVideoMiniatureFromExisting({
213 inputPath: previewPhysicalFile.path, 213 inputPath: previewPhysicalFile.path,
214 video, 214 video,
215 type: ThumbnailType.PREVIEW, 215 type: ThumbnailType.PREVIEW,
diff --git a/server/controllers/api/videos/live.ts b/server/controllers/api/videos/live.ts
index cf82c9791..e19e8c652 100644
--- a/server/controllers/api/videos/live.ts
+++ b/server/controllers/api/videos/live.ts
@@ -21,7 +21,7 @@ import { buildUUID, uuidToShort } from '@shared/extra-utils'
21import { HttpStatusCode, LiveVideoCreate, LiveVideoLatencyMode, LiveVideoUpdate, UserRight, VideoPrivacy, VideoState } from '@shared/models' 21import { HttpStatusCode, LiveVideoCreate, LiveVideoLatencyMode, LiveVideoUpdate, UserRight, VideoPrivacy, VideoState } from '@shared/models'
22import { logger } from '../../../helpers/logger' 22import { logger } from '../../../helpers/logger'
23import { sequelizeTypescript } from '../../../initializers/database' 23import { sequelizeTypescript } from '../../../initializers/database'
24import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail' 24import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail'
25import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, optionalAuthenticate } from '../../../middlewares' 25import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, optionalAuthenticate } from '../../../middlewares'
26import { VideoModel } from '../../../models/video/video' 26import { VideoModel } from '../../../models/video/video'
27import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' 27import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting'
@@ -166,7 +166,7 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
166 video, 166 video,
167 files: req.files, 167 files: req.files,
168 fallback: type => { 168 fallback: type => {
169 return updateVideoMiniatureFromExisting({ 169 return updateLocalVideoMiniatureFromExisting({
170 inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND, 170 inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND,
171 video, 171 video,
172 type, 172 type,
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 6c471ff90..0e07302d2 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -21,7 +21,7 @@ import { logger, loggerTagsFactory } from '../../../helpers/logger'
21import { MIMETYPES } from '../../../initializers/constants' 21import { MIMETYPES } from '../../../initializers/constants'
22import { sequelizeTypescript } from '../../../initializers/database' 22import { sequelizeTypescript } from '../../../initializers/database'
23import { Hooks } from '../../../lib/plugins/hooks' 23import { Hooks } from '../../../lib/plugins/hooks'
24import { generateVideoMiniature } from '../../../lib/thumbnail' 24import { generateLocalVideoMiniature } from '../../../lib/thumbnail'
25import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' 25import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
26import { 26import {
27 asyncMiddleware, 27 asyncMiddleware,
@@ -153,7 +153,7 @@ async function addVideo (options: {
153 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ 153 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
154 video, 154 video,
155 files, 155 files,
156 fallback: type => generateVideoMiniature({ video, videoFile, type }) 156 fallback: type => generateLocalVideoMiniature({ video, videoFile, type })
157 }) 157 })
158 158
159 const { videoCreated } = await sequelizeTypescript.transaction(async t => { 159 const { videoCreated } = await sequelizeTypescript.transaction(async t => {
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index 4c3ab0163..4b94e34bd 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -1,7 +1,7 @@
1import cors from 'cors' 1import cors from 'cors'
2import express from 'express' 2import express from 'express'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' 4import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache'
5import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage' 5import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage'
6import { Hooks } from '@server/lib/plugins/hooks' 6import { Hooks } from '@server/lib/plugins/hooks'
7import { VideoPathManager } from '@server/lib/video-path-manager' 7import { VideoPathManager } from '@server/lib/video-path-manager'
@@ -43,7 +43,7 @@ export {
43// --------------------------------------------------------------------------- 43// ---------------------------------------------------------------------------
44 44
45async function downloadTorrent (req: express.Request, res: express.Response) { 45async function downloadTorrent (req: express.Request, res: express.Response) {
46 const result = await VideosTorrentCache.Instance.getFilePath(req.params.filename) 46 const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename)
47 if (!result) { 47 if (!result) {
48 return res.fail({ 48 return res.fail({
49 status: HttpStatusCode.NOT_FOUND_404, 49 status: HttpStatusCode.NOT_FOUND_404,
diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts
index 6ffd39730..8e18b0642 100644
--- a/server/controllers/lazy-static.ts
+++ b/server/controllers/lazy-static.ts
@@ -1,14 +1,27 @@
1import cors from 'cors' 1import cors from 'cors'
2import express from 'express' 2import express from 'express'
3import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' 3import { CONFIG } from '@server/initializers/config'
4import { MActorImage } from '@server/types/models'
5import { HttpStatusCode } from '../../shared/models/http/http-error-codes' 4import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
6import { logger } from '../helpers/logger' 5import { FILES_CACHE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
7import { ACTOR_IMAGES_SIZE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' 6import {
8import { VideosCaptionCache, VideosPreviewCache, VideosStoryboardCache } from '../lib/files-cache' 7 AvatarPermanentFileCache,
9import { actorImagePathUnsafeCache, downloadActorImageFromWorker } from '../lib/local-actor' 8 VideoCaptionsSimpleFileCache,
9 VideoPreviewsSimpleFileCache,
10 VideoStoryboardsSimpleFileCache,
11 VideoTorrentsSimpleFileCache
12} from '../lib/files-cache'
10import { asyncMiddleware, handleStaticError } from '../middlewares' 13import { asyncMiddleware, handleStaticError } from '../middlewares'
11import { ActorImageModel } from '../models/actor/actor-image' 14
15// ---------------------------------------------------------------------------
16// Cache initializations
17// ---------------------------------------------------------------------------
18
19VideoPreviewsSimpleFileCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE)
20VideoCaptionsSimpleFileCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE)
21VideoTorrentsSimpleFileCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE)
22VideoStoryboardsSimpleFileCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE)
23
24// ---------------------------------------------------------------------------
12 25
13const lazyStaticRouter = express.Router() 26const lazyStaticRouter = express.Router()
14 27
@@ -60,94 +73,37 @@ export {
60 73
61// --------------------------------------------------------------------------- 74// ---------------------------------------------------------------------------
62 75
63async function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) { 76const avatarPermanentFileCache = new AvatarPermanentFileCache()
64 const filename = req.params.filename
65
66 if (actorImagePathUnsafeCache.has(filename)) {
67 return res.sendFile(actorImagePathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
68 }
69
70 const image = await ActorImageModel.loadByName(filename)
71 if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
72
73 if (image.onDisk === false) {
74 if (!image.fileUrl) return res.status(HttpStatusCode.NOT_FOUND_404).end()
75
76 logger.info('Lazy serve remote actor image %s.', image.fileUrl)
77 77
78 try { 78function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) {
79 await downloadActorImageFromWorker({ 79 const filename = req.params.filename
80 filename: image.filename,
81 fileUrl: image.fileUrl,
82 size: getActorImageSize(image),
83 type: image.type
84 })
85 } catch (err) {
86 logger.warn('Cannot process remote actor image %s.', image.fileUrl, { err })
87 return res.status(HttpStatusCode.NOT_FOUND_404).end()
88 }
89
90 image.onDisk = true
91 image.save()
92 .catch(err => logger.error('Cannot save new actor image disk state.', { err }))
93 }
94
95 const path = image.getPath()
96
97 actorImagePathUnsafeCache.set(filename, path)
98
99 return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => {
100 if (!err) return
101
102 // It seems this actor image is not on the disk anymore
103 if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) {
104 logger.error('Cannot lazy serve actor image %s.', filename, { err })
105
106 actorImagePathUnsafeCache.delete(filename)
107
108 image.onDisk = false
109 image.save()
110 .catch(err => logger.error('Cannot save new actor image disk state.', { err }))
111 }
112
113 return next(err)
114 })
115}
116
117function getActorImageSize (image: MActorImage): { width: number, height: number } {
118 if (image.width && image.height) {
119 return {
120 height: image.height,
121 width: image.width
122 }
123 }
124 80
125 return ACTOR_IMAGES_SIZE[image.type][0] 81 return avatarPermanentFileCache.lazyServe({ filename, res, next })
126} 82}
127 83
128async function getPreview (req: express.Request, res: express.Response) { 84async function getPreview (req: express.Request, res: express.Response) {
129 const result = await VideosPreviewCache.Instance.getFilePath(req.params.filename) 85 const result = await VideoPreviewsSimpleFileCache.Instance.getFilePath(req.params.filename)
130 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() 86 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
131 87
132 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) 88 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
133} 89}
134 90
135async function getStoryboard (req: express.Request, res: express.Response) { 91async function getStoryboard (req: express.Request, res: express.Response) {
136 const result = await VideosStoryboardCache.Instance.getFilePath(req.params.filename) 92 const result = await VideoStoryboardsSimpleFileCache.Instance.getFilePath(req.params.filename)
137 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() 93 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
138 94
139 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) 95 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
140} 96}
141 97
142async function getVideoCaption (req: express.Request, res: express.Response) { 98async function getVideoCaption (req: express.Request, res: express.Response) {
143 const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename) 99 const result = await VideoCaptionsSimpleFileCache.Instance.getFilePath(req.params.filename)
144 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() 100 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
145 101
146 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) 102 return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER })
147} 103}
148 104
149async function getTorrent (req: express.Request, res: express.Response) { 105async function getTorrent (req: express.Request, res: express.Response) {
150 const result = await VideosTorrentCache.Instance.getFilePath(req.params.filename) 106 const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename)
151 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() 107 if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end()
152 108
153 // Torrents still use the old naming convention (video uuid + .torrent) 109 // Torrents still use the old naming convention (video uuid + .torrent)
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 3a643a60b..511aa91cc 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -854,8 +854,8 @@ const LRU_CACHE = {
854 USER_TOKENS: { 854 USER_TOKENS: {
855 MAX_SIZE: 1000 855 MAX_SIZE: 1000
856 }, 856 },
857 ACTOR_IMAGE_STATIC: { 857 FILENAME_TO_PATH_PERMANENT_FILE_CACHE: {
858 MAX_SIZE: 500 858 MAX_SIZE: 1000
859 }, 859 },
860 STATIC_VIDEO_FILES_RIGHTS_CHECK: { 860 STATIC_VIDEO_FILES_RIGHTS_CHECK: {
861 MAX_SIZE: 5000, 861 MAX_SIZE: 5000,
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
index 8af67ecac..e50bf29dc 100644
--- a/server/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -1,7 +1,7 @@
1import { CreationAttributes, Transaction } from 'sequelize/types' 1import { CreationAttributes, Transaction } from 'sequelize/types'
2import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' 2import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
3import { logger, LoggerTagsFn } from '@server/helpers/logger' 3import { logger, LoggerTagsFn } from '@server/helpers/logger'
4import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' 4import { updateRemoteThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
5import { setVideoTags } from '@server/lib/video' 5import { setVideoTags } from '@server/lib/video'
6import { StoryboardModel } from '@server/models/video/storyboard' 6import { StoryboardModel } from '@server/models/video/storyboard'
7import { VideoCaptionModel } from '@server/models/video/video-caption' 7import { VideoCaptionModel } from '@server/models/video/video-caption'
@@ -55,15 +55,15 @@ export abstract class APVideoAbstractBuilder {
55 } 55 }
56 56
57 protected async setPreview (video: MVideoFullLight, t?: Transaction) { 57 protected async setPreview (video: MVideoFullLight, t?: Transaction) {
58 // Don't fetch the preview that could be big, create a placeholder instead
59 const previewIcon = getPreviewFromIcons(this.videoObject) 58 const previewIcon = getPreviewFromIcons(this.videoObject)
60 if (!previewIcon) return 59 if (!previewIcon) return
61 60
62 const previewModel = updatePlaceholderThumbnail({ 61 const previewModel = updateRemoteThumbnail({
63 fileUrl: previewIcon.url, 62 fileUrl: previewIcon.url,
64 video, 63 video,
65 type: ThumbnailType.PREVIEW, 64 type: ThumbnailType.PREVIEW,
66 size: previewIcon 65 size: previewIcon,
66 onDisk: false // Don't fetch the preview that could be big, create a placeholder instead
67 }) 67 })
68 68
69 await video.addAndSaveThumbnail(previewModel, t) 69 await video.addAndSaveThumbnail(previewModel, t)
diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts
new file mode 100644
index 000000000..89228c5a5
--- /dev/null
+++ b/server/lib/files-cache/avatar-permanent-file-cache.ts
@@ -0,0 +1,27 @@
1import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants'
2import { ActorImageModel } from '@server/models/actor/actor-image'
3import { MActorImage } from '@server/types/models'
4import { AbstractPermanentFileCache } from './shared'
5import { CONFIG } from '@server/initializers/config'
6
7export class AvatarPermanentFileCache extends AbstractPermanentFileCache<ActorImageModel> {
8
9 constructor () {
10 super(CONFIG.STORAGE.ACTOR_IMAGES)
11 }
12
13 protected loadModel (filename: string) {
14 return ActorImageModel.loadByName(filename)
15 }
16
17 protected getImageSize (image: MActorImage): { width: number, height: number } {
18 if (image.width && image.height) {
19 return {
20 height: image.height,
21 width: image.width
22 }
23 }
24
25 return ACTOR_IMAGES_SIZE[image.type][0]
26 }
27}
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts
index 59cec7215..cc11d5385 100644
--- a/server/lib/files-cache/index.ts
+++ b/server/lib/files-cache/index.ts
@@ -1,4 +1,5 @@
1export * from './videos-caption-cache' 1export * from './avatar-permanent-file-cache'
2export * from './videos-preview-cache' 2export * from './video-captions-simple-file-cache'
3export * from './videos-storyboard-cache' 3export * from './video-previews-simple-file-cache'
4export * from './videos-torrent-cache' 4export * from './video-storyboards-simple-file-cache'
5export * from './video-torrents-simple-file-cache'
diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
new file mode 100644
index 000000000..22596c3eb
--- /dev/null
+++ b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
@@ -0,0 +1,119 @@
1import express from 'express'
2import { LRUCache } from 'lru-cache'
3import { logger } from '@server/helpers/logger'
4import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants'
5import { downloadImageFromWorker } from '@server/lib/worker/parent-process'
6import { HttpStatusCode } from '@shared/models'
7import { Model } from 'sequelize'
8
9type ImageModel = {
10 fileUrl: string
11 filename: string
12 onDisk: boolean
13
14 isOwned (): boolean
15 getPath (): string
16
17 save (): Promise<Model>
18}
19
20export abstract class AbstractPermanentFileCache <M extends ImageModel> {
21 // Unsafe because it can return paths that do not exist anymore
22 private readonly filenameToPathUnsafeCache = new LRUCache<string, string>({
23 max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE
24 })
25
26 protected abstract getImageSize (image: M): { width: number, height: number }
27 protected abstract loadModel (filename: string): Promise<M>
28
29 constructor (private readonly directory: string) {
30
31 }
32
33 async lazyServe (options: {
34 filename: string
35 res: express.Response
36 next: express.NextFunction
37 }) {
38 const { filename, res, next } = options
39
40 if (this.filenameToPathUnsafeCache.has(filename)) {
41 return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
42 }
43
44 const image = await this.loadModel(filename)
45 if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
46
47 if (image.onDisk === false) {
48 if (!image.fileUrl) return res.status(HttpStatusCode.NOT_FOUND_404).end()
49
50 try {
51 await this.downloadRemoteFile(image)
52 } catch (err) {
53 logger.warn('Cannot process remote image %s.', image.fileUrl, { err })
54
55 return res.status(HttpStatusCode.NOT_FOUND_404).end()
56 }
57 }
58
59 const path = image.getPath()
60 this.filenameToPathUnsafeCache.set(filename, path)
61
62 return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => {
63 if (!err) return
64
65 this.onServeError({ err, image, next, filename })
66 })
67 }
68
69 private async downloadRemoteFile (image: M) {
70 logger.info('Download remote image %s lazily.', image.fileUrl)
71
72 await this.downloadImage({
73 filename: image.filename,
74 fileUrl: image.fileUrl,
75 size: this.getImageSize(image)
76 })
77
78 image.onDisk = true
79 image.save()
80 .catch(err => logger.error('Cannot save new image disk state.', { err }))
81 }
82
83 private onServeError (options: {
84 err: any
85 image: M
86 filename: string
87 next: express.NextFunction
88 }) {
89 const { err, image, filename, next } = options
90
91 // It seems this actor image is not on the disk anymore
92 if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) {
93 logger.error('Cannot lazy serve image %s.', filename, { err })
94
95 this.filenameToPathUnsafeCache.delete(filename)
96
97 image.onDisk = false
98 image.save()
99 .catch(err => logger.error('Cannot save new image disk state.', { err }))
100 }
101
102 return next(err)
103 }
104
105 private downloadImage (options: {
106 fileUrl: string
107 filename: string
108 size: { width: number, height: number }
109 }) {
110 const downloaderOptions = {
111 url: options.fileUrl,
112 destDir: this.directory,
113 destName: options.filename,
114 size: options.size
115 }
116
117 return downloadImageFromWorker(downloaderOptions)
118 }
119}
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts
index a7ac88525..6fab322cd 100644
--- a/server/lib/files-cache/abstract-video-static-file-cache.ts
+++ b/server/lib/files-cache/shared/abstract-simple-file-cache.ts
@@ -1,10 +1,10 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import { logger } from '../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import memoizee from 'memoizee' 3import memoizee from 'memoizee'
4 4
5type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined 5type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined
6 6
7export abstract class AbstractVideoStaticFileCache <T> { 7export abstract class AbstractSimpleFileCache <T> {
8 8
9 getFilePath: (params: T) => Promise<GetFilePathResult> 9 getFilePath: (params: T) => Promise<GetFilePathResult>
10 10
diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts
new file mode 100644
index 000000000..61c4aacc7
--- /dev/null
+++ b/server/lib/files-cache/shared/index.ts
@@ -0,0 +1,2 @@
1export * from './abstract-permanent-file-cache'
2export * from './abstract-simple-file-cache'
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/video-captions-simple-file-cache.ts
index d21acf4ef..cbeeff732 100644
--- a/server/lib/files-cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/video-captions-simple-file-cache.ts
@@ -5,11 +5,11 @@ import { CONFIG } from '../../initializers/config'
5import { FILES_CACHE } from '../../initializers/constants' 5import { FILES_CACHE } from '../../initializers/constants'
6import { VideoModel } from '../../models/video/video' 6import { VideoModel } from '../../models/video/video'
7import { VideoCaptionModel } from '../../models/video/video-caption' 7import { VideoCaptionModel } from '../../models/video/video-caption'
8import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 8import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
9 9
10class VideosCaptionCache extends AbstractVideoStaticFileCache <string> { 10class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> {
11 11
12 private static instance: VideosCaptionCache 12 private static instance: VideoCaptionsSimpleFileCache
13 13
14 private constructor () { 14 private constructor () {
15 super() 15 super()
@@ -23,7 +23,9 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
23 const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) 23 const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename)
24 if (!videoCaption) return undefined 24 if (!videoCaption) return undefined
25 25
26 if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } 26 if (videoCaption.isOwned()) {
27 return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
28 }
27 29
28 return this.loadRemoteFile(filename) 30 return this.loadRemoteFile(filename)
29 } 31 }
@@ -55,5 +57,5 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
55} 57}
56 58
57export { 59export {
58 VideosCaptionCache 60 VideoCaptionsSimpleFileCache
59} 61}
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/video-previews-simple-file-cache.ts
index d19c3f4f4..a05e80e16 100644
--- a/server/lib/files-cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/video-previews-simple-file-cache.ts
@@ -1,15 +1,15 @@
1import { join } from 'path' 1import { join } from 'path'
2import { FILES_CACHE } from '../../initializers/constants' 2import { FILES_CACHE } from '../../initializers/constants'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 4import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
5import { doRequestAndSaveToFile } from '@server/helpers/requests' 5import { doRequestAndSaveToFile } from '@server/helpers/requests'
6import { ThumbnailModel } from '@server/models/video/thumbnail' 6import { ThumbnailModel } from '@server/models/video/thumbnail'
7import { ThumbnailType } from '@shared/models' 7import { ThumbnailType } from '@shared/models'
8import { logger } from '@server/helpers/logger' 8import { logger } from '@server/helpers/logger'
9 9
10class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { 10class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache <string> {
11 11
12 private static instance: VideosPreviewCache 12 private static instance: VideoPreviewsSimpleFileCache
13 13
14 private constructor () { 14 private constructor () {
15 super() 15 super()
@@ -54,5 +54,5 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
54} 54}
55 55
56export { 56export {
57 VideosPreviewCache 57 VideoPreviewsSimpleFileCache
58} 58}
diff --git a/server/lib/files-cache/videos-storyboard-cache.ts b/server/lib/files-cache/video-storyboards-simple-file-cache.ts
index b0a55104f..4cd96e70c 100644
--- a/server/lib/files-cache/videos-storyboard-cache.ts
+++ b/server/lib/files-cache/video-storyboards-simple-file-cache.ts
@@ -3,11 +3,11 @@ import { logger } from '@server/helpers/logger'
3import { doRequestAndSaveToFile } from '@server/helpers/requests' 3import { doRequestAndSaveToFile } from '@server/helpers/requests'
4import { StoryboardModel } from '@server/models/video/storyboard' 4import { StoryboardModel } from '@server/models/video/storyboard'
5import { FILES_CACHE } from '../../initializers/constants' 5import { FILES_CACHE } from '../../initializers/constants'
6import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 6import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
7 7
8class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> { 8class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache <string> {
9 9
10 private static instance: VideosStoryboardCache 10 private static instance: VideoStoryboardsSimpleFileCache
11 11
12 private constructor () { 12 private constructor () {
13 super() 13 super()
@@ -49,5 +49,5 @@ class VideosStoryboardCache extends AbstractVideoStaticFileCache <string> {
49} 49}
50 50
51export { 51export {
52 VideosStoryboardCache 52 VideoStoryboardsSimpleFileCache
53} 53}
diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/video-torrents-simple-file-cache.ts
index a6bf98dd4..8bcd0b9bf 100644
--- a/server/lib/files-cache/videos-torrent-cache.ts
+++ b/server/lib/files-cache/video-torrents-simple-file-cache.ts
@@ -6,11 +6,11 @@ import { MVideo, MVideoFile } from '@server/types/models'
6import { CONFIG } from '../../initializers/config' 6import { CONFIG } from '../../initializers/config'
7import { FILES_CACHE } from '../../initializers/constants' 7import { FILES_CACHE } from '../../initializers/constants'
8import { VideoModel } from '../../models/video/video' 8import { VideoModel } from '../../models/video/video'
9import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' 9import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
10 10
11class VideosTorrentCache extends AbstractVideoStaticFileCache <string> { 11class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache <string> {
12 12
13 private static instance: VideosTorrentCache 13 private static instance: VideoTorrentsSimpleFileCache
14 14
15 private constructor () { 15 private constructor () {
16 super() 16 super()
@@ -66,5 +66,5 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
66} 66}
67 67
68export { 68export {
69 VideosTorrentCache 69 VideoTorrentsSimpleFileCache
70} 70}
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index c1355dcef..436bf3175 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -39,7 +39,7 @@ import { VideoFileModel } from '../../../models/video/video-file'
39import { VideoImportModel } from '../../../models/video/video-import' 39import { VideoImportModel } from '../../../models/video/video-import'
40import { federateVideoIfNeeded } from '../../activitypub/videos' 40import { federateVideoIfNeeded } from '../../activitypub/videos'
41import { Notifier } from '../../notifier' 41import { Notifier } from '../../notifier'
42import { generateVideoMiniature } from '../../thumbnail' 42import { generateLocalVideoMiniature } from '../../thumbnail'
43import { JobQueue } from '../job-queue' 43import { JobQueue } from '../job-queue'
44 44
45async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> { 45async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
@@ -274,7 +274,7 @@ async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles
274 } 274 }
275 } 275 }
276 276
277 const miniatureModel = await generateVideoMiniature({ 277 const miniatureModel = await generateLocalVideoMiniature({
278 video: videoImportWithFiles.Video, 278 video: videoImportWithFiles.Video,
279 videoFile, 279 videoFile,
280 type: thumbnailType 280 type: thumbnailType
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 95d4f5e64..ae886de35 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -7,7 +7,7 @@ import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
7import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' 7import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
8import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' 8import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
9import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' 9import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
10import { generateVideoMiniature } from '@server/lib/thumbnail' 10import { generateLocalVideoMiniature } from '@server/lib/thumbnail'
11import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' 11import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding'
12import { VideoPathManager } from '@server/lib/video-path-manager' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { moveToNextState } from '@server/lib/video-state' 13import { moveToNextState } from '@server/lib/video-state'
@@ -143,7 +143,7 @@ async function saveReplayToExternalVideo (options: {
143 await remove(replayDirectory) 143 await remove(replayDirectory)
144 144
145 for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { 145 for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) {
146 const image = await generateVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) 146 const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type })
147 await replayVideo.addAndSaveThumbnail(image) 147 await replayVideo.addAndSaveThumbnail(image)
148 } 148 }
149 149
@@ -198,7 +198,7 @@ async function replaceLiveByReplay (options: {
198 198
199 // Regenerate the thumbnail & preview? 199 // Regenerate the thumbnail & preview?
200 if (videoWithFiles.getMiniature().automaticallyGenerated === true) { 200 if (videoWithFiles.getMiniature().automaticallyGenerated === true) {
201 const miniature = await generateVideoMiniature({ 201 const miniature = await generateLocalVideoMiniature({
202 video: videoWithFiles, 202 video: videoWithFiles,
203 videoFile: videoWithFiles.getMaxQualityFile(), 203 videoFile: videoWithFiles.getMaxQualityFile(),
204 type: ThumbnailType.MINIATURE 204 type: ThumbnailType.MINIATURE
@@ -207,7 +207,7 @@ async function replaceLiveByReplay (options: {
207 } 207 }
208 208
209 if (videoWithFiles.getPreview().automaticallyGenerated === true) { 209 if (videoWithFiles.getPreview().automaticallyGenerated === true) {
210 const preview = await generateVideoMiniature({ 210 const preview = await generateLocalVideoMiniature({
211 video: videoWithFiles, 211 video: videoWithFiles,
212 videoFile: videoWithFiles.getMaxQualityFile(), 212 videoFile: videoWithFiles.getMaxQualityFile(),
213 type: ThumbnailType.PREVIEW 213 type: ThumbnailType.PREVIEW
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts
index 16dc265a3..872addc58 100644
--- a/server/lib/local-actor.ts
+++ b/server/lib/local-actor.ts
@@ -1,5 +1,4 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import { LRUCache } from 'lru-cache'
3import { join } from 'path' 2import { join } from 'path'
4import { Transaction } from 'sequelize/types' 3import { Transaction } from 'sequelize/types'
5import { ActorModel } from '@server/models/actor/actor' 4import { ActorModel } from '@server/models/actor/actor'
@@ -8,14 +7,14 @@ import { buildUUID } from '@shared/extra-utils'
8import { ActivityPubActorType, ActorImageType } from '@shared/models' 7import { ActivityPubActorType, ActorImageType } from '@shared/models'
9import { retryTransactionWrapper } from '../helpers/database-utils' 8import { retryTransactionWrapper } from '../helpers/database-utils'
10import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
11import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants' 10import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants'
12import { sequelizeTypescript } from '../initializers/database' 11import { sequelizeTypescript } from '../initializers/database'
13import { MAccountDefault, MActor, MChannelDefault } from '../types/models' 12import { MAccountDefault, MActor, MChannelDefault } from '../types/models'
14import { deleteActorImages, updateActorImages } from './activitypub/actors' 13import { deleteActorImages, updateActorImages } from './activitypub/actors'
15import { sendUpdateActor } from './activitypub/send' 14import { sendUpdateActor } from './activitypub/send'
16import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' 15import { processImageFromWorker } from './worker/parent-process'
17 16
18function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { 17export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) {
19 return new ActorModel({ 18 return new ActorModel({
20 type, 19 type,
21 url, 20 url,
@@ -32,7 +31,7 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU
32 }) as MActor 31 }) as MActor
33} 32}
34 33
35async function updateLocalActorImageFiles ( 34export async function updateLocalActorImageFiles (
36 accountOrChannel: MAccountDefault | MChannelDefault, 35 accountOrChannel: MAccountDefault | MChannelDefault,
37 imagePhysicalFile: Express.Multer.File, 36 imagePhysicalFile: Express.Multer.File,
38 type: ActorImageType 37 type: ActorImageType
@@ -73,7 +72,7 @@ async function updateLocalActorImageFiles (
73 })) 72 }))
74} 73}
75 74
76async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) { 75export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) {
77 return retryTransactionWrapper(() => { 76 return retryTransactionWrapper(() => {
78 return sequelizeTypescript.transaction(async t => { 77 return sequelizeTypescript.transaction(async t => {
79 const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) 78 const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t)
@@ -88,7 +87,7 @@ async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MC
88 87
89// --------------------------------------------------------------------------- 88// ---------------------------------------------------------------------------
90 89
91async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { 90export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) {
92 let actor = await ActorModel.loadLocalByName(baseActorName, transaction) 91 let actor = await ActorModel.loadLocalByName(baseActorName, transaction)
93 if (!actor) return baseActorName 92 if (!actor) return baseActorName
94 93
@@ -101,34 +100,3 @@ async function findAvailableLocalActorName (baseActorName: string, transaction?:
101 100
102 throw new Error('Cannot find available actor local name (too much iterations).') 101 throw new Error('Cannot find available actor local name (too much iterations).')
103} 102}
104
105// ---------------------------------------------------------------------------
106
107function downloadActorImageFromWorker (options: {
108 fileUrl: string
109 filename: string
110 type: ActorImageType
111 size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0]
112}) {
113 const downloaderOptions = {
114 url: options.fileUrl,
115 destDir: CONFIG.STORAGE.ACTOR_IMAGES,
116 destName: options.filename,
117 size: options.size
118 }
119
120 return downloadImageFromWorker(downloaderOptions)
121}
122
123// Unsafe so could returns paths that does not exist anymore
124const actorImagePathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.ACTOR_IMAGE_STATIC.MAX_SIZE })
125
126export {
127 actorImagePathUnsafeCache,
128 updateLocalActorImageFiles,
129 findAvailableLocalActorName,
130 downloadActorImageFromWorker,
131 deleteLocalActorImageFile,
132 downloadImageFromWorker,
133 buildActorInstance
134}
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index 02b867a91..e792567ff 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -7,13 +7,12 @@ import { ThumbnailModel } from '../models/video/thumbnail'
7import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' 7import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
8import { MThumbnail } from '../types/models/video/thumbnail' 8import { MThumbnail } from '../types/models/video/thumbnail'
9import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' 9import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
10import { downloadImageFromWorker } from './local-actor'
11import { VideoPathManager } from './video-path-manager' 10import { VideoPathManager } from './video-path-manager'
12import { processImageFromWorker } from './worker/parent-process' 11import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process'
13 12
14type ImageSize = { height?: number, width?: number } 13type ImageSize = { height?: number, width?: number }
15 14
16function updatePlaylistMiniatureFromExisting (options: { 15function updateLocalPlaylistMiniatureFromExisting (options: {
17 inputPath: string 16 inputPath: string
18 playlist: MVideoPlaylistThumbnail 17 playlist: MVideoPlaylistThumbnail
19 automaticallyGenerated: boolean 18 automaticallyGenerated: boolean
@@ -35,6 +34,7 @@ function updatePlaylistMiniatureFromExisting (options: {
35 width, 34 width,
36 type, 35 type,
37 automaticallyGenerated, 36 automaticallyGenerated,
37 onDisk: true,
38 existingThumbnail 38 existingThumbnail
39 }) 39 })
40} 40}
@@ -57,7 +57,7 @@ function updatePlaylistMiniatureFromUrl (options: {
57 return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) 57 return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })
58 } 58 }
59 59
60 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) 60 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true })
61} 61}
62 62
63function updateVideoMiniatureFromUrl (options: { 63function updateVideoMiniatureFromUrl (options: {
@@ -89,10 +89,10 @@ function updateVideoMiniatureFromUrl (options: {
89 return Promise.resolve() 89 return Promise.resolve()
90 } 90 }
91 91
92 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) 92 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true })
93} 93}
94 94
95function updateVideoMiniatureFromExisting (options: { 95function updateLocalVideoMiniatureFromExisting (options: {
96 inputPath: string 96 inputPath: string
97 video: MVideoThumbnail 97 video: MVideoThumbnail
98 type: ThumbnailType 98 type: ThumbnailType
@@ -115,11 +115,12 @@ function updateVideoMiniatureFromExisting (options: {
115 width, 115 width,
116 type, 116 type,
117 automaticallyGenerated, 117 automaticallyGenerated,
118 existingThumbnail 118 existingThumbnail,
119 onDisk: true
119 }) 120 })
120} 121}
121 122
122function generateVideoMiniature (options: { 123function generateLocalVideoMiniature (options: {
123 video: MVideoThumbnail 124 video: MVideoThumbnail
124 videoFile: MVideoFile 125 videoFile: MVideoFile
125 type: ThumbnailType 126 type: ThumbnailType
@@ -150,34 +151,36 @@ function generateVideoMiniature (options: {
150 width, 151 width,
151 type, 152 type,
152 automaticallyGenerated: true, 153 automaticallyGenerated: true,
154 onDisk: true,
153 existingThumbnail 155 existingThumbnail
154 }) 156 })
155 }) 157 })
156} 158}
157 159
158function updatePlaceholderThumbnail (options: { 160function updateRemoteThumbnail (options: {
159 fileUrl: string 161 fileUrl: string
160 video: MVideoThumbnail 162 video: MVideoThumbnail
161 type: ThumbnailType 163 type: ThumbnailType
162 size: ImageSize 164 size: ImageSize
165 onDisk: boolean
163}) { 166}) {
164 const { fileUrl, video, type, size } = options 167 const { fileUrl, video, type, size, onDisk } = options
165 const { filename: updatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) 168 const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
166 169
167 const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, fileUrl, video) 170 const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)
168 171
169 const thumbnail = existingThumbnail || new ThumbnailModel() 172 const thumbnail = existingThumbnail || new ThumbnailModel()
170 173
171 // Do not change the thumbnail filename if the file did not change 174 // Do not change the thumbnail filename if the file did not change
172 const filename = thumbnailUrlChanged 175 if (thumbnailUrlChanged) {
173 ? updatedFilename 176 thumbnail.filename = generatedFilename
174 : existingThumbnail.filename 177 }
175 178
176 thumbnail.filename = filename
177 thumbnail.height = height 179 thumbnail.height = height
178 thumbnail.width = width 180 thumbnail.width = width
179 thumbnail.type = type 181 thumbnail.type = type
180 thumbnail.fileUrl = fileUrl 182 thumbnail.fileUrl = fileUrl
183 thumbnail.onDisk = onDisk
181 184
182 return thumbnail 185 return thumbnail
183} 186}
@@ -185,14 +188,18 @@ function updatePlaceholderThumbnail (options: {
185// --------------------------------------------------------------------------- 188// ---------------------------------------------------------------------------
186 189
187export { 190export {
188 generateVideoMiniature, 191 generateLocalVideoMiniature,
189 updateVideoMiniatureFromUrl, 192 updateVideoMiniatureFromUrl,
190 updateVideoMiniatureFromExisting, 193 updateLocalVideoMiniatureFromExisting,
191 updatePlaceholderThumbnail, 194 updateRemoteThumbnail,
192 updatePlaylistMiniatureFromUrl, 195 updatePlaylistMiniatureFromUrl,
193 updatePlaylistMiniatureFromExisting 196 updateLocalPlaylistMiniatureFromExisting
194} 197}
195 198
199// ---------------------------------------------------------------------------
200// Private
201// ---------------------------------------------------------------------------
202
196function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { 203function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) {
197 const existingUrl = existingThumbnail 204 const existingUrl = existingThumbnail
198 ? existingThumbnail.fileUrl 205 ? existingThumbnail.fileUrl
@@ -258,6 +265,7 @@ async function updateThumbnailFromFunction (parameters: {
258 height: number 265 height: number
259 width: number 266 width: number
260 type: ThumbnailType 267 type: ThumbnailType
268 onDisk: boolean
261 automaticallyGenerated?: boolean 269 automaticallyGenerated?: boolean
262 fileUrl?: string 270 fileUrl?: string
263 existingThumbnail?: MThumbnail 271 existingThumbnail?: MThumbnail
@@ -269,6 +277,7 @@ async function updateThumbnailFromFunction (parameters: {
269 height, 277 height,
270 type, 278 type,
271 existingThumbnail, 279 existingThumbnail,
280 onDisk,
272 automaticallyGenerated = null, 281 automaticallyGenerated = null,
273 fileUrl = null 282 fileUrl = null
274 } = parameters 283 } = parameters
@@ -285,6 +294,7 @@ async function updateThumbnailFromFunction (parameters: {
285 thumbnail.type = type 294 thumbnail.type = type
286 thumbnail.fileUrl = fileUrl 295 thumbnail.fileUrl = fileUrl
287 thumbnail.automaticallyGenerated = automaticallyGenerated 296 thumbnail.automaticallyGenerated = automaticallyGenerated
297 thumbnail.onDisk = onDisk
288 298
289 if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename 299 if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename
290 300
diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts
index 0ac667ba3..ef9c38731 100644
--- a/server/lib/video-pre-import.ts
+++ b/server/lib/video-pre-import.ts
@@ -29,7 +29,7 @@ import {
29} from '@server/types/models' 29} from '@server/types/models'
30import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' 30import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
31import { getLocalVideoActivityPubUrl } from './activitypub/url' 31import { getLocalVideoActivityPubUrl } from './activitypub/url'
32import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail' 32import { updateLocalVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
33import { VideoPasswordModel } from '@server/models/video/video-password' 33import { VideoPasswordModel } from '@server/models/video/video-password'
34 34
35class YoutubeDlImportError extends Error { 35class YoutubeDlImportError extends Error {
@@ -256,7 +256,7 @@ async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
256 type: ThumbnailType 256 type: ThumbnailType
257}): Promise<MThumbnail> { 257}): Promise<MThumbnail> {
258 if (inputPath) { 258 if (inputPath) {
259 return updateVideoMiniatureFromExisting({ 259 return updateLocalVideoMiniatureFromExisting({
260 inputPath, 260 inputPath,
261 video, 261 video,
262 type, 262 type,
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 588dc553f..362c861a5 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -10,7 +10,7 @@ import { FilteredModelAttributes } from '@server/types'
10import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 10import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
11import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' 11import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models'
12import { CreateJobArgument, JobQueue } from './job-queue/job-queue' 12import { CreateJobArgument, JobQueue } from './job-queue/job-queue'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 13import { updateLocalVideoMiniatureFromExisting } from './thumbnail'
14import { moveFilesIfPrivacyChanged } from './video-privacy' 14import { moveFilesIfPrivacyChanged } from './video-privacy'
15 15
16function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 16function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
@@ -55,7 +55,7 @@ async function buildVideoThumbnailsFromReq (options: {
55 const fields = files?.[p.fieldName] 55 const fields = files?.[p.fieldName]
56 56
57 if (fields) { 57 if (fields) {
58 return updateVideoMiniatureFromExisting({ 58 return updateLocalVideoMiniatureFromExisting({
59 inputPath: fields[0].path, 59 inputPath: fields[0].path,
60 video, 60 video,
61 type: p.type, 61 type: p.type,
diff --git a/server/models/video/sql/video/shared/video-table-attributes.ts b/server/models/video/sql/video/shared/video-table-attributes.ts
index 34967cd20..e0fa9d7c1 100644
--- a/server/models/video/sql/video/shared/video-table-attributes.ts
+++ b/server/models/video/sql/video/shared/video-table-attributes.ts
@@ -60,6 +60,7 @@ export class VideoTableAttributes {
60 'height', 60 'height',
61 'width', 61 'width',
62 'fileUrl', 62 'fileUrl',
63 'onDisk',
63 'automaticallyGenerated', 64 'automaticallyGenerated',
64 'videoId', 65 'videoId',
65 'videoPlaylistId', 66 'videoPlaylistId',
diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts
index a4ac581e5..2a1f6a7b4 100644
--- a/server/models/video/thumbnail.ts
+++ b/server/models/video/thumbnail.ts
@@ -69,6 +69,10 @@ export class ThumbnailModel extends Model<Partial<AttributesOnly<ThumbnailModel>
69 @Column 69 @Column
70 automaticallyGenerated: boolean 70 automaticallyGenerated: boolean
71 71
72 @AllowNull(false)
73 @Column
74 onDisk: boolean
75
72 @ForeignKey(() => VideoModel) 76 @ForeignKey(() => VideoModel)
73 @Column 77 @Column
74 videoId: number 78 videoId: number
diff --git a/support/nginx/peertube b/support/nginx/peertube
index 05a59c072..f5b9d131a 100644
--- a/support/nginx/peertube
+++ b/support/nginx/peertube
@@ -199,28 +199,6 @@ server {
199 alias /var/www/peertube/peertube-latest/client/dist/$1; 199 alias /var/www/peertube/peertube-latest/client/dist/$1;
200 } 200 }
201 201
202 # Bypass PeerTube for performance reasons. Optional.
203 location ~ ^/static/(thumbnails|avatars)/ {
204 if ($request_method = 'OPTIONS') {
205 add_header Access-Control-Allow-Origin '*';
206 add_header Access-Control-Allow-Methods 'GET, OPTIONS';
207 add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
208 add_header Access-Control-Max-Age 1728000; # Preflight request can be cached 20 days
209 add_header Content-Type 'text/plain charset=UTF-8';
210 add_header Content-Length 0;
211 return 204;
212 }
213
214 add_header Access-Control-Allow-Origin '*';
215 add_header Access-Control-Allow-Methods 'GET, OPTIONS';
216 add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
217 add_header Cache-Control "public, max-age=7200"; # Cache response 2 hours
218
219 rewrite ^/static/(.*)$ /$1 break;
220
221 try_files $uri @api;
222 }
223
224 location ~ ^(/static/(webseed|streaming-playlists)/private/)|^/download { 202 location ~ ^(/static/(webseed|streaming-playlists)/private/)|^/download {
225 # We can't rate limit a try_files directive, so we need to duplicate @api 203 # We can't rate limit a try_files directive, so we need to duplicate @api
226 204