10 files changed, 265 insertions, 18 deletions

diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts
new file mode 100644
index 000000000..0c508b063
--- /dev/null
+++ b/server/lib/files-cache/avatar-permanent-file-cache.ts
@@ -0,0 +1,27 @@
+import { CONFIG } from '@server/initializers/config'
+import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants'
+import { ActorImageModel } from '@server/models/actor/actor-image'
+import { MActorImage } from '@server/types/models'
+import { AbstractPermanentFileCache } from './shared'
+
+export class AvatarPermanentFileCache extends AbstractPermanentFileCache<MActorImage> {
+
+  constructor () {
+    super(CONFIG.STORAGE.ACTOR_IMAGES_DIR)
+  }
+
+  protected loadModel (filename: string) {
+    return ActorImageModel.loadByName(filename)
+  }
+
+  protected getImageSize (image: MActorImage): { width: number, height: number } {
+    if (image.width && image.height) {
+      return {
+        height: image.height,
+        width: image.width
+      }
+    }
+
+    return ACTOR_IMAGES_SIZE[image.type][0]
+  }
+}
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts
index e5853f7d6..5630a9b80 100644
--- a/server/lib/files-cache/index.ts
+++ b/server/lib/files-cache/index.ts
@@ -1,3 +1,6 @@
-export * from './videos-preview-cache'
-export * from './videos-caption-cache'
-export * from './videos-torrent-cache'
+export * from './avatar-permanent-file-cache'
+export * from './video-miniature-permanent-file-cache'
+export * from './video-captions-simple-file-cache'
+export * from './video-previews-simple-file-cache'
+export * from './video-storyboards-simple-file-cache'
+export * from './video-torrents-simple-file-cache'
diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
new file mode 100644
index 000000000..f990e9872
--- /dev/null
+++ b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
@@ -0,0 +1,132 @@
+import express from 'express'
+import { LRUCache } from 'lru-cache'
+import { Model } from 'sequelize'
+import { logger } from '@server/helpers/logger'
+import { CachePromise } from '@server/helpers/promise-cache'
+import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants'
+import { downloadImageFromWorker } from '@server/lib/worker/parent-process'
+import { HttpStatusCode } from '@shared/models'
+
+type ImageModel = {
+  fileUrl: string
+  filename: string
+  onDisk: boolean
+
+  isOwned (): boolean
+  getPath (): string
+
+  save (): Promise<Model>
+}
+
+export abstract class AbstractPermanentFileCache <M extends ImageModel> {
+  // Unsafe because it can return paths that do not exist anymore
+  private readonly filenameToPathUnsafeCache = new LRUCache<string, string>({
+    max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE
+  })
+
+  protected abstract getImageSize (image: M): { width: number, height: number }
+  protected abstract loadModel (filename: string): Promise<M>
+
+  constructor (private readonly directory: string) {
+
+  }
+
+  async lazyServe (options: {
+    filename: string
+    res: express.Response
+    next: express.NextFunction
+  }) {
+    const { filename, res, next } = options
+
+    if (this.filenameToPathUnsafeCache.has(filename)) {
+      return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
+    }
+
+    const image = await this.lazyLoadIfNeeded(filename)
+    if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
+
+    const path = image.getPath()
+    this.filenameToPathUnsafeCache.set(filename, path)
+
+    return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => {
+      if (!err) return
+
+      this.onServeError({ err, image, next, filename })
+    })
+  }
+
+  @CachePromise({
+    keyBuilder: filename => filename
+  })
+  private async lazyLoadIfNeeded (filename: string) {
+    const image = await this.loadModel(filename)
+    if (!image) return undefined
+
+    if (image.onDisk === false) {
+      if (!image.fileUrl) return undefined
+
+      try {
+        await this.downloadRemoteFile(image)
+      } catch (err) {
+        logger.warn('Cannot process remote image %s.', image.fileUrl, { err })
+
+        return undefined
+      }
+    }
+
+    return image
+  }
+
+  async downloadRemoteFile (image: M) {
+    logger.info('Download remote image %s lazily.', image.fileUrl)
+
+    const destination = await this.downloadImage({
+      filename: image.filename,
+      fileUrl: image.fileUrl,
+      size: this.getImageSize(image)
+    })
+
+    image.onDisk = true
+    image.save()
+      .catch(err => logger.error('Cannot save new image disk state.', { err }))
+
+    return destination
+  }
+
+  private onServeError (options: {
+    err: any
+    image: M
+    filename: string
+    next: express.NextFunction
+  }) {
+    const { err, image, filename, next } = options
+
+    // It seems this actor image is not on the disk anymore
+    if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) {
+      logger.error('Cannot lazy serve image %s.', filename, { err })
+
+      this.filenameToPathUnsafeCache.delete(filename)
+
+      image.onDisk = false
+      image.save()
+        .catch(err => logger.error('Cannot save new image disk state.', { err }))
+    }
+
+    return next(err)
+  }
+
+  private downloadImage (options: {
+    fileUrl: string
+    filename: string
+    size: { width: number, height: number }
+  }) {
+    const downloaderOptions = {
+      url: options.fileUrl,
+      destDir: this.directory,
+      destName: options.filename,
+      size: options.size
+    }
+
+    return downloadImageFromWorker(downloaderOptions)
+  }
+}
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts
index a7ac88525..6fab322cd 100644
--- a/server/lib/files-cache/abstract-video-static-file-cache.ts
+++ b/server/lib/files-cache/shared/abstract-simple-file-cache.ts
@@ -1,10 +1,10 @@
 import { remove } from 'fs-extra'
-import { logger } from '../../helpers/logger'
+import { logger } from '../../../helpers/logger'
 import memoizee from 'memoizee'
 
 type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined
 
-export abstract class AbstractVideoStaticFileCache <T> {
+export abstract class AbstractSimpleFileCache <T> {
 
   getFilePath: (params: T) => Promise<GetFilePathResult>
 
diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts
new file mode 100644
index 000000000..61c4aacc7
--- /dev/null
+++ b/server/lib/files-cache/shared/index.ts
@@ -0,0 +1,2 @@
+export * from './abstract-permanent-file-cache'
+export * from './abstract-simple-file-cache'
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/video-captions-simple-file-cache.ts
index d21acf4ef..cbeeff732 100644
--- a/server/lib/files-cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/video-captions-simple-file-cache.ts
@@ -5,11 +5,11 @@ import { CONFIG } from '../../initializers/config'
 import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
 import { VideoCaptionModel } from '../../models/video/video-caption'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 
-class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
+class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosCaptionCache
+  private static instance: VideoCaptionsSimpleFileCache
 
   private constructor () {
     super()
@@ -23,7 +23,9 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
     const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename)
     if (!videoCaption) return undefined
 
-    if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
+    if (videoCaption.isOwned()) {
+      return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
+    }
 
     return this.loadRemoteFile(filename)
   }
@@ -55,5 +57,5 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosCaptionCache
+  VideoCaptionsSimpleFileCache
 }
diff --git a/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/lib/files-cache/video-miniature-permanent-file-cache.ts
new file mode 100644
index 000000000..35d9466f7
--- /dev/null
+++ b/server/lib/files-cache/video-miniature-permanent-file-cache.ts
@@ -0,0 +1,28 @@
+import { CONFIG } from '@server/initializers/config'
+import { THUMBNAILS_SIZE } from '@server/initializers/constants'
+import { ThumbnailModel } from '@server/models/video/thumbnail'
+import { MThumbnail } from '@server/types/models'
+import { ThumbnailType } from '@shared/models'
+import { AbstractPermanentFileCache } from './shared'
+
+export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache<MThumbnail> {
+
+  constructor () {
+    super(CONFIG.STORAGE.THUMBNAILS_DIR)
+  }
+
+  protected loadModel (filename: string) {
+    return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE)
+  }
+
+  protected getImageSize (image: MThumbnail): { width: number, height: number } {
+    if (image.width && image.height) {
+      return {
+        height: image.height,
+        width: image.width
+      }
+    }
+
+    return THUMBNAILS_SIZE
+  }
+}
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/video-previews-simple-file-cache.ts
index d19c3f4f4..a05e80e16 100644
--- a/server/lib/files-cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/video-previews-simple-file-cache.ts
@@ -1,15 +1,15 @@
 import { join } from 'path'
 import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 import { doRequestAndSaveToFile } from '@server/helpers/requests'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
 import { ThumbnailType } from '@shared/models'
 import { logger } from '@server/helpers/logger'
 
-class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
+class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosPreviewCache
+  private static instance: VideoPreviewsSimpleFileCache
 
   private constructor () {
     super()
@@ -54,5 +54,5 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosPreviewCache
+  VideoPreviewsSimpleFileCache
 }
diff --git a/server/lib/files-cache/video-storyboards-simple-file-cache.ts b/server/lib/files-cache/video-storyboards-simple-file-cache.ts
new file mode 100644
index 000000000..4cd96e70c
--- /dev/null
+++ b/server/lib/files-cache/video-storyboards-simple-file-cache.ts
@@ -0,0 +1,53 @@
+import { join } from 'path'
+import { logger } from '@server/helpers/logger'
+import { doRequestAndSaveToFile } from '@server/helpers/requests'
+import { StoryboardModel } from '@server/models/video/storyboard'
+import { FILES_CACHE } from '../../initializers/constants'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
+
+class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache <string> {
+
+  private static instance: VideoStoryboardsSimpleFileCache
+
+  private constructor () {
+    super()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  async getFilePathImpl (filename: string) {
+    const storyboard = await StoryboardModel.loadWithVideoByFilename(filename)
+    if (!storyboard) return undefined
+
+    if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() }
+
+    return this.loadRemoteFile(storyboard.filename)
+  }
+
+  // Key is the storyboard filename
+  protected async loadRemoteFile (key: string) {
+    const storyboard = await StoryboardModel.loadWithVideoByFilename(key)
+    if (!storyboard) return undefined
+
+    const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename)
+    const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video)
+
+    try {
+      await doRequestAndSaveToFile(remoteUrl, destPath)
+
+      logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath)
+
+      return { isOwned: false, path: destPath }
+    } catch (err) {
+      logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err })
+
+      return undefined
+    }
+  }
+}
+
+export {
+  VideoStoryboardsSimpleFileCache
+}
diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/video-torrents-simple-file-cache.ts
index a6bf98dd4..8bcd0b9bf 100644
--- a/server/lib/files-cache/videos-torrent-cache.ts
+++ b/server/lib/files-cache/video-torrents-simple-file-cache.ts
@@ -6,11 +6,11 @@ import { MVideo, MVideoFile } from '@server/types/models'
 import { CONFIG } from '../../initializers/config'
 import { FILES_CACHE } from '../../initializers/constants'
 import { VideoModel } from '../../models/video/video'
-import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache'
 
-class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
+class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache <string> {
 
-  private static instance: VideosTorrentCache
+  private static instance: VideoTorrentsSimpleFileCache
 
   private constructor () {
     super()
@@ -66,5 +66,5 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> {
 }
 
 export {
-  VideosTorrentCache
+  VideoTorrentsSimpleFileCache
 }