aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-04-17 10:07:00 +0200
committerChocobozzz <me@florianbigard.com>2019-04-24 16:25:52 +0200
commite8bafea35bc930cb8ac5b2d521a188642a1adffe (patch)
tree7537f957ed7307b464e3c90b71b813d992acaade /server/lib
parent94565d52bb2883e09f16d1363170ac9c0dccb7a1 (diff)
downloadPeerTube-e8bafea35bc930cb8ac5b2d521a188642a1adffe.tar.gz
PeerTube-e8bafea35bc930cb8ac5b2d521a188642a1adffe.tar.zst
PeerTube-e8bafea35bc930cb8ac5b2d521a188642a1adffe.zip
Create a dedicated table to track video thumbnails
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/playlist.ts25
-rw-r--r--server/lib/activitypub/videos.ts92
-rw-r--r--server/lib/files-cache/abstract-video-static-file-cache.ts32
-rw-r--r--server/lib/files-cache/videos-caption-cache.ts5
-rw-r--r--server/lib/files-cache/videos-preview-cache.ts11
-rw-r--r--server/lib/job-queue/handlers/video-import.ts41
-rw-r--r--server/lib/thumbnail.ts151
7 files changed, 272 insertions, 85 deletions
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
index f312409bc..341e469f3 100644
--- a/server/lib/activitypub/playlist.ts
+++ b/server/lib/activitypub/playlist.ts
@@ -1,12 +1,12 @@
1import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' 1import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
2import { crawlCollectionPage } from './crawl' 2import { crawlCollectionPage } from './crawl'
3import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY, THUMBNAILS_SIZE } from '../../initializers/constants' 3import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
4import { AccountModel } from '../../models/account/account' 4import { AccountModel } from '../../models/account/account'
5import { isArray } from '../../helpers/custom-validators/misc' 5import { isArray } from '../../helpers/custom-validators/misc'
6import { getOrCreateActorAndServerAndModel } from './actor' 6import { getOrCreateActorAndServerAndModel } from './actor'
7import { logger } from '../../helpers/logger' 7import { logger } from '../../helpers/logger'
8import { VideoPlaylistModel } from '../../models/video/video-playlist' 8import { VideoPlaylistModel } from '../../models/video/video-playlist'
9import { doRequest, downloadImage } from '../../helpers/requests' 9import { doRequest } from '../../helpers/requests'
10import { checkUrlsSameHost } from '../../helpers/activitypub' 10import { checkUrlsSameHost } from '../../helpers/activitypub'
11import * as Bluebird from 'bluebird' 11import * as Bluebird from 'bluebird'
12import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' 12import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
@@ -16,9 +16,8 @@ import { VideoPlaylistElementModel } from '../../models/video/video-playlist-ele
16import { VideoModel } from '../../models/video/video' 16import { VideoModel } from '../../models/video/video'
17import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' 17import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
18import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' 18import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
19import { ActivityIconObject } from '../../../shared/models/activitypub/objects'
20import { CONFIG } from '../../initializers/config'
21import { sequelizeTypescript } from '../../initializers/database' 19import { sequelizeTypescript } from '../../initializers/database'
20import { createPlaylistThumbnailFromUrl } from '../thumbnail'
22 21
23function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) { 22function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) {
24 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED 23 const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED
@@ -97,16 +96,20 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
97 return Promise.resolve() 96 return Promise.resolve()
98 }) 97 })
99 98
100 // Empty playlists generally do not have a miniature, so skip this 99 const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
101 if (accItems.length !== 0) { 100
101 if (playlistObject.icon) {
102 try { 102 try {
103 await generateThumbnailFromUrl(playlist, playlistObject.icon) 103 const thumbnailModel = await createPlaylistThumbnailFromUrl(playlistObject.icon.url, refreshedPlaylist)
104 thumbnailModel.videoPlaylistId = refreshedPlaylist.id
105
106 refreshedPlaylist.setThumbnail(await thumbnailModel.save())
104 } catch (err) { 107 } catch (err) {
105 logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err }) 108 logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
106 } 109 }
107 } 110 }
108 111
109 return resetVideoPlaylistElements(accItems, playlist) 112 return resetVideoPlaylistElements(accItems, refreshedPlaylist)
110} 113}
111 114
112async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> { 115async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
@@ -191,12 +194,6 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: Vide
191 return undefined 194 return undefined
192} 195}
193 196
194function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityIconObject) {
195 const thumbnailName = playlist.getThumbnailName()
196
197 return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
198}
199
200async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { 197async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
201 const options = { 198 const options = {
202 uri: playlistUrl, 199 uri: playlistUrl,
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index b9252e363..16c37a55f 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -3,11 +3,10 @@ import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as request from 'request' 4import * as request from 'request'
5import { 5import {
6 ActivityIconObject,
7 ActivityPlaylistSegmentHashesObject, 6 ActivityPlaylistSegmentHashesObject,
8 ActivityPlaylistUrlObject, 7 ActivityPlaylistUrlObject,
9 ActivityUrlObject, 8 ActivityUrlObject,
10 ActivityVideoUrlObject, 9 ActivityVideoUrlObject, VideoCreate,
11 VideoState 10 VideoState
12} from '../../../shared/index' 11} from '../../../shared/index'
13import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 12import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
@@ -16,8 +15,15 @@ import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validat
16import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 15import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
17import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 16import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
18import { logger } from '../../helpers/logger' 17import { logger } from '../../helpers/logger'
19import { doRequest, downloadImage } from '../../helpers/requests' 18import { doRequest } from '../../helpers/requests'
20import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, REMOTE_SCHEME, THUMBNAILS_SIZE } from '../../initializers/constants' 19import {
20 ACTIVITY_PUB,
21 MIMETYPES,
22 P2P_MEDIA_LOADER_PEER_VERSION,
23 PREVIEWS_SIZE,
24 REMOTE_SCHEME,
25 STATIC_PATHS
26} from '../../initializers/constants'
21import { ActorModel } from '../../models/activitypub/actor' 27import { ActorModel } from '../../models/activitypub/actor'
22import { TagModel } from '../../models/video/tag' 28import { TagModel } from '../../models/video/tag'
23import { VideoModel } from '../../models/video/video' 29import { VideoModel } from '../../models/video/video'
@@ -43,8 +49,11 @@ import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
43import { AccountVideoRateModel } from '../../models/account/account-video-rate' 49import { AccountVideoRateModel } from '../../models/account/account-video-rate'
44import { VideoShareModel } from '../../models/video/video-share' 50import { VideoShareModel } from '../../models/video/video-share'
45import { VideoCommentModel } from '../../models/video/video-comment' 51import { VideoCommentModel } from '../../models/video/video-comment'
46import { CONFIG } from '../../initializers/config'
47import { sequelizeTypescript } from '../../initializers/database' 52import { sequelizeTypescript } from '../../initializers/database'
53import { createPlaceholderThumbnail, createVideoThumbnailFromUrl } from '../thumbnail'
54import { ThumbnailModel } from '../../models/video/thumbnail'
55import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
56import { join } from 'path'
48 57
49async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 58async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
50 // If the video is not private and is published, we federate it 59 // If the video is not private and is published, we federate it
@@ -100,18 +109,18 @@ async function fetchRemoteVideoDescription (video: VideoModel) {
100} 109}
101 110
102function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { 111function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) {
103 const host = video.VideoChannel.Account.Actor.Server.host 112 const url = buildRemoteBaseUrl(video, path)
104 113
105 // We need to provide a callback, if no we could have an uncaught exception 114 // We need to provide a callback, if no we could have an uncaught exception
106 return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { 115 return request.get(url, err => {
107 if (err) reject(err) 116 if (err) reject(err)
108 }) 117 })
109} 118}
110 119
111function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { 120function buildRemoteBaseUrl (video: VideoModel, path: string) {
112 const thumbnailName = video.getThumbnailName() 121 const host = video.VideoChannel.Account.Actor.Server.host
113 122
114 return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) 123 return REMOTE_SCHEME.HTTP + '://' + host + path
115} 124}
116 125
117function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { 126function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
@@ -236,6 +245,14 @@ async function updateVideoFromAP (options: {
236 const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED 245 const wasUnlistedVideo = options.video.privacy === VideoPrivacy.UNLISTED
237 246
238 try { 247 try {
248 let thumbnailModel: ThumbnailModel
249
250 try {
251 thumbnailModel = await createVideoThumbnailFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.THUMBNAIL)
252 } catch (err) {
253 logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
254 }
255
239 await sequelizeTypescript.transaction(async t => { 256 await sequelizeTypescript.transaction(async t => {
240 const sequelizeOptions = { transaction: t } 257 const sequelizeOptions = { transaction: t }
241 258
@@ -272,6 +289,17 @@ async function updateVideoFromAP (options: {
272 289
273 await options.video.save(sequelizeOptions) 290 await options.video.save(sequelizeOptions)
274 291
292 if (thumbnailModel) {
293 thumbnailModel.videoId = options.video.id
294 options.video.addThumbnail(await thumbnailModel.save({ transaction: t }))
295 }
296
297 // FIXME: use icon URL instead
298 const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename))
299 const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
300
301 options.video.addThumbnail(await previewModel.save({ transaction: t }))
302
275 { 303 {
276 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) 304 const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
277 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) 305 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
@@ -347,12 +375,6 @@ async function updateVideoFromAP (options: {
347 logger.debug('Cannot update the remote video.', { err }) 375 logger.debug('Cannot update the remote video.', { err })
348 throw err 376 throw err
349 } 377 }
350
351 try {
352 await generateThumbnailFromUrl(options.video, options.videoObject.icon)
353 } catch (err) {
354 logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
355 }
356} 378}
357 379
358async function refreshVideoIfNeeded (options: { 380async function refreshVideoIfNeeded (options: {
@@ -412,7 +434,6 @@ export {
412 getOrCreateVideoAndAccountAndChannel, 434 getOrCreateVideoAndAccountAndChannel,
413 fetchRemoteVideoStaticFile, 435 fetchRemoteVideoStaticFile,
414 fetchRemoteVideoDescription, 436 fetchRemoteVideoDescription,
415 generateThumbnailFromUrl,
416 getOrCreateVideoChannelFromVideoObject 437 getOrCreateVideoChannelFromVideoObject
417} 438}
418 439
@@ -440,13 +461,34 @@ function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistS
440async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 461async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
441 logger.debug('Adding remote video %s.', videoObject.id) 462 logger.debug('Adding remote video %s.', videoObject.id)
442 463
464 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
465 const video = VideoModel.build(videoData)
466
467 const promiseThumbnail = createVideoThumbnailFromUrl(videoObject.icon.url, video, ThumbnailType.THUMBNAIL)
468
469 let thumbnailModel: ThumbnailModel
470 if (waitThumbnail === true) {
471 thumbnailModel = await promiseThumbnail
472 }
473
443 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { 474 const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
444 const sequelizeOptions = { transaction: t } 475 const sequelizeOptions = { transaction: t }
445 476
446 const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
447 const video = VideoModel.build(videoData)
448
449 const videoCreated = await video.save(sequelizeOptions) 477 const videoCreated = await video.save(sequelizeOptions)
478 videoCreated.VideoChannel = channelActor.VideoChannel
479
480 if (thumbnailModel) {
481 thumbnailModel.videoId = videoCreated.id
482
483 videoCreated.addThumbnail(await thumbnailModel.save({ transaction: t }))
484 }
485
486 // FIXME: use icon URL instead
487 const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName()))
488 const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE)
489 previewModel.videoId = videoCreated.id
490
491 videoCreated.addThumbnail(await previewModel.save({ transaction: t }))
450 492
451 // Process files 493 // Process files
452 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) 494 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
@@ -476,14 +518,16 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
476 518
477 logger.info('Remote video with uuid %s inserted.', videoObject.uuid) 519 logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
478 520
479 videoCreated.VideoChannel = channelActor.VideoChannel
480 return videoCreated 521 return videoCreated
481 }) 522 })
482 523
483 const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) 524 if (waitThumbnail === false) {
484 .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) 525 promiseThumbnail.then(thumbnailModel => {
526 thumbnailModel = videoCreated.id
485 527
486 if (waitThumbnail === true) await p 528 return thumbnailModel.save()
529 })
530 }
487 531
488 return videoCreated 532 return videoCreated
489} 533}
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts
index 7512f2b9d..61837e0f8 100644
--- a/server/lib/files-cache/abstract-video-static-file-cache.ts
+++ b/server/lib/files-cache/abstract-video-static-file-cache.ts
@@ -1,41 +1,29 @@
1import * as AsyncLRU from 'async-lru'
2import { createWriteStream, remove } from 'fs-extra' 1import { createWriteStream, remove } from 'fs-extra'
3import { logger } from '../../helpers/logger' 2import { logger } from '../../helpers/logger'
4import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
5import { fetchRemoteVideoStaticFile } from '../activitypub' 4import { fetchRemoteVideoStaticFile } from '../activitypub'
5import * as memoizee from 'memoizee'
6 6
7export abstract class AbstractVideoStaticFileCache <T> { 7export abstract class AbstractVideoStaticFileCache <T> {
8 8
9 protected lru 9 getFilePath: (params: T) => Promise<string>
10 10
11 abstract getFilePath (params: T): Promise<string> 11 abstract getFilePathImpl (params: T): Promise<string>
12 12
13 // Load and save the remote file, then return the local path from filesystem 13 // Load and save the remote file, then return the local path from filesystem
14 protected abstract loadRemoteFile (key: string): Promise<string> 14 protected abstract loadRemoteFile (key: string): Promise<string>
15 15
16 init (max: number, maxAge: number) { 16 init (max: number, maxAge: number) {
17 this.lru = new AsyncLRU({ 17 this.getFilePath = memoizee(this.getFilePathImpl, {
18 max,
19 maxAge, 18 maxAge,
20 load: (key, cb) => { 19 max,
21 this.loadRemoteFile(key) 20 promise: true,
22 .then(res => cb(null, res)) 21 dispose: (value: string) => {
23 .catch(err => cb(err)) 22 remove(value)
23 .then(() => logger.debug('%s evicted from %s', value, this.constructor.name))
24 .catch(err => logger.error('Cannot remove %s from cache %s.', value, this.constructor.name, { err }))
24 } 25 }
25 }) 26 })
26
27 this.lru.on('evict', (obj: { key: string, value: string }) => {
28 remove(obj.value)
29 .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name))
30 })
31 }
32
33 protected loadFromLRU (key: string) {
34 return new Promise<string>((res, rej) => {
35 this.lru.get(key, (err, value) => {
36 err ? rej(err) : res(value)
37 })
38 })
39 } 27 }
40 28
41 protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) { 29 protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) {
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts
index 0926f4009..d4a0a3345 100644
--- a/server/lib/files-cache/videos-caption-cache.ts
+++ b/server/lib/files-cache/videos-caption-cache.ts
@@ -20,14 +20,14 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
20 return this.instance || (this.instance = new this()) 20 return this.instance || (this.instance = new this())
21 } 21 }
22 22
23 async getFilePath (params: GetPathParam) { 23 async getFilePathImpl (params: GetPathParam) {
24 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language) 24 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language)
25 if (!videoCaption) return undefined 25 if (!videoCaption) return undefined
26 26
27 if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) 27 if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName())
28 28
29 const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language 29 const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language
30 return this.loadFromLRU(key) 30 return this.loadRemoteFile(key)
31 } 31 }
32 32
33 protected async loadRemoteFile (key: string) { 33 protected async loadRemoteFile (key: string) {
@@ -42,6 +42,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
42 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) 42 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
43 if (!video) return undefined 43 if (!video) return undefined
44 44
45 // FIXME: use URL
45 const remoteStaticPath = videoCaption.getCaptionStaticPath() 46 const remoteStaticPath = videoCaption.getCaptionStaticPath()
46 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) 47 const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
47 48
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts
index 6575e1c83..fc0d92c78 100644
--- a/server/lib/files-cache/videos-preview-cache.ts
+++ b/server/lib/files-cache/videos-preview-cache.ts
@@ -16,13 +16,13 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
16 return this.instance || (this.instance = new this()) 16 return this.instance || (this.instance = new this())
17 } 17 }
18 18
19 async getFilePath (videoUUID: string) { 19 async getFilePathImpl (videoUUID: string) {
20 const video = await VideoModel.loadByUUIDWithFile(videoUUID) 20 const video = await VideoModel.loadByUUIDWithFile(videoUUID)
21 if (!video) return undefined 21 if (!video) return undefined
22 22
23 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) 23 if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreview().filename)
24 24
25 return this.loadFromLRU(videoUUID) 25 return this.loadRemoteFile(videoUUID)
26 } 26 }
27 27
28 protected async loadRemoteFile (key: string) { 28 protected async loadRemoteFile (key: string) {
@@ -31,8 +31,9 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
31 31
32 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') 32 if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
33 33
34 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) 34 // FIXME: use URL
35 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName()) 35 const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreview().filename)
36 const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreview().filename)
36 37
37 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) 38 return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
38 } 39 }
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 8e8aa1597..3fa0dd65d 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -6,8 +6,7 @@ import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path' 7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file' 8import { VideoFileModel } from '../../../models/video/video-file'
9import { PREVIEWS_SIZE, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
10import { downloadImage } from '../../../helpers/requests'
11import { VideoState } from '../../../../shared' 10import { VideoState } from '../../../../shared'
12import { JobQueue } from '../index' 11import { JobQueue } from '../index'
13import { federateVideoIfNeeded } from '../../activitypub' 12import { federateVideoIfNeeded } from '../../activitypub'
@@ -18,6 +17,9 @@ import { move, remove, stat } from 'fs-extra'
18import { Notifier } from '../../notifier' 17import { Notifier } from '../../notifier'
19import { CONFIG } from '../../../initializers/config' 18import { CONFIG } from '../../../initializers/config'
20import { sequelizeTypescript } from '../../../initializers/database' 19import { sequelizeTypescript } from '../../../initializers/database'
20import { ThumbnailModel } from '../../../models/video/thumbnail'
21import { createVideoThumbnailFromUrl, generateVideoThumbnail } from '../../thumbnail'
22import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
21 23
22type VideoImportYoutubeDLPayload = { 24type VideoImportYoutubeDLPayload = {
23 type: 'youtube-dl' 25 type: 'youtube-dl'
@@ -146,25 +148,19 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
146 tempVideoPath = null // This path is not used anymore 148 tempVideoPath = null // This path is not used anymore
147 149
148 // Process thumbnail 150 // Process thumbnail
149 if (options.downloadThumbnail) { 151 let thumbnailModel: ThumbnailModel
150 if (options.thumbnailUrl) { 152 if (options.downloadThumbnail && options.thumbnailUrl) {
151 await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE) 153 thumbnailModel = await createVideoThumbnailFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.THUMBNAIL)
152 } else { 154 } else if (options.generateThumbnail || options.downloadThumbnail) {
153 await videoImport.Video.createThumbnail(videoFile) 155 thumbnailModel = await generateVideoThumbnail(videoImport.Video, videoFile, ThumbnailType.THUMBNAIL)
154 }
155 } else if (options.generateThumbnail) {
156 await videoImport.Video.createThumbnail(videoFile)
157 } 156 }
158 157
159 // Process preview 158 // Process preview
160 if (options.downloadPreview) { 159 let previewModel: ThumbnailModel
161 if (options.thumbnailUrl) { 160 if (options.downloadPreview && options.thumbnailUrl) {
162 await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE) 161 previewModel = await createVideoThumbnailFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.PREVIEW)
163 } else { 162 } else if (options.generatePreview || options.downloadPreview) {
164 await videoImport.Video.createPreview(videoFile) 163 previewModel = await generateVideoThumbnail(videoImport.Video, videoFile, ThumbnailType.PREVIEW)
165 }
166 } else if (options.generatePreview) {
167 await videoImport.Video.createPreview(videoFile)
168 } 164 }
169 165
170 // Create torrent 166 // Create torrent
@@ -184,6 +180,15 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
184 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 180 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
185 await video.save({ transaction: t }) 181 await video.save({ transaction: t })
186 182
183 if (thumbnailModel) {
184 thumbnailModel.videoId = video.id
185 video.addThumbnail(await thumbnailModel.save({ transaction: t }))
186 }
187 if (previewModel) {
188 previewModel.videoId = video.id
189 video.addThumbnail(await previewModel.save({ transaction: t }))
190 }
191
187 // Now we can federate the video (reload from database, we need more attributes) 192 // Now we can federate the video (reload from database, we need more attributes)
188 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 193 const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
189 await federateVideoIfNeeded(videoForFederation, true, t) 194 await federateVideoIfNeeded(videoForFederation, true, t)
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
new file mode 100644
index 000000000..344c28566
--- /dev/null
+++ b/server/lib/thumbnail.ts
@@ -0,0 +1,151 @@
1import { VideoFileModel } from '../models/video/video-file'
2import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
3import { CONFIG } from '../initializers/config'
4import { PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants'
5import { VideoModel } from '../models/video/video'
6import { ThumbnailModel } from '../models/video/thumbnail'
7import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
8import { processImage } from '../helpers/image-utils'
9import { join } from 'path'
10import { downloadImage } from '../helpers/requests'
11import { VideoPlaylistModel } from '../models/video/video-playlist'
12
13type ImageSize = { height: number, width: number }
14
15function createPlaylistThumbnailFromExisting (inputPath: string, playlist: VideoPlaylistModel, keepOriginal = false, size?: ImageSize) {
16 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
17 const type = ThumbnailType.THUMBNAIL
18
19 const thumbnailCreator = () => processImage({ path: inputPath }, outputPath, { width, height }, keepOriginal)
20 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail })
21}
22
23function createPlaylistThumbnailFromUrl (url: string, playlist: VideoPlaylistModel, size?: ImageSize) {
24 const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
25 const type = ThumbnailType.THUMBNAIL
26
27 const thumbnailCreator = () => downloadImage(url, basePath, filename, { width, height })
28 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, url })
29}
30
31function createVideoThumbnailFromUrl (url: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) {
32 const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
33 const thumbnailCreator = () => downloadImage(url, basePath, filename, { width, height })
34
35 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, url })
36}
37
38function createVideoThumbnailFromExisting (inputPath: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) {
39 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
40 const thumbnailCreator = () => processImage({ path: inputPath }, outputPath, { width, height })
41
42 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail })
43}
44
45function generateVideoThumbnail (video: VideoModel, videoFile: VideoFileModel, type: ThumbnailType) {
46 const input = video.getVideoFilePath(videoFile)
47
48 const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type)
49 const thumbnailCreator = () => generateImageFromVideoFile(input, basePath, filename, { height, width })
50
51 return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail })
52}
53
54function createPlaceholderThumbnail (url: string, video: VideoModel, type: ThumbnailType, size: ImageSize) {
55 const { filename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
56
57 const thumbnail = existingThumbnail ? existingThumbnail : new ThumbnailModel()
58
59 thumbnail.filename = filename
60 thumbnail.height = height
61 thumbnail.width = width
62 thumbnail.type = type
63 thumbnail.url = url
64
65 return thumbnail
66}
67
68// ---------------------------------------------------------------------------
69
70export {
71 generateVideoThumbnail,
72 createVideoThumbnailFromUrl,
73 createVideoThumbnailFromExisting,
74 createPlaceholderThumbnail,
75 createPlaylistThumbnailFromUrl,
76 createPlaylistThumbnailFromExisting
77}
78
79function buildMetadataFromPlaylist (playlist: VideoPlaylistModel, size: ImageSize) {
80 const filename = playlist.generateThumbnailName()
81 const basePath = CONFIG.STORAGE.THUMBNAILS_DIR
82
83 return {
84 filename,
85 basePath,
86 existingThumbnail: playlist.Thumbnail,
87 outputPath: join(basePath, filename),
88 height: size ? size.height : THUMBNAILS_SIZE.height,
89 width: size ? size.width : THUMBNAILS_SIZE.width
90 }
91}
92
93function buildMetadataFromVideo (video: VideoModel, type: ThumbnailType, size?: ImageSize) {
94 const existingThumbnail = Array.isArray(video.Thumbnails)
95 ? video.Thumbnails.find(t => t.type === type)
96 : undefined
97
98 if (type === ThumbnailType.THUMBNAIL) {
99 const filename = video.generateThumbnailName()
100 const basePath = CONFIG.STORAGE.THUMBNAILS_DIR
101
102 return {
103 filename,
104 basePath,
105 existingThumbnail,
106 outputPath: join(basePath, filename),
107 height: size ? size.height : THUMBNAILS_SIZE.height,
108 width: size ? size.width : THUMBNAILS_SIZE.width
109 }
110 }
111
112 if (type === ThumbnailType.PREVIEW) {
113 const filename = video.generatePreviewName()
114 const basePath = CONFIG.STORAGE.PREVIEWS_DIR
115
116 return {
117 filename,
118 basePath,
119 existingThumbnail,
120 outputPath: join(basePath, filename),
121 height: size ? size.height : PREVIEWS_SIZE.height,
122 width: size ? size.width : PREVIEWS_SIZE.width
123 }
124 }
125
126 return undefined
127}
128
129async function createThumbnailFromFunction (parameters: {
130 thumbnailCreator: () => Promise<any>,
131 filename: string,
132 height: number,
133 width: number,
134 type: ThumbnailType,
135 url?: string,
136 existingThumbnail?: ThumbnailModel
137}) {
138 const { thumbnailCreator, filename, width, height, type, existingThumbnail, url = null } = parameters
139
140 const thumbnail = existingThumbnail ? existingThumbnail : new ThumbnailModel()
141
142 thumbnail.filename = filename
143 thumbnail.height = height
144 thumbnail.width = width
145 thumbnail.type = type
146 thumbnail.url = url
147
148 await thumbnailCreator()
149
150 return thumbnail
151}