aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/process/process-create.ts2
-rw-r--r--server/lib/activitypub/videos/shared/creator.ts2
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts2
-rw-r--r--server/lib/client-html.ts6
-rw-r--r--server/lib/emailer.ts2
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts17
-rw-r--r--server/lib/job-queue/job-queue.ts11
-rw-r--r--server/lib/local-actor.ts38
-rw-r--r--server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts2
-rw-r--r--server/lib/redis.ts10
-rw-r--r--server/lib/schedulers/geo-ip-update-scheduler.ts2
-rw-r--r--server/lib/signup.ts2
-rw-r--r--server/lib/thumbnail.ts38
-rw-r--r--server/lib/video.ts24
-rw-r--r--server/lib/views/shared/video-viewer-counters.ts8
-rw-r--r--server/lib/views/shared/video-viewer-stats.ts6
-rw-r--r--server/lib/views/shared/video-views.ts17
-rw-r--r--server/lib/views/video-views-manager.ts4
-rw-r--r--server/lib/worker/parent-process.ts40
-rw-r--r--server/lib/worker/workers/image-downloader.ts33
-rw-r--r--server/lib/worker/workers/image-processor.ts7
21 files changed, 204 insertions, 69 deletions
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 3e7931bb2..76ed37aae 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -124,7 +124,7 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc
124 return 124 return
125 } 125 }
126 126
127 // Try to not forward unwanted commments on our videos 127 // Try to not forward unwanted comments on our videos
128 if (video.isOwned()) { 128 if (video.isOwned()) {
129 if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { 129 if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) {
130 logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) 130 logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url)
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts
index 688bcbb53..07252fea2 100644
--- a/server/lib/activitypub/videos/shared/creator.ts
+++ b/server/lib/activitypub/videos/shared/creator.ts
@@ -24,7 +24,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
24 const channel = channelActor.VideoChannel 24 const channel = channelActor.VideoChannel
25 25
26 const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) 26 const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to)
27 const video = VideoModel.build(videoData) as MVideoThumbnail 27 const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail
28 28
29 const promiseThumbnail = this.tryToGenerateThumbnail(video) 29 const promiseThumbnail = this.tryToGenerateThumbnail(video)
30 30
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index f02b9cba6..86699c5b8 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -210,8 +210,6 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi
210 210
211 updatedAt: new Date(videoObject.updated), 211 updatedAt: new Date(videoObject.updated),
212 views: videoObject.views, 212 views: videoObject.views,
213 likes: 0,
214 dislikes: 0,
215 remote: true, 213 remote: true,
216 privacy 214 privacy
217 } 215 }
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts
index 337364ac9..1e8d03023 100644
--- a/server/lib/client-html.ts
+++ b/server/lib/client-html.ts
@@ -30,6 +30,7 @@ import { MAccountActor, MChannelActor } from '../types/models'
30import { getActivityStreamDuration } from './activitypub/activity' 30import { getActivityStreamDuration } from './activitypub/activity'
31import { getBiggestActorImage } from './actor-image' 31import { getBiggestActorImage } from './actor-image'
32import { ServerConfigManager } from './server-config-manager' 32import { ServerConfigManager } from './server-config-manager'
33import { isTestInstance } from '@server/helpers/core-utils'
33 34
34type Tags = { 35type Tags = {
35 ogType: string 36 ogType: string
@@ -232,7 +233,10 @@ class ClientHtml {
232 static async getEmbedHTML () { 233 static async getEmbedHTML () {
233 const path = ClientHtml.getEmbedPath() 234 const path = ClientHtml.getEmbedPath()
234 235
235 if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] 236 // Disable HTML cache in dev mode because webpack can regenerate JS files
237 if (!isTestInstance() && ClientHtml.htmlCache[path]) {
238 return ClientHtml.htmlCache[path]
239 }
236 240
237 const buffer = await readFile(path) 241 const buffer = await readFile(path)
238 const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() 242 const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig()
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts
index aebca04fe..edc99057c 100644
--- a/server/lib/emailer.ts
+++ b/server/lib/emailer.ts
@@ -179,7 +179,7 @@ class Emailer {
179 } 179 }
180 } 180 }
181 181
182 // overriden/new variables given for a specific template in the payload 182 // overridden/new variables given for a specific template in the payload
183 const sendOptions = merge(baseOptions, options) 183 const sendOptions = merge(baseOptions, options)
184 184
185 await email.send(sendOptions) 185 await email.send(sendOptions)
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
index f480b32cd..49064052c 100644
--- a/server/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -48,15 +48,24 @@ export async function processMoveToObjectStorage (job: Job) {
48 await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) 48 await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
49 } 49 }
50 } catch (err) { 50 } catch (err) {
51 logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags }) 51 await onMoveToObjectStorageFailure(job, err)
52
53 await moveToFailedMoveToObjectStorageState(video)
54 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
55 } 52 }
56 53
57 return payload.videoUUID 54 return payload.videoUUID
58} 55}
59 56
57export async function onMoveToObjectStorageFailure (job: Job, err: any) {
58 const payload = job.data as MoveObjectStoragePayload
59
60 const video = await VideoModel.loadWithFiles(payload.videoUUID)
61 if (!video) return
62
63 logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
64
65 await moveToFailedMoveToObjectStorageState(video)
66 await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
67}
68
60// --------------------------------------------------------------------------- 69// ---------------------------------------------------------------------------
61 70
62async function moveWebTorrentFiles (video: MVideoWithAllFiles) { 71async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index f339e9135..ce24763f1 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -33,7 +33,7 @@ import { refreshAPObject } from './handlers/activitypub-refresher'
33import { processActorKeys } from './handlers/actor-keys' 33import { processActorKeys } from './handlers/actor-keys'
34import { processEmail } from './handlers/email' 34import { processEmail } from './handlers/email'
35import { processManageVideoTorrent } from './handlers/manage-video-torrent' 35import { processManageVideoTorrent } from './handlers/manage-video-torrent'
36import { processMoveToObjectStorage } from './handlers/move-to-object-storage' 36import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
37import { processVideoFileImport } from './handlers/video-file-import' 37import { processVideoFileImport } from './handlers/video-file-import'
38import { processVideoImport } from './handlers/video-import' 38import { processVideoImport } from './handlers/video-import'
39import { processVideoLiveEnding } from './handlers/video-live-ending' 39import { processVideoLiveEnding } from './handlers/video-live-ending'
@@ -88,6 +88,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
88 'video-studio-edition': processVideoStudioEdition 88 'video-studio-edition': processVideoStudioEdition
89} 89}
90 90
91const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
92 'move-to-object-storage': onMoveToObjectStorageFailure
93}
94
91const jobTypes: JobType[] = [ 95const jobTypes: JobType[] = [
92 'activitypub-follow', 96 'activitypub-follow',
93 'activitypub-http-broadcast', 97 'activitypub-http-broadcast',
@@ -162,6 +166,11 @@ class JobQueue {
162 : 'error' 166 : 'error'
163 167
164 logger.log(logLevel, 'Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) 168 logger.log(logLevel, 'Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err })
169
170 if (errorHandlers[job.name]) {
171 errorHandlers[job.name](job, err)
172 .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err }))
173 }
165 }) 174 })
166 175
167 queue.on('error', err => { 176 queue.on('error', err => {
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts
index 01046d017..1d9be76e2 100644
--- a/server/lib/local-actor.ts
+++ b/server/lib/local-actor.ts
@@ -1,4 +1,3 @@
1import { queue } from 'async'
2import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
3import LRUCache from 'lru-cache' 2import LRUCache from 'lru-cache'
4import { join } from 'path' 3import { join } from 'path'
@@ -7,14 +6,13 @@ import { getLowercaseExtension } from '@shared/core-utils'
7import { buildUUID } from '@shared/extra-utils' 6import { buildUUID } from '@shared/extra-utils'
8import { ActivityPubActorType, ActorImageType } from '@shared/models' 7import { ActivityPubActorType, ActorImageType } from '@shared/models'
9import { retryTransactionWrapper } from '../helpers/database-utils' 8import { retryTransactionWrapper } from '../helpers/database-utils'
10import { processImage } from '../helpers/image-utils'
11import { downloadImage } from '../helpers/requests'
12import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
13import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY, WEBSERVER } from '../initializers/constants' 10import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants'
14import { sequelizeTypescript } from '../initializers/database' 11import { sequelizeTypescript } from '../initializers/database'
15import { MAccountDefault, MActor, MChannelDefault } from '../types/models' 12import { MAccountDefault, MActor, MChannelDefault } from '../types/models'
16import { deleteActorImages, updateActorImages } from './activitypub/actors' 13import { deleteActorImages, updateActorImages } from './activitypub/actors'
17import { sendUpdateActor } from './activitypub/send' 14import { sendUpdateActor } from './activitypub/send'
15import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process'
18 16
19function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { 17function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) {
20 return new ActorModel({ 18 return new ActorModel({
@@ -43,7 +41,7 @@ async function updateLocalActorImageFiles (
43 41
44 const imageName = buildUUID() + extension 42 const imageName = buildUUID() + extension
45 const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) 43 const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName)
46 await processImage(imagePhysicalFile.path, destination, imageSize, true) 44 await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true })
47 45
48 return { 46 return {
49 imageName, 47 imageName,
@@ -87,27 +85,22 @@ async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MC
87 }) 85 })
88} 86}
89 87
90type DownloadImageQueueTask = { 88// ---------------------------------------------------------------------------
89
90function downloadActorImageFromWorker (options: {
91 fileUrl: string 91 fileUrl: string
92 filename: string 92 filename: string
93 type: ActorImageType 93 type: ActorImageType
94 size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0] 94 size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0]
95} 95}) {
96 96 const downloaderOptions = {
97const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => { 97 url: options.fileUrl,
98 downloadImage(task.fileUrl, CONFIG.STORAGE.ACTOR_IMAGES, task.filename, task.size) 98 destDir: CONFIG.STORAGE.ACTOR_IMAGES,
99 .then(() => cb()) 99 destName: options.filename,
100 .catch(err => cb(err)) 100 size: options.size
101}, QUEUE_CONCURRENCY.ACTOR_PROCESS_IMAGE) 101 }
102
103function pushActorImageProcessInQueue (task: DownloadImageQueueTask) {
104 return new Promise<void>((res, rej) => {
105 downloadImageQueue.push(task, err => {
106 if (err) return rej(err)
107 102
108 return res() 103 return downloadImageFromWorker(downloaderOptions)
109 })
110 })
111} 104}
112 105
113// Unsafe so could returns paths that does not exist anymore 106// Unsafe so could returns paths that does not exist anymore
@@ -116,7 +109,8 @@ const actorImagePathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.
116export { 109export {
117 actorImagePathUnsafeCache, 110 actorImagePathUnsafeCache,
118 updateLocalActorImageFiles, 111 updateLocalActorImageFiles,
112 downloadActorImageFromWorker,
119 deleteLocalActorImageFile, 113 deleteLocalActorImageFile,
120 pushActorImageProcessInQueue, 114 downloadImageFromWorker,
121 buildActorInstance 115 buildActorInstance
122} 116}
diff --git a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts
index daefa25bd..a7292de69 100644
--- a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts
+++ b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts
@@ -5,7 +5,7 @@ import { MAbuseFull, MAbuseMessage, MAccountDefault, MUserWithNotificationSettin
5import { UserNotificationType } from '@shared/models' 5import { UserNotificationType } from '@shared/models'
6import { AbstractNotification } from '../common/abstract-notification' 6import { AbstractNotification } from '../common/abstract-notification'
7 7
8export type NewAbuseMessagePayload = { 8type NewAbuseMessagePayload = {
9 abuse: MAbuseFull 9 abuse: MAbuseFull
10 message: MAbuseMessage 10 message: MAbuseMessage
11} 11}
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index d052de786..d6d053d2f 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -1,4 +1,4 @@
1import { createClient, RedisClientOptions, RedisModules, RedisScripts } from 'redis' 1import { createClient, RedisClientOptions, RedisModules } from 'redis'
2import { exists } from '@server/helpers/custom-validators/misc' 2import { exists } from '@server/helpers/custom-validators/misc'
3import { sha256 } from '@shared/extra-utils' 3import { sha256 } from '@shared/extra-utils'
4import { logger } from '../helpers/logger' 4import { logger } from '../helpers/logger'
@@ -16,16 +16,12 @@ import {
16 WEBSERVER 16 WEBSERVER
17} from '../initializers/constants' 17} from '../initializers/constants'
18 18
19// Only used for typings
20// TODO: remove when https://github.com/microsoft/TypeScript/issues/37181 is fixed
21const redisClientWrapperForType = () => createClient<{}, RedisScripts>()
22
23class Redis { 19class Redis {
24 20
25 private static instance: Redis 21 private static instance: Redis
26 private initialized = false 22 private initialized = false
27 private connected = false 23 private connected = false
28 private client: ReturnType<typeof redisClientWrapperForType> 24 private client: ReturnType<typeof createClient>
29 private prefix: string 25 private prefix: string
30 26
31 private constructor () { 27 private constructor () {
@@ -308,7 +304,7 @@ class Redis {
308 return this.deleteKey('resumable-upload-' + uploadId) 304 return this.deleteKey('resumable-upload-' + uploadId)
309 } 305 }
310 306
311 /* ************ AP ressource unavailability ************ */ 307 /* ************ AP resource unavailability ************ */
312 308
313 async addAPUnavailability (url: string) { 309 async addAPUnavailability (url: string) {
314 const key = this.generateAPUnavailabilityKey(url) 310 const key = this.generateAPUnavailabilityKey(url)
diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts
index 9dda6d76c..b06f5a9b5 100644
--- a/server/lib/schedulers/geo-ip-update-scheduler.ts
+++ b/server/lib/schedulers/geo-ip-update-scheduler.ts
@@ -6,7 +6,7 @@ export class GeoIPUpdateScheduler extends AbstractScheduler {
6 6
7 private static instance: AbstractScheduler 7 private static instance: AbstractScheduler
8 8
9 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE 9 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE
10 10
11 private constructor () { 11 private constructor () {
12 super() 12 super()
diff --git a/server/lib/signup.ts b/server/lib/signup.ts
index 3c1397a12..f094531eb 100644
--- a/server/lib/signup.ts
+++ b/server/lib/signup.ts
@@ -26,7 +26,7 @@ function isSignupAllowedForCurrentIP (ip: string) {
26 const excludeList = [ 'blacklist' ] 26 const excludeList = [ 'blacklist' ]
27 let matched = '' 27 let matched = ''
28 28
29 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too 29 // if there is a valid, non-empty whitelist, we exclude all unknown addresses too
30 if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { 30 if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) {
31 excludeList.push('unknown') 31 excludeList.push('unknown')
32 } 32 }
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index aa2d7a813..02b867a91 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -1,14 +1,15 @@
1import { join } from 'path' 1import { join } from 'path'
2import { ThumbnailType } from '@shared/models' 2import { ThumbnailType } from '@shared/models'
3import { generateImageFilename, generateImageFromVideoFile, processImage } from '../helpers/image-utils' 3import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils'
4import { downloadImage } from '../helpers/requests'
5import { CONFIG } from '../initializers/config' 4import { CONFIG } from '../initializers/config'
6import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' 5import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants'
7import { ThumbnailModel } from '../models/video/thumbnail' 6import { ThumbnailModel } from '../models/video/thumbnail'
8import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' 7import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
9import { MThumbnail } from '../types/models/video/thumbnail' 8import { MThumbnail } from '../types/models/video/thumbnail'
10import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' 9import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
10import { downloadImageFromWorker } from './local-actor'
11import { VideoPathManager } from './video-path-manager' 11import { VideoPathManager } from './video-path-manager'
12import { processImageFromWorker } from './worker/parent-process'
12 13
13type ImageSize = { height?: number, width?: number } 14type ImageSize = { height?: number, width?: number }
14 15
@@ -23,7 +24,10 @@ function updatePlaylistMiniatureFromExisting (options: {
23 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) 24 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size)
24 const type = ThumbnailType.MINIATURE 25 const type = ThumbnailType.MINIATURE
25 26
26 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) 27 const thumbnailCreator = () => {
28 return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal })
29 }
30
27 return updateThumbnailFromFunction({ 31 return updateThumbnailFromFunction({
28 thumbnailCreator, 32 thumbnailCreator,
29 filename, 33 filename,
@@ -49,7 +53,10 @@ function updatePlaylistMiniatureFromUrl (options: {
49 ? null 53 ? null
50 : downloadUrl 54 : downloadUrl
51 55
52 const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height }) 56 const thumbnailCreator = () => {
57 return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })
58 }
59
53 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) 60 return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl })
54} 61}
55 62
@@ -75,7 +82,9 @@ function updateVideoMiniatureFromUrl (options: {
75 : existingThumbnail.filename 82 : existingThumbnail.filename
76 83
77 const thumbnailCreator = () => { 84 const thumbnailCreator = () => {
78 if (thumbnailUrlChanged) return downloadImage(downloadUrl, basePath, filename, { width, height }) 85 if (thumbnailUrlChanged) {
86 return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })
87 }
79 88
80 return Promise.resolve() 89 return Promise.resolve()
81 } 90 }
@@ -94,7 +103,10 @@ function updateVideoMiniatureFromExisting (options: {
94 const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options 103 const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options
95 104
96 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) 105 const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size)
97 const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) 106
107 const thumbnailCreator = () => {
108 return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal })
109 }
98 110
99 return updateThumbnailFromFunction({ 111 return updateThumbnailFromFunction({
100 thumbnailCreator, 112 thumbnailCreator,
@@ -118,8 +130,18 @@ function generateVideoMiniature (options: {
118 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) 130 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
119 131
120 const thumbnailCreator = videoFile.isAudio() 132 const thumbnailCreator = videoFile.isAudio()
121 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) 133 ? () => processImageFromWorker({
122 : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) 134 path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND,
135 destination: outputPath,
136 newSize: { width, height },
137 keepOriginal: true
138 })
139 : () => generateImageFromVideoFile({
140 fromPath: input,
141 folder: basePath,
142 imageName: filename,
143 size: { height, width }
144 })
123 145
124 return updateThumbnailFromFunction({ 146 return updateThumbnailFromFunction({
125 thumbnailCreator, 147 thumbnailCreator,
diff --git a/server/lib/video.ts b/server/lib/video.ts
index a98e45c60..86718abbe 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -1,6 +1,6 @@
1import { UploadFiles } from 'express' 1import { UploadFiles } from 'express'
2import { Transaction } from 'sequelize/types' 2import { Transaction } from 'sequelize/types'
3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' 3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants'
4import { TagModel } from '@server/models/video/tag' 4import { TagModel } from '@server/models/video/tag'
5import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info' 6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@@ -10,6 +10,7 @@ import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingP
10import { CreateJobOptions, JobQueue } from './job-queue/job-queue' 10import { CreateJobOptions, JobQueue } from './job-queue/job-queue'
11import { updateVideoMiniatureFromExisting } from './thumbnail' 11import { updateVideoMiniatureFromExisting } from './thumbnail'
12import { CONFIG } from '@server/initializers/config' 12import { CONFIG } from '@server/initializers/config'
13import memoizee from 'memoizee'
13 14
14function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
15 return { 16 return {
@@ -150,6 +151,24 @@ async function addMoveToObjectStorageJob (options: {
150 151
151// --------------------------------------------------------------------------- 152// ---------------------------------------------------------------------------
152 153
154async function getVideoDuration (videoId: number | string) {
155 const video = await VideoModel.load(videoId)
156
157 const duration = video.isLive
158 ? undefined
159 : video.duration
160
161 return { duration, isLive: video.isLive }
162}
163
164const getCachedVideoDuration = memoizee(getVideoDuration, {
165 promise: true,
166 max: MEMOIZE_LENGTH.VIDEO_DURATION,
167 maxAge: MEMOIZE_TTL.VIDEO_DURATION
168})
169
170// ---------------------------------------------------------------------------
171
153export { 172export {
154 buildLocalVideoFromReq, 173 buildLocalVideoFromReq,
155 buildVideoThumbnailsFromReq, 174 buildVideoThumbnailsFromReq,
@@ -157,5 +176,6 @@ export {
157 addOptimizeOrMergeAudioJob, 176 addOptimizeOrMergeAudioJob,
158 addTranscodingJob, 177 addTranscodingJob,
159 addMoveToObjectStorageJob, 178 addMoveToObjectStorageJob,
160 getTranscodingJobPriority 179 getTranscodingJobPriority,
180 getCachedVideoDuration
161} 181}
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts
index 5158f8f93..cf3fa5882 100644
--- a/server/lib/views/shared/video-viewer-counters.ts
+++ b/server/lib/views/shared/video-viewer-counters.ts
@@ -5,7 +5,7 @@ import { sendView } from '@server/lib/activitypub/send/send-view'
5import { PeerTubeSocket } from '@server/lib/peertube-socket' 5import { PeerTubeSocket } from '@server/lib/peertube-socket'
6import { getServerActor } from '@server/models/application/application' 6import { getServerActor } from '@server/models/application/application'
7import { VideoModel } from '@server/models/video/video' 7import { VideoModel } from '@server/models/video/video'
8import { MVideo } from '@server/types/models' 8import { MVideo, MVideoImmutable } from '@server/types/models'
9import { buildUUID, sha256 } from '@shared/extra-utils' 9import { buildUUID, sha256 } from '@shared/extra-utils'
10 10
11const lTags = loggerTagsFactory('views') 11const lTags = loggerTagsFactory('views')
@@ -33,7 +33,7 @@ export class VideoViewerCounters {
33 // --------------------------------------------------------------------------- 33 // ---------------------------------------------------------------------------
34 34
35 async addLocalViewer (options: { 35 async addLocalViewer (options: {
36 video: MVideo 36 video: MVideoImmutable
37 ip: string 37 ip: string
38 }) { 38 }) {
39 const { video, ip } = options 39 const { video, ip } = options
@@ -86,7 +86,7 @@ export class VideoViewerCounters {
86 // --------------------------------------------------------------------------- 86 // ---------------------------------------------------------------------------
87 87
88 private async addViewerToVideo (options: { 88 private async addViewerToVideo (options: {
89 video: MVideo 89 video: MVideoImmutable
90 viewerId: string 90 viewerId: string
91 viewerExpires?: Date 91 viewerExpires?: Date
92 }) { 92 }) {
@@ -162,7 +162,7 @@ export class VideoViewerCounters {
162 return sha256(this.salt + '-' + ip + '-' + videoUUID) 162 return sha256(this.salt + '-' + ip + '-' + videoUUID)
163 } 163 }
164 164
165 private async federateViewerIfNeeded (video: MVideo, viewer: Viewer) { 165 private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) {
166 // Federate the viewer if it's been a "long" time we did not 166 // Federate the viewer if it's been a "long" time we did not
167 const now = new Date().getTime() 167 const now = new Date().getTime()
168 const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) 168 const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75)
diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts
index a9ba25b47..a56c20559 100644
--- a/server/lib/views/shared/video-viewer-stats.ts
+++ b/server/lib/views/shared/video-viewer-stats.ts
@@ -10,7 +10,7 @@ import { Redis } from '@server/lib/redis'
10import { VideoModel } from '@server/models/video/video' 10import { VideoModel } from '@server/models/video/video'
11import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' 11import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
12import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' 12import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
13import { MVideo } from '@server/types/models' 13import { MVideo, MVideoImmutable } from '@server/types/models'
14import { VideoViewEvent } from '@shared/models' 14import { VideoViewEvent } from '@shared/models'
15 15
16const lTags = loggerTagsFactory('views') 16const lTags = loggerTagsFactory('views')
@@ -41,7 +41,7 @@ export class VideoViewerStats {
41 // --------------------------------------------------------------------------- 41 // ---------------------------------------------------------------------------
42 42
43 async addLocalViewer (options: { 43 async addLocalViewer (options: {
44 video: MVideo 44 video: MVideoImmutable
45 currentTime: number 45 currentTime: number
46 ip: string 46 ip: string
47 viewEvent?: VideoViewEvent 47 viewEvent?: VideoViewEvent
@@ -64,7 +64,7 @@ export class VideoViewerStats {
64 // --------------------------------------------------------------------------- 64 // ---------------------------------------------------------------------------
65 65
66 private async updateLocalViewerStats (options: { 66 private async updateLocalViewerStats (options: {
67 video: MVideo 67 video: MVideoImmutable
68 ip: string 68 ip: string
69 currentTime: number 69 currentTime: number
70 viewEvent?: VideoViewEvent 70 viewEvent?: VideoViewEvent
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts
index 275f7a014..e563287e1 100644
--- a/server/lib/views/shared/video-views.ts
+++ b/server/lib/views/shared/video-views.ts
@@ -1,7 +1,8 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger' 1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { sendView } from '@server/lib/activitypub/send/send-view' 2import { sendView } from '@server/lib/activitypub/send/send-view'
3import { getCachedVideoDuration } from '@server/lib/video'
3import { getServerActor } from '@server/models/application/application' 4import { getServerActor } from '@server/models/application/application'
4import { MVideo } from '@server/types/models' 5import { MVideo, MVideoImmutable } from '@server/types/models'
5import { buildUUID } from '@shared/extra-utils' 6import { buildUUID } from '@shared/extra-utils'
6import { Redis } from '../../redis' 7import { Redis } from '../../redis'
7 8
@@ -10,7 +11,7 @@ const lTags = loggerTagsFactory('views')
10export class VideoViews { 11export class VideoViews {
11 12
12 async addLocalView (options: { 13 async addLocalView (options: {
13 video: MVideo 14 video: MVideoImmutable
14 ip: string 15 ip: string
15 watchTime: number 16 watchTime: number
16 }) { 17 }) {
@@ -18,7 +19,7 @@ export class VideoViews {
18 19
19 logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) 20 logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) })
20 21
21 if (!this.hasEnoughWatchTime(video, watchTime)) return false 22 if (!await this.hasEnoughWatchTime(video, watchTime)) return false
22 23
23 const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) 24 const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid)
24 if (viewExists) return false 25 if (viewExists) return false
@@ -46,7 +47,7 @@ export class VideoViews {
46 47
47 // --------------------------------------------------------------------------- 48 // ---------------------------------------------------------------------------
48 49
49 private async addView (video: MVideo) { 50 private async addView (video: MVideoImmutable) {
50 const promises: Promise<any>[] = [] 51 const promises: Promise<any>[] = []
51 52
52 if (video.isOwned()) { 53 if (video.isOwned()) {
@@ -58,10 +59,12 @@ export class VideoViews {
58 await Promise.all(promises) 59 await Promise.all(promises)
59 } 60 }
60 61
61 private hasEnoughWatchTime (video: MVideo, watchTime: number) { 62 private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) {
62 if (video.isLive || video.duration >= 30) return watchTime >= 30 63 const { duration, isLive } = await getCachedVideoDuration(video.id)
64
65 if (isLive || duration >= 30) return watchTime >= 30
63 66
64 // Check more than 50% of the video is watched 67 // Check more than 50% of the video is watched
65 return video.duration / watchTime < 2 68 return duration / watchTime < 2
66 } 69 }
67} 70}
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts
index ea3b35c6c..86758e8d8 100644
--- a/server/lib/views/video-views-manager.ts
+++ b/server/lib/views/video-views-manager.ts
@@ -1,5 +1,5 @@
1import { logger, loggerTagsFactory } from '@server/helpers/logger' 1import { logger, loggerTagsFactory } from '@server/helpers/logger'
2import { MVideo } from '@server/types/models' 2import { MVideo, MVideoImmutable } from '@server/types/models'
3import { VideoViewEvent } from '@shared/models' 3import { VideoViewEvent } from '@shared/models'
4import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared' 4import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared'
5 5
@@ -41,7 +41,7 @@ export class VideoViewsManager {
41 } 41 }
42 42
43 async processLocalView (options: { 43 async processLocalView (options: {
44 video: MVideo 44 video: MVideoImmutable
45 currentTime: number 45 currentTime: number
46 ip: string | null 46 ip: string | null
47 viewEvent?: VideoViewEvent 47 viewEvent?: VideoViewEvent
diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts
new file mode 100644
index 000000000..4bc7f2620
--- /dev/null
+++ b/server/lib/worker/parent-process.ts
@@ -0,0 +1,40 @@
1import { join } from 'path'
2import Piscina from 'piscina'
3import { processImage } from '@server/helpers/image-utils'
4import { WORKER_THREADS } from '@server/initializers/constants'
5import { downloadImage } from './workers/image-downloader'
6
7let downloadImageWorker: Piscina
8
9function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> {
10 if (!downloadImageWorker) {
11 downloadImageWorker = new Piscina({
12 filename: join(__dirname, 'workers', 'image-downloader.js'),
13 concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,
14 maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS
15 })
16 }
17
18 return downloadImageWorker.run(options)
19}
20
21// ---------------------------------------------------------------------------
22
23let processImageWorker: Piscina
24
25function processImageFromWorker (options: Parameters<typeof processImage>[0]): Promise<ReturnType<typeof processImage>> {
26 if (!processImageWorker) {
27 processImageWorker = new Piscina({
28 filename: join(__dirname, 'workers', 'image-processor.js'),
29 concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY,
30 maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS
31 })
32 }
33
34 return processImageWorker.run(options)
35}
36
37export {
38 downloadImageFromWorker,
39 processImageFromWorker
40}
diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts
new file mode 100644
index 000000000..4b32f723e
--- /dev/null
+++ b/server/lib/worker/workers/image-downloader.ts
@@ -0,0 +1,33 @@
1import { remove } from 'fs-extra'
2import { join } from 'path'
3import { processImage } from '@server/helpers/image-utils'
4import { doRequestAndSaveToFile } from '@server/helpers/requests'
5import { CONFIG } from '@server/initializers/config'
6
7async function downloadImage (options: {
8 url: string
9 destDir: string
10 destName: string
11 size: { width: number, height: number }
12}) {
13 const { url, destDir, destName, size } = options
14
15 const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
16 await doRequestAndSaveToFile(url, tmpPath)
17
18 const destPath = join(destDir, destName)
19
20 try {
21 await processImage({ path: tmpPath, destination: destPath, newSize: size })
22 } catch (err) {
23 await remove(tmpPath)
24
25 throw err
26 }
27}
28
29module.exports = downloadImage
30
31export {
32 downloadImage
33}
diff --git a/server/lib/worker/workers/image-processor.ts b/server/lib/worker/workers/image-processor.ts
new file mode 100644
index 000000000..0ab41a5a0
--- /dev/null
+++ b/server/lib/worker/workers/image-processor.ts
@@ -0,0 +1,7 @@
1import { processImage } from '@server/helpers/image-utils'
2
3module.exports = processImage
4
5export {
6 processImage
7}