diff options
Diffstat (limited to 'server/lib')
21 files changed, 204 insertions, 69 deletions
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 3e7931bb2..76ed37aae 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -124,7 +124,7 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc | |||
124 | return | 124 | return |
125 | } | 125 | } |
126 | 126 | ||
127 | // Try to not forward unwanted commments on our videos | 127 | // Try to not forward unwanted comments on our videos |
128 | if (video.isOwned()) { | 128 | if (video.isOwned()) { |
129 | if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { | 129 | if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { |
130 | logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) | 130 | logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) |
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts index 688bcbb53..07252fea2 100644 --- a/server/lib/activitypub/videos/shared/creator.ts +++ b/server/lib/activitypub/videos/shared/creator.ts | |||
@@ -24,7 +24,7 @@ export class APVideoCreator extends APVideoAbstractBuilder { | |||
24 | const channel = channelActor.VideoChannel | 24 | const channel = channelActor.VideoChannel |
25 | 25 | ||
26 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) | 26 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) |
27 | const video = VideoModel.build(videoData) as MVideoThumbnail | 27 | const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail |
28 | 28 | ||
29 | const promiseThumbnail = this.tryToGenerateThumbnail(video) | 29 | const promiseThumbnail = this.tryToGenerateThumbnail(video) |
30 | 30 | ||
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index f02b9cba6..86699c5b8 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -210,8 +210,6 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi | |||
210 | 210 | ||
211 | updatedAt: new Date(videoObject.updated), | 211 | updatedAt: new Date(videoObject.updated), |
212 | views: videoObject.views, | 212 | views: videoObject.views, |
213 | likes: 0, | ||
214 | dislikes: 0, | ||
215 | remote: true, | 213 | remote: true, |
216 | privacy | 214 | privacy |
217 | } | 215 | } |
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 337364ac9..1e8d03023 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -30,6 +30,7 @@ import { MAccountActor, MChannelActor } from '../types/models' | |||
30 | import { getActivityStreamDuration } from './activitypub/activity' | 30 | import { getActivityStreamDuration } from './activitypub/activity' |
31 | import { getBiggestActorImage } from './actor-image' | 31 | import { getBiggestActorImage } from './actor-image' |
32 | import { ServerConfigManager } from './server-config-manager' | 32 | import { ServerConfigManager } from './server-config-manager' |
33 | import { isTestInstance } from '@server/helpers/core-utils' | ||
33 | 34 | ||
34 | type Tags = { | 35 | type Tags = { |
35 | ogType: string | 36 | ogType: string |
@@ -232,7 +233,10 @@ class ClientHtml { | |||
232 | static async getEmbedHTML () { | 233 | static async getEmbedHTML () { |
233 | const path = ClientHtml.getEmbedPath() | 234 | const path = ClientHtml.getEmbedPath() |
234 | 235 | ||
235 | if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] | 236 | // Disable HTML cache in dev mode because webpack can regenerate JS files |
237 | if (!isTestInstance() && ClientHtml.htmlCache[path]) { | ||
238 | return ClientHtml.htmlCache[path] | ||
239 | } | ||
236 | 240 | ||
237 | const buffer = await readFile(path) | 241 | const buffer = await readFile(path) |
238 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() | 242 | const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() |
diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index aebca04fe..edc99057c 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts | |||
@@ -179,7 +179,7 @@ class Emailer { | |||
179 | } | 179 | } |
180 | } | 180 | } |
181 | 181 | ||
182 | // overriden/new variables given for a specific template in the payload | 182 | // overridden/new variables given for a specific template in the payload |
183 | const sendOptions = merge(baseOptions, options) | 183 | const sendOptions = merge(baseOptions, options) |
184 | 184 | ||
185 | await email.send(sendOptions) | 185 | await email.send(sendOptions) |
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index f480b32cd..49064052c 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -48,15 +48,24 @@ export async function processMoveToObjectStorage (job: Job) { | |||
48 | await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) | 48 | await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) |
49 | } | 49 | } |
50 | } catch (err) { | 50 | } catch (err) { |
51 | logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags }) | 51 | await onMoveToObjectStorageFailure(job, err) |
52 | |||
53 | await moveToFailedMoveToObjectStorageState(video) | ||
54 | await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') | ||
55 | } | 52 | } |
56 | 53 | ||
57 | return payload.videoUUID | 54 | return payload.videoUUID |
58 | } | 55 | } |
59 | 56 | ||
57 | export async function onMoveToObjectStorageFailure (job: Job, err: any) { | ||
58 | const payload = job.data as MoveObjectStoragePayload | ||
59 | |||
60 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | ||
61 | if (!video) return | ||
62 | |||
63 | logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) }) | ||
64 | |||
65 | await moveToFailedMoveToObjectStorageState(video) | ||
66 | await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') | ||
67 | } | ||
68 | |||
60 | // --------------------------------------------------------------------------- | 69 | // --------------------------------------------------------------------------- |
61 | 70 | ||
62 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | 71 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index f339e9135..ce24763f1 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -33,7 +33,7 @@ import { refreshAPObject } from './handlers/activitypub-refresher' | |||
33 | import { processActorKeys } from './handlers/actor-keys' | 33 | import { processActorKeys } from './handlers/actor-keys' |
34 | import { processEmail } from './handlers/email' | 34 | import { processEmail } from './handlers/email' |
35 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' | 35 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' |
36 | import { processMoveToObjectStorage } from './handlers/move-to-object-storage' | 36 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' |
37 | import { processVideoFileImport } from './handlers/video-file-import' | 37 | import { processVideoFileImport } from './handlers/video-file-import' |
38 | import { processVideoImport } from './handlers/video-import' | 38 | import { processVideoImport } from './handlers/video-import' |
39 | import { processVideoLiveEnding } from './handlers/video-live-ending' | 39 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
@@ -88,6 +88,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { | |||
88 | 'video-studio-edition': processVideoStudioEdition | 88 | 'video-studio-edition': processVideoStudioEdition |
89 | } | 89 | } |
90 | 90 | ||
91 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { | ||
92 | 'move-to-object-storage': onMoveToObjectStorageFailure | ||
93 | } | ||
94 | |||
91 | const jobTypes: JobType[] = [ | 95 | const jobTypes: JobType[] = [ |
92 | 'activitypub-follow', | 96 | 'activitypub-follow', |
93 | 'activitypub-http-broadcast', | 97 | 'activitypub-http-broadcast', |
@@ -162,6 +166,11 @@ class JobQueue { | |||
162 | : 'error' | 166 | : 'error' |
163 | 167 | ||
164 | logger.log(logLevel, 'Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | 168 | logger.log(logLevel, 'Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) |
169 | |||
170 | if (errorHandlers[job.name]) { | ||
171 | errorHandlers[job.name](job, err) | ||
172 | .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) | ||
173 | } | ||
165 | }) | 174 | }) |
166 | 175 | ||
167 | queue.on('error', err => { | 176 | queue.on('error', err => { |
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts index 01046d017..1d9be76e2 100644 --- a/server/lib/local-actor.ts +++ b/server/lib/local-actor.ts | |||
@@ -1,4 +1,3 @@ | |||
1 | import { queue } from 'async' | ||
2 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
3 | import LRUCache from 'lru-cache' | 2 | import LRUCache from 'lru-cache' |
4 | import { join } from 'path' | 3 | import { join } from 'path' |
@@ -7,14 +6,13 @@ import { getLowercaseExtension } from '@shared/core-utils' | |||
7 | import { buildUUID } from '@shared/extra-utils' | 6 | import { buildUUID } from '@shared/extra-utils' |
8 | import { ActivityPubActorType, ActorImageType } from '@shared/models' | 7 | import { ActivityPubActorType, ActorImageType } from '@shared/models' |
9 | import { retryTransactionWrapper } from '../helpers/database-utils' | 8 | import { retryTransactionWrapper } from '../helpers/database-utils' |
10 | import { processImage } from '../helpers/image-utils' | ||
11 | import { downloadImage } from '../helpers/requests' | ||
12 | import { CONFIG } from '../initializers/config' | 9 | import { CONFIG } from '../initializers/config' |
13 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY, WEBSERVER } from '../initializers/constants' | 10 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants' |
14 | import { sequelizeTypescript } from '../initializers/database' | 11 | import { sequelizeTypescript } from '../initializers/database' |
15 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' | 12 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' |
16 | import { deleteActorImages, updateActorImages } from './activitypub/actors' | 13 | import { deleteActorImages, updateActorImages } from './activitypub/actors' |
17 | import { sendUpdateActor } from './activitypub/send' | 14 | import { sendUpdateActor } from './activitypub/send' |
15 | import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' | ||
18 | 16 | ||
19 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { | 17 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { |
20 | return new ActorModel({ | 18 | return new ActorModel({ |
@@ -43,7 +41,7 @@ async function updateLocalActorImageFiles ( | |||
43 | 41 | ||
44 | const imageName = buildUUID() + extension | 42 | const imageName = buildUUID() + extension |
45 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) | 43 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) |
46 | await processImage(imagePhysicalFile.path, destination, imageSize, true) | 44 | await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) |
47 | 45 | ||
48 | return { | 46 | return { |
49 | imageName, | 47 | imageName, |
@@ -87,27 +85,22 @@ async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MC | |||
87 | }) | 85 | }) |
88 | } | 86 | } |
89 | 87 | ||
90 | type DownloadImageQueueTask = { | 88 | // --------------------------------------------------------------------------- |
89 | |||
90 | function downloadActorImageFromWorker (options: { | ||
91 | fileUrl: string | 91 | fileUrl: string |
92 | filename: string | 92 | filename: string |
93 | type: ActorImageType | 93 | type: ActorImageType |
94 | size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0] | 94 | size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0] |
95 | } | 95 | }) { |
96 | 96 | const downloaderOptions = { | |
97 | const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => { | 97 | url: options.fileUrl, |
98 | downloadImage(task.fileUrl, CONFIG.STORAGE.ACTOR_IMAGES, task.filename, task.size) | 98 | destDir: CONFIG.STORAGE.ACTOR_IMAGES, |
99 | .then(() => cb()) | 99 | destName: options.filename, |
100 | .catch(err => cb(err)) | 100 | size: options.size |
101 | }, QUEUE_CONCURRENCY.ACTOR_PROCESS_IMAGE) | 101 | } |
102 | |||
103 | function pushActorImageProcessInQueue (task: DownloadImageQueueTask) { | ||
104 | return new Promise<void>((res, rej) => { | ||
105 | downloadImageQueue.push(task, err => { | ||
106 | if (err) return rej(err) | ||
107 | 102 | ||
108 | return res() | 103 | return downloadImageFromWorker(downloaderOptions) |
109 | }) | ||
110 | }) | ||
111 | } | 104 | } |
112 | 105 | ||
113 | // Unsafe so could returns paths that does not exist anymore | 106 | // Unsafe so could returns paths that does not exist anymore |
@@ -116,7 +109,8 @@ const actorImagePathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE. | |||
116 | export { | 109 | export { |
117 | actorImagePathUnsafeCache, | 110 | actorImagePathUnsafeCache, |
118 | updateLocalActorImageFiles, | 111 | updateLocalActorImageFiles, |
112 | downloadActorImageFromWorker, | ||
119 | deleteLocalActorImageFile, | 113 | deleteLocalActorImageFile, |
120 | pushActorImageProcessInQueue, | 114 | downloadImageFromWorker, |
121 | buildActorInstance | 115 | buildActorInstance |
122 | } | 116 | } |
diff --git a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts index daefa25bd..a7292de69 100644 --- a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts +++ b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts | |||
@@ -5,7 +5,7 @@ import { MAbuseFull, MAbuseMessage, MAccountDefault, MUserWithNotificationSettin | |||
5 | import { UserNotificationType } from '@shared/models' | 5 | import { UserNotificationType } from '@shared/models' |
6 | import { AbstractNotification } from '../common/abstract-notification' | 6 | import { AbstractNotification } from '../common/abstract-notification' |
7 | 7 | ||
8 | export type NewAbuseMessagePayload = { | 8 | type NewAbuseMessagePayload = { |
9 | abuse: MAbuseFull | 9 | abuse: MAbuseFull |
10 | message: MAbuseMessage | 10 | message: MAbuseMessage |
11 | } | 11 | } |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index d052de786..d6d053d2f 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { createClient, RedisClientOptions, RedisModules, RedisScripts } from 'redis' | 1 | import { createClient, RedisClientOptions, RedisModules } from 'redis' |
2 | import { exists } from '@server/helpers/custom-validators/misc' | 2 | import { exists } from '@server/helpers/custom-validators/misc' |
3 | import { sha256 } from '@shared/extra-utils' | 3 | import { sha256 } from '@shared/extra-utils' |
4 | import { logger } from '../helpers/logger' | 4 | import { logger } from '../helpers/logger' |
@@ -16,16 +16,12 @@ import { | |||
16 | WEBSERVER | 16 | WEBSERVER |
17 | } from '../initializers/constants' | 17 | } from '../initializers/constants' |
18 | 18 | ||
19 | // Only used for typings | ||
20 | // TODO: remove when https://github.com/microsoft/TypeScript/issues/37181 is fixed | ||
21 | const redisClientWrapperForType = () => createClient<{}, RedisScripts>() | ||
22 | |||
23 | class Redis { | 19 | class Redis { |
24 | 20 | ||
25 | private static instance: Redis | 21 | private static instance: Redis |
26 | private initialized = false | 22 | private initialized = false |
27 | private connected = false | 23 | private connected = false |
28 | private client: ReturnType<typeof redisClientWrapperForType> | 24 | private client: ReturnType<typeof createClient> |
29 | private prefix: string | 25 | private prefix: string |
30 | 26 | ||
31 | private constructor () { | 27 | private constructor () { |
@@ -308,7 +304,7 @@ class Redis { | |||
308 | return this.deleteKey('resumable-upload-' + uploadId) | 304 | return this.deleteKey('resumable-upload-' + uploadId) |
309 | } | 305 | } |
310 | 306 | ||
311 | /* ************ AP ressource unavailability ************ */ | 307 | /* ************ AP resource unavailability ************ */ |
312 | 308 | ||
313 | async addAPUnavailability (url: string) { | 309 | async addAPUnavailability (url: string) { |
314 | const key = this.generateAPUnavailabilityKey(url) | 310 | const key = this.generateAPUnavailabilityKey(url) |
diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts index 9dda6d76c..b06f5a9b5 100644 --- a/server/lib/schedulers/geo-ip-update-scheduler.ts +++ b/server/lib/schedulers/geo-ip-update-scheduler.ts | |||
@@ -6,7 +6,7 @@ export class GeoIPUpdateScheduler extends AbstractScheduler { | |||
6 | 6 | ||
7 | private static instance: AbstractScheduler | 7 | private static instance: AbstractScheduler |
8 | 8 | ||
9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE | 9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE |
10 | 10 | ||
11 | private constructor () { | 11 | private constructor () { |
12 | super() | 12 | super() |
diff --git a/server/lib/signup.ts b/server/lib/signup.ts index 3c1397a12..f094531eb 100644 --- a/server/lib/signup.ts +++ b/server/lib/signup.ts | |||
@@ -26,7 +26,7 @@ function isSignupAllowedForCurrentIP (ip: string) { | |||
26 | const excludeList = [ 'blacklist' ] | 26 | const excludeList = [ 'blacklist' ] |
27 | let matched = '' | 27 | let matched = '' |
28 | 28 | ||
29 | // if there is a valid, non-empty whitelist, we exclude all unknown adresses too | 29 | // if there is a valid, non-empty whitelist, we exclude all unknown addresses too |
30 | if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { | 30 | if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { |
31 | excludeList.push('unknown') | 31 | excludeList.push('unknown') |
32 | } | 32 | } |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index aa2d7a813..02b867a91 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -1,14 +1,15 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { ThumbnailType } from '@shared/models' | 2 | import { ThumbnailType } from '@shared/models' |
3 | import { generateImageFilename, generateImageFromVideoFile, processImage } from '../helpers/image-utils' | 3 | import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils' |
4 | import { downloadImage } from '../helpers/requests' | ||
5 | import { CONFIG } from '../initializers/config' | 4 | import { CONFIG } from '../initializers/config' |
6 | import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' | 5 | import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' |
7 | import { ThumbnailModel } from '../models/video/thumbnail' | 6 | import { ThumbnailModel } from '../models/video/thumbnail' |
8 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' | 7 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' |
9 | import { MThumbnail } from '../types/models/video/thumbnail' | 8 | import { MThumbnail } from '../types/models/video/thumbnail' |
10 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | 9 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' |
10 | import { downloadImageFromWorker } from './local-actor' | ||
11 | import { VideoPathManager } from './video-path-manager' | 11 | import { VideoPathManager } from './video-path-manager' |
12 | import { processImageFromWorker } from './worker/parent-process' | ||
12 | 13 | ||
13 | type ImageSize = { height?: number, width?: number } | 14 | type ImageSize = { height?: number, width?: number } |
14 | 15 | ||
@@ -23,7 +24,10 @@ function updatePlaylistMiniatureFromExisting (options: { | |||
23 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) | 24 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) |
24 | const type = ThumbnailType.MINIATURE | 25 | const type = ThumbnailType.MINIATURE |
25 | 26 | ||
26 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | 27 | const thumbnailCreator = () => { |
28 | return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) | ||
29 | } | ||
30 | |||
27 | return updateThumbnailFromFunction({ | 31 | return updateThumbnailFromFunction({ |
28 | thumbnailCreator, | 32 | thumbnailCreator, |
29 | filename, | 33 | filename, |
@@ -49,7 +53,10 @@ function updatePlaylistMiniatureFromUrl (options: { | |||
49 | ? null | 53 | ? null |
50 | : downloadUrl | 54 | : downloadUrl |
51 | 55 | ||
52 | const thumbnailCreator = () => downloadImage(downloadUrl, basePath, filename, { width, height }) | 56 | const thumbnailCreator = () => { |
57 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | ||
58 | } | ||
59 | |||
53 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | 60 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) |
54 | } | 61 | } |
55 | 62 | ||
@@ -75,7 +82,9 @@ function updateVideoMiniatureFromUrl (options: { | |||
75 | : existingThumbnail.filename | 82 | : existingThumbnail.filename |
76 | 83 | ||
77 | const thumbnailCreator = () => { | 84 | const thumbnailCreator = () => { |
78 | if (thumbnailUrlChanged) return downloadImage(downloadUrl, basePath, filename, { width, height }) | 85 | if (thumbnailUrlChanged) { |
86 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | ||
87 | } | ||
79 | 88 | ||
80 | return Promise.resolve() | 89 | return Promise.resolve() |
81 | } | 90 | } |
@@ -94,7 +103,10 @@ function updateVideoMiniatureFromExisting (options: { | |||
94 | const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options | 103 | const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options |
95 | 104 | ||
96 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | 105 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) |
97 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | 106 | |
107 | const thumbnailCreator = () => { | ||
108 | return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) | ||
109 | } | ||
98 | 110 | ||
99 | return updateThumbnailFromFunction({ | 111 | return updateThumbnailFromFunction({ |
100 | thumbnailCreator, | 112 | thumbnailCreator, |
@@ -118,8 +130,18 @@ function generateVideoMiniature (options: { | |||
118 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | 130 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) |
119 | 131 | ||
120 | const thumbnailCreator = videoFile.isAudio() | 132 | const thumbnailCreator = videoFile.isAudio() |
121 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) | 133 | ? () => processImageFromWorker({ |
122 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | 134 | path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, |
135 | destination: outputPath, | ||
136 | newSize: { width, height }, | ||
137 | keepOriginal: true | ||
138 | }) | ||
139 | : () => generateImageFromVideoFile({ | ||
140 | fromPath: input, | ||
141 | folder: basePath, | ||
142 | imageName: filename, | ||
143 | size: { height, width } | ||
144 | }) | ||
123 | 145 | ||
124 | return updateThumbnailFromFunction({ | 146 | return updateThumbnailFromFunction({ |
125 | thumbnailCreator, | 147 | thumbnailCreator, |
diff --git a/server/lib/video.ts b/server/lib/video.ts index a98e45c60..86718abbe 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { UploadFiles } from 'express' | 1 | import { UploadFiles } from 'express' |
2 | import { Transaction } from 'sequelize/types' | 2 | import { Transaction } from 'sequelize/types' |
3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' | 3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' |
4 | import { TagModel } from '@server/models/video/tag' | 4 | import { TagModel } from '@server/models/video/tag' |
5 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
@@ -10,6 +10,7 @@ import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingP | |||
10 | import { CreateJobOptions, JobQueue } from './job-queue/job-queue' | 10 | import { CreateJobOptions, JobQueue } from './job-queue/job-queue' |
11 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 11 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
12 | import { CONFIG } from '@server/initializers/config' | 12 | import { CONFIG } from '@server/initializers/config' |
13 | import memoizee from 'memoizee' | ||
13 | 14 | ||
14 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
15 | return { | 16 | return { |
@@ -150,6 +151,24 @@ async function addMoveToObjectStorageJob (options: { | |||
150 | 151 | ||
151 | // --------------------------------------------------------------------------- | 152 | // --------------------------------------------------------------------------- |
152 | 153 | ||
154 | async function getVideoDuration (videoId: number | string) { | ||
155 | const video = await VideoModel.load(videoId) | ||
156 | |||
157 | const duration = video.isLive | ||
158 | ? undefined | ||
159 | : video.duration | ||
160 | |||
161 | return { duration, isLive: video.isLive } | ||
162 | } | ||
163 | |||
164 | const getCachedVideoDuration = memoizee(getVideoDuration, { | ||
165 | promise: true, | ||
166 | max: MEMOIZE_LENGTH.VIDEO_DURATION, | ||
167 | maxAge: MEMOIZE_TTL.VIDEO_DURATION | ||
168 | }) | ||
169 | |||
170 | // --------------------------------------------------------------------------- | ||
171 | |||
153 | export { | 172 | export { |
154 | buildLocalVideoFromReq, | 173 | buildLocalVideoFromReq, |
155 | buildVideoThumbnailsFromReq, | 174 | buildVideoThumbnailsFromReq, |
@@ -157,5 +176,6 @@ export { | |||
157 | addOptimizeOrMergeAudioJob, | 176 | addOptimizeOrMergeAudioJob, |
158 | addTranscodingJob, | 177 | addTranscodingJob, |
159 | addMoveToObjectStorageJob, | 178 | addMoveToObjectStorageJob, |
160 | getTranscodingJobPriority | 179 | getTranscodingJobPriority, |
180 | getCachedVideoDuration | ||
161 | } | 181 | } |
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts index 5158f8f93..cf3fa5882 100644 --- a/server/lib/views/shared/video-viewer-counters.ts +++ b/server/lib/views/shared/video-viewer-counters.ts | |||
@@ -5,7 +5,7 @@ import { sendView } from '@server/lib/activitypub/send/send-view' | |||
5 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | 5 | import { PeerTubeSocket } from '@server/lib/peertube-socket' |
6 | import { getServerActor } from '@server/models/application/application' | 6 | import { getServerActor } from '@server/models/application/application' |
7 | import { VideoModel } from '@server/models/video/video' | 7 | import { VideoModel } from '@server/models/video/video' |
8 | import { MVideo } from '@server/types/models' | 8 | import { MVideo, MVideoImmutable } from '@server/types/models' |
9 | import { buildUUID, sha256 } from '@shared/extra-utils' | 9 | import { buildUUID, sha256 } from '@shared/extra-utils' |
10 | 10 | ||
11 | const lTags = loggerTagsFactory('views') | 11 | const lTags = loggerTagsFactory('views') |
@@ -33,7 +33,7 @@ export class VideoViewerCounters { | |||
33 | // --------------------------------------------------------------------------- | 33 | // --------------------------------------------------------------------------- |
34 | 34 | ||
35 | async addLocalViewer (options: { | 35 | async addLocalViewer (options: { |
36 | video: MVideo | 36 | video: MVideoImmutable |
37 | ip: string | 37 | ip: string |
38 | }) { | 38 | }) { |
39 | const { video, ip } = options | 39 | const { video, ip } = options |
@@ -86,7 +86,7 @@ export class VideoViewerCounters { | |||
86 | // --------------------------------------------------------------------------- | 86 | // --------------------------------------------------------------------------- |
87 | 87 | ||
88 | private async addViewerToVideo (options: { | 88 | private async addViewerToVideo (options: { |
89 | video: MVideo | 89 | video: MVideoImmutable |
90 | viewerId: string | 90 | viewerId: string |
91 | viewerExpires?: Date | 91 | viewerExpires?: Date |
92 | }) { | 92 | }) { |
@@ -162,7 +162,7 @@ export class VideoViewerCounters { | |||
162 | return sha256(this.salt + '-' + ip + '-' + videoUUID) | 162 | return sha256(this.salt + '-' + ip + '-' + videoUUID) |
163 | } | 163 | } |
164 | 164 | ||
165 | private async federateViewerIfNeeded (video: MVideo, viewer: Viewer) { | 165 | private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { |
166 | // Federate the viewer if it's been a "long" time we did not | 166 | // Federate the viewer if it's been a "long" time we did not |
167 | const now = new Date().getTime() | 167 | const now = new Date().getTime() |
168 | const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) | 168 | const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) |
diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts index a9ba25b47..a56c20559 100644 --- a/server/lib/views/shared/video-viewer-stats.ts +++ b/server/lib/views/shared/video-viewer-stats.ts | |||
@@ -10,7 +10,7 @@ import { Redis } from '@server/lib/redis' | |||
10 | import { VideoModel } from '@server/models/video/video' | 10 | import { VideoModel } from '@server/models/video/video' |
11 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | 11 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' |
12 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | 12 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' |
13 | import { MVideo } from '@server/types/models' | 13 | import { MVideo, MVideoImmutable } from '@server/types/models' |
14 | import { VideoViewEvent } from '@shared/models' | 14 | import { VideoViewEvent } from '@shared/models' |
15 | 15 | ||
16 | const lTags = loggerTagsFactory('views') | 16 | const lTags = loggerTagsFactory('views') |
@@ -41,7 +41,7 @@ export class VideoViewerStats { | |||
41 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
42 | 42 | ||
43 | async addLocalViewer (options: { | 43 | async addLocalViewer (options: { |
44 | video: MVideo | 44 | video: MVideoImmutable |
45 | currentTime: number | 45 | currentTime: number |
46 | ip: string | 46 | ip: string |
47 | viewEvent?: VideoViewEvent | 47 | viewEvent?: VideoViewEvent |
@@ -64,7 +64,7 @@ export class VideoViewerStats { | |||
64 | // --------------------------------------------------------------------------- | 64 | // --------------------------------------------------------------------------- |
65 | 65 | ||
66 | private async updateLocalViewerStats (options: { | 66 | private async updateLocalViewerStats (options: { |
67 | video: MVideo | 67 | video: MVideoImmutable |
68 | ip: string | 68 | ip: string |
69 | currentTime: number | 69 | currentTime: number |
70 | viewEvent?: VideoViewEvent | 70 | viewEvent?: VideoViewEvent |
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts index 275f7a014..e563287e1 100644 --- a/server/lib/views/shared/video-views.ts +++ b/server/lib/views/shared/video-views.ts | |||
@@ -1,7 +1,8 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { sendView } from '@server/lib/activitypub/send/send-view' | 2 | import { sendView } from '@server/lib/activitypub/send/send-view' |
3 | import { getCachedVideoDuration } from '@server/lib/video' | ||
3 | import { getServerActor } from '@server/models/application/application' | 4 | import { getServerActor } from '@server/models/application/application' |
4 | import { MVideo } from '@server/types/models' | 5 | import { MVideo, MVideoImmutable } from '@server/types/models' |
5 | import { buildUUID } from '@shared/extra-utils' | 6 | import { buildUUID } from '@shared/extra-utils' |
6 | import { Redis } from '../../redis' | 7 | import { Redis } from '../../redis' |
7 | 8 | ||
@@ -10,7 +11,7 @@ const lTags = loggerTagsFactory('views') | |||
10 | export class VideoViews { | 11 | export class VideoViews { |
11 | 12 | ||
12 | async addLocalView (options: { | 13 | async addLocalView (options: { |
13 | video: MVideo | 14 | video: MVideoImmutable |
14 | ip: string | 15 | ip: string |
15 | watchTime: number | 16 | watchTime: number |
16 | }) { | 17 | }) { |
@@ -18,7 +19,7 @@ export class VideoViews { | |||
18 | 19 | ||
19 | logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) | 20 | logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) |
20 | 21 | ||
21 | if (!this.hasEnoughWatchTime(video, watchTime)) return false | 22 | if (!await this.hasEnoughWatchTime(video, watchTime)) return false |
22 | 23 | ||
23 | const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) | 24 | const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) |
24 | if (viewExists) return false | 25 | if (viewExists) return false |
@@ -46,7 +47,7 @@ export class VideoViews { | |||
46 | 47 | ||
47 | // --------------------------------------------------------------------------- | 48 | // --------------------------------------------------------------------------- |
48 | 49 | ||
49 | private async addView (video: MVideo) { | 50 | private async addView (video: MVideoImmutable) { |
50 | const promises: Promise<any>[] = [] | 51 | const promises: Promise<any>[] = [] |
51 | 52 | ||
52 | if (video.isOwned()) { | 53 | if (video.isOwned()) { |
@@ -58,10 +59,12 @@ export class VideoViews { | |||
58 | await Promise.all(promises) | 59 | await Promise.all(promises) |
59 | } | 60 | } |
60 | 61 | ||
61 | private hasEnoughWatchTime (video: MVideo, watchTime: number) { | 62 | private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { |
62 | if (video.isLive || video.duration >= 30) return watchTime >= 30 | 63 | const { duration, isLive } = await getCachedVideoDuration(video.id) |
64 | |||
65 | if (isLive || duration >= 30) return watchTime >= 30 | ||
63 | 66 | ||
64 | // Check more than 50% of the video is watched | 67 | // Check more than 50% of the video is watched |
65 | return video.duration / watchTime < 2 | 68 | return duration / watchTime < 2 |
66 | } | 69 | } |
67 | } | 70 | } |
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts index ea3b35c6c..86758e8d8 100644 --- a/server/lib/views/video-views-manager.ts +++ b/server/lib/views/video-views-manager.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { MVideo } from '@server/types/models' | 2 | import { MVideo, MVideoImmutable } from '@server/types/models' |
3 | import { VideoViewEvent } from '@shared/models' | 3 | import { VideoViewEvent } from '@shared/models' |
4 | import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared' | 4 | import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared' |
5 | 5 | ||
@@ -41,7 +41,7 @@ export class VideoViewsManager { | |||
41 | } | 41 | } |
42 | 42 | ||
43 | async processLocalView (options: { | 43 | async processLocalView (options: { |
44 | video: MVideo | 44 | video: MVideoImmutable |
45 | currentTime: number | 45 | currentTime: number |
46 | ip: string | null | 46 | ip: string | null |
47 | viewEvent?: VideoViewEvent | 47 | viewEvent?: VideoViewEvent |
diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts new file mode 100644 index 000000000..4bc7f2620 --- /dev/null +++ b/server/lib/worker/parent-process.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { join } from 'path' | ||
2 | import Piscina from 'piscina' | ||
3 | import { processImage } from '@server/helpers/image-utils' | ||
4 | import { WORKER_THREADS } from '@server/initializers/constants' | ||
5 | import { downloadImage } from './workers/image-downloader' | ||
6 | |||
7 | let downloadImageWorker: Piscina | ||
8 | |||
9 | function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> { | ||
10 | if (!downloadImageWorker) { | ||
11 | downloadImageWorker = new Piscina({ | ||
12 | filename: join(__dirname, 'workers', 'image-downloader.js'), | ||
13 | concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY, | ||
14 | maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS | ||
15 | }) | ||
16 | } | ||
17 | |||
18 | return downloadImageWorker.run(options) | ||
19 | } | ||
20 | |||
21 | // --------------------------------------------------------------------------- | ||
22 | |||
23 | let processImageWorker: Piscina | ||
24 | |||
25 | function processImageFromWorker (options: Parameters<typeof processImage>[0]): Promise<ReturnType<typeof processImage>> { | ||
26 | if (!processImageWorker) { | ||
27 | processImageWorker = new Piscina({ | ||
28 | filename: join(__dirname, 'workers', 'image-processor.js'), | ||
29 | concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY, | ||
30 | maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS | ||
31 | }) | ||
32 | } | ||
33 | |||
34 | return processImageWorker.run(options) | ||
35 | } | ||
36 | |||
37 | export { | ||
38 | downloadImageFromWorker, | ||
39 | processImageFromWorker | ||
40 | } | ||
diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts new file mode 100644 index 000000000..4b32f723e --- /dev/null +++ b/server/lib/worker/workers/image-downloader.ts | |||
@@ -0,0 +1,33 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { join } from 'path' | ||
3 | import { processImage } from '@server/helpers/image-utils' | ||
4 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | |||
7 | async function downloadImage (options: { | ||
8 | url: string | ||
9 | destDir: string | ||
10 | destName: string | ||
11 | size: { width: number, height: number } | ||
12 | }) { | ||
13 | const { url, destDir, destName, size } = options | ||
14 | |||
15 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) | ||
16 | await doRequestAndSaveToFile(url, tmpPath) | ||
17 | |||
18 | const destPath = join(destDir, destName) | ||
19 | |||
20 | try { | ||
21 | await processImage({ path: tmpPath, destination: destPath, newSize: size }) | ||
22 | } catch (err) { | ||
23 | await remove(tmpPath) | ||
24 | |||
25 | throw err | ||
26 | } | ||
27 | } | ||
28 | |||
29 | module.exports = downloadImage | ||
30 | |||
31 | export { | ||
32 | downloadImage | ||
33 | } | ||
diff --git a/server/lib/worker/workers/image-processor.ts b/server/lib/worker/workers/image-processor.ts new file mode 100644 index 000000000..0ab41a5a0 --- /dev/null +++ b/server/lib/worker/workers/image-processor.ts | |||
@@ -0,0 +1,7 @@ | |||
1 | import { processImage } from '@server/helpers/image-utils' | ||
2 | |||
3 | module.exports = processImage | ||
4 | |||
5 | export { | ||
6 | processImage | ||
7 | } | ||