diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/videos/upload.ts | 9 | ||||
-rw-r--r-- | server/initializers/constants.ts | 2 | ||||
-rw-r--r-- | server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts | 35 | ||||
-rw-r--r-- | server/lib/uploadx.ts | 10 |
4 files changed, 21 insertions, 35 deletions
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 6773b500f..c827f6bf0 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts | |||
@@ -8,6 +8,7 @@ import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | |||
8 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 8 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | 9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' |
10 | import { Redis } from '@server/lib/redis' | 10 | import { Redis } from '@server/lib/redis' |
11 | import { uploadx } from '@server/lib/uploadx' | ||
11 | import { | 12 | import { |
12 | addMoveToObjectStorageJob, | 13 | addMoveToObjectStorageJob, |
13 | addOptimizeOrMergeAudioJob, | 14 | addOptimizeOrMergeAudioJob, |
@@ -19,7 +20,6 @@ import { VideoPathManager } from '@server/lib/video-path-manager' | |||
19 | import { buildNextVideoState } from '@server/lib/video-state' | 20 | import { buildNextVideoState } from '@server/lib/video-state' |
20 | import { openapiOperationDoc } from '@server/middlewares/doc' | 21 | import { openapiOperationDoc } from '@server/middlewares/doc' |
21 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 22 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
22 | import { Uploadx } from '@uploadx/core' | ||
23 | import { VideoCreate, VideoState } from '../../../../shared' | 23 | import { VideoCreate, VideoState } from '../../../../shared' |
24 | import { HttpStatusCode } from '../../../../shared/models' | 24 | import { HttpStatusCode } from '../../../../shared/models' |
25 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' | 25 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' |
@@ -41,8 +41,8 @@ import { | |||
41 | authenticate, | 41 | authenticate, |
42 | videosAddLegacyValidator, | 42 | videosAddLegacyValidator, |
43 | videosAddResumableInitValidator, | 43 | videosAddResumableInitValidator, |
44 | videosResumableUploadIdValidator, | 44 | videosAddResumableValidator, |
45 | videosAddResumableValidator | 45 | videosResumableUploadIdValidator |
46 | } from '../../../middlewares' | 46 | } from '../../../middlewares' |
47 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' | 47 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' |
48 | import { VideoModel } from '../../../models/video/video' | 48 | import { VideoModel } from '../../../models/video/video' |
@@ -52,9 +52,6 @@ const lTags = loggerTagsFactory('api', 'video') | |||
52 | const auditLogger = auditLoggerFactory('videos') | 52 | const auditLogger = auditLoggerFactory('videos') |
53 | const uploadRouter = express.Router() | 53 | const uploadRouter = express.Router() |
54 | 54 | ||
55 | const uploadx = new Uploadx({ directory: getResumableUploadPath() }) | ||
56 | uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id | ||
57 | |||
58 | const reqVideoFileAdd = createReqFiles( | 55 | const reqVideoFileAdd = createReqFiles( |
59 | [ 'videofile', 'thumbnailfile', 'previewfile' ], | 56 | [ 'videofile', 'thumbnailfile', 'previewfile' ], |
60 | Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT), | 57 | Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT), |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index b8633e83e..c61c01d62 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -223,7 +223,7 @@ const SCHEDULER_INTERVALS_MS = { | |||
223 | REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day | 223 | REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day |
224 | REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day | 224 | REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day |
225 | UPDATE_INBOX_STATS: 1000 * 60, // 1 minute | 225 | UPDATE_INBOX_STATS: 1000 * 60, // 1 minute |
226 | REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 * 16 // 16 hours | 226 | REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour |
227 | } | 227 | } |
228 | 228 | ||
229 | // --------------------------------------------------------------------------- | 229 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts index d6e561cad..61e93eafa 100644 --- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts +++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts | |||
@@ -1,9 +1,7 @@ | |||
1 | import { map } from 'bluebird' | 1 | |
2 | import { readdir, remove, stat } from 'fs-extra' | ||
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
4 | import { getResumableUploadPath } from '@server/helpers/upload' | ||
5 | import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' | 3 | import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' |
6 | import { METAFILE_EXTNAME } from '@uploadx/core' | 4 | import { uploadx } from '../uploadx' |
7 | import { AbstractScheduler } from './abstract-scheduler' | 5 | import { AbstractScheduler } from './abstract-scheduler' |
8 | 6 | ||
9 | const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') | 7 | const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') |
@@ -22,36 +20,17 @@ export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { | |||
22 | } | 20 | } |
23 | 21 | ||
24 | protected async internalExecute () { | 22 | protected async internalExecute () { |
25 | const path = getResumableUploadPath() | 23 | logger.debug('Removing dangling resumable uploads', lTags()) |
26 | const files = await readdir(path) | ||
27 | |||
28 | const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME)) | ||
29 | 24 | ||
30 | if (metafiles.length === 0) return | 25 | const now = new Date().getTime() |
31 | |||
32 | logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags()) | ||
33 | 26 | ||
34 | try { | 27 | try { |
35 | await map(metafiles, metafile => { | 28 | // Remove files that were not updated since the last execution |
36 | return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs) | 29 | await uploadx.storage.purge(now - this.lastExecutionTimeMs) |
37 | }, { concurrency: 5 }) | ||
38 | } catch (error) { | 30 | } catch (error) { |
39 | logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) | 31 | logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) |
40 | } finally { | 32 | } finally { |
41 | this.lastExecutionTimeMs = new Date().getTime() | 33 | this.lastExecutionTimeMs = now |
42 | } | ||
43 | } | ||
44 | |||
45 | private async deleteIfOlderThan (metafile: string, olderThan: number) { | ||
46 | const metafilePath = getResumableUploadPath(metafile) | ||
47 | const statResult = await stat(metafilePath) | ||
48 | |||
49 | // Delete uploads that started since a long time | ||
50 | if (statResult.ctimeMs < olderThan) { | ||
51 | await remove(metafilePath) | ||
52 | |||
53 | const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '') | ||
54 | await remove(datafile) | ||
55 | } | 34 | } |
56 | } | 35 | } |
57 | 36 | ||
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts new file mode 100644 index 000000000..11b1044db --- /dev/null +++ b/server/lib/uploadx.ts | |||
@@ -0,0 +1,10 @@ | |||
1 | import express from 'express' | ||
2 | import { getResumableUploadPath } from '@server/helpers/upload' | ||
3 | import { Uploadx } from '@uploadx/core' | ||
4 | |||
5 | const uploadx = new Uploadx({ directory: getResumableUploadPath() }) | ||
6 | uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id | ||
7 | |||
8 | export { | ||
9 | uploadx | ||
10 | } | ||