aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-12-03 14:00:40 +0100
committerChocobozzz <me@florianbigard.com>2021-12-03 14:01:38 +0100
commit6d472b4046084e5f477124b11ac62dca9f6a1a63 (patch)
treed00c0223b92d8f77e68d46f7bc9e861725172b8e /server
parent390366472d7239864060753c63513d0f316ca468 (diff)
downloadPeerTube-6d472b4046084e5f477124b11ac62dca9f6a1a63.tar.gz
PeerTube-6d472b4046084e5f477124b11ac62dca9f6a1a63.tar.zst
PeerTube-6d472b4046084e5f477124b11ac62dca9f6a1a63.zip
Decrease time to cleanup resumable uploads
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/upload.ts9
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts35
-rw-r--r--server/lib/uploadx.ts10
4 files changed, 21 insertions, 35 deletions
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 6773b500f..c827f6bf0 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -8,6 +8,7 @@ import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
8import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 8import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
9import { generateWebTorrentVideoFilename } from '@server/lib/paths' 9import { generateWebTorrentVideoFilename } from '@server/lib/paths'
10import { Redis } from '@server/lib/redis' 10import { Redis } from '@server/lib/redis'
11import { uploadx } from '@server/lib/uploadx'
11import { 12import {
12 addMoveToObjectStorageJob, 13 addMoveToObjectStorageJob,
13 addOptimizeOrMergeAudioJob, 14 addOptimizeOrMergeAudioJob,
@@ -19,7 +20,6 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
19import { buildNextVideoState } from '@server/lib/video-state' 20import { buildNextVideoState } from '@server/lib/video-state'
20import { openapiOperationDoc } from '@server/middlewares/doc' 21import { openapiOperationDoc } from '@server/middlewares/doc'
21import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 22import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
22import { Uploadx } from '@uploadx/core'
23import { VideoCreate, VideoState } from '../../../../shared' 23import { VideoCreate, VideoState } from '../../../../shared'
24import { HttpStatusCode } from '../../../../shared/models' 24import { HttpStatusCode } from '../../../../shared/models'
25import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' 25import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
@@ -41,8 +41,8 @@ import {
41 authenticate, 41 authenticate,
42 videosAddLegacyValidator, 42 videosAddLegacyValidator,
43 videosAddResumableInitValidator, 43 videosAddResumableInitValidator,
44 videosResumableUploadIdValidator, 44 videosAddResumableValidator,
45 videosAddResumableValidator 45 videosResumableUploadIdValidator
46} from '../../../middlewares' 46} from '../../../middlewares'
47import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' 47import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
48import { VideoModel } from '../../../models/video/video' 48import { VideoModel } from '../../../models/video/video'
@@ -52,9 +52,6 @@ const lTags = loggerTagsFactory('api', 'video')
52const auditLogger = auditLoggerFactory('videos') 52const auditLogger = auditLoggerFactory('videos')
53const uploadRouter = express.Router() 53const uploadRouter = express.Router()
54 54
55const uploadx = new Uploadx({ directory: getResumableUploadPath() })
56uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id
57
58const reqVideoFileAdd = createReqFiles( 55const reqVideoFileAdd = createReqFiles(
59 [ 'videofile', 'thumbnailfile', 'previewfile' ], 56 [ 'videofile', 'thumbnailfile', 'previewfile' ],
60 Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT), 57 Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index b8633e83e..c61c01d62 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -223,7 +223,7 @@ const SCHEDULER_INTERVALS_MS = {
223 REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day 223 REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
224 REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day 224 REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
225 UPDATE_INBOX_STATS: 1000 * 60, // 1 minute 225 UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
226 REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 * 16 // 16 hours 226 REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
227} 227}
228 228
229// --------------------------------------------------------------------------- 229// ---------------------------------------------------------------------------
diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
index d6e561cad..61e93eafa 100644
--- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
+++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
@@ -1,9 +1,7 @@
1import { map } from 'bluebird' 1
2import { readdir, remove, stat } from 'fs-extra'
3import { logger, loggerTagsFactory } from '@server/helpers/logger' 2import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { getResumableUploadPath } from '@server/helpers/upload'
5import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' 3import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants'
6import { METAFILE_EXTNAME } from '@uploadx/core' 4import { uploadx } from '../uploadx'
7import { AbstractScheduler } from './abstract-scheduler' 5import { AbstractScheduler } from './abstract-scheduler'
8 6
9const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') 7const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
@@ -22,36 +20,17 @@ export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
22 } 20 }
23 21
24 protected async internalExecute () { 22 protected async internalExecute () {
25 const path = getResumableUploadPath() 23 logger.debug('Removing dangling resumable uploads', lTags())
26 const files = await readdir(path)
27
28 const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME))
29 24
30 if (metafiles.length === 0) return 25 const now = new Date().getTime()
31
32 logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags())
33 26
34 try { 27 try {
35 await map(metafiles, metafile => { 28 // Remove files that were not updated since the last execution
36 return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs) 29 await uploadx.storage.purge(now - this.lastExecutionTimeMs)
37 }, { concurrency: 5 })
38 } catch (error) { 30 } catch (error) {
39 logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) 31 logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
40 } finally { 32 } finally {
41 this.lastExecutionTimeMs = new Date().getTime() 33 this.lastExecutionTimeMs = now
42 }
43 }
44
45 private async deleteIfOlderThan (metafile: string, olderThan: number) {
46 const metafilePath = getResumableUploadPath(metafile)
47 const statResult = await stat(metafilePath)
48
49 // Delete uploads that started since a long time
50 if (statResult.ctimeMs < olderThan) {
51 await remove(metafilePath)
52
53 const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '')
54 await remove(datafile)
55 } 34 }
56 } 35 }
57 36
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts
new file mode 100644
index 000000000..11b1044db
--- /dev/null
+++ b/server/lib/uploadx.ts
@@ -0,0 +1,10 @@
1import express from 'express'
2import { getResumableUploadPath } from '@server/helpers/upload'
3import { Uploadx } from '@uploadx/core'
4
5const uploadx = new Uploadx({ directory: getResumableUploadPath() })
6uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id
7
8export {
9 uploadx
10}