diff options
author | Chocobozzz <me@florianbigard.com> | 2022-10-25 11:51:20 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2022-10-25 11:51:20 +0200 |
commit | 508c1b1e9f3b26752a961e945b7fa59b72b30827 (patch) | |
tree | a027845a959291dad787041396c2f9e30dc1af98 /server | |
parent | 849f0fd3b2d00056a2c6252230814d6c2e3e3919 (diff) | |
download | PeerTube-508c1b1e9f3b26752a961e945b7fa59b72b30827.tar.gz PeerTube-508c1b1e9f3b26752a961e945b7fa59b72b30827.tar.zst PeerTube-508c1b1e9f3b26752a961e945b7fa59b72b30827.zip |
Correctly cleanup files from object storage
Diffstat (limited to 'server')
-rw-r--r-- | server/lib/object-storage/shared/object-storage-helpers.ts | 15 | ||||
-rw-r--r-- | server/models/video/video.ts | 5 | ||||
-rw-r--r-- | server/tests/api/object-storage/video-static-file-privacy.ts | 22 |
3 files changed, 24 insertions, 18 deletions
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts index 05b52f412..d13c25798 100644 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import { map } from 'bluebird' | ||
1 | import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' | 2 | import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' |
2 | import { dirname } from 'path' | 3 | import { dirname } from 'path' |
3 | import { Readable } from 'stream' | 4 | import { Readable } from 'stream' |
@@ -93,6 +94,8 @@ function updatePrefixACL (options: { | |||
93 | prefix, | 94 | prefix, |
94 | bucketInfo, | 95 | bucketInfo, |
95 | commandBuilder: obj => { | 96 | commandBuilder: obj => { |
97 | logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
98 | |||
96 | return new PutObjectAclCommand({ | 99 | return new PutObjectAclCommand({ |
97 | Bucket: bucketInfo.BUCKET_NAME, | 100 | Bucket: bucketInfo.BUCKET_NAME, |
98 | Key: obj.Key, | 101 | Key: obj.Key, |
@@ -117,7 +120,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { | |||
117 | return getClient().send(command) | 120 | return getClient().send(command) |
118 | } | 121 | } |
119 | 122 | ||
120 | function removePrefix (prefix: string, bucketInfo: BucketInfo) { | 123 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { |
121 | // FIXME: use bulk delete when s3ninja will support this operation | 124 | // FIXME: use bulk delete when s3ninja will support this operation |
122 | 125 | ||
123 | logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) | 126 | logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) |
@@ -126,6 +129,8 @@ function removePrefix (prefix: string, bucketInfo: BucketInfo) { | |||
126 | prefix, | 129 | prefix, |
127 | bucketInfo, | 130 | bucketInfo, |
128 | commandBuilder: obj => { | 131 | commandBuilder: obj => { |
132 | logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
133 | |||
129 | return new DeleteObjectCommand({ | 134 | return new DeleteObjectCommand({ |
130 | Bucket: bucketInfo.BUCKET_NAME, | 135 | Bucket: bucketInfo.BUCKET_NAME, |
131 | Key: obj.Key | 136 | Key: obj.Key |
@@ -259,7 +264,7 @@ async function applyOnPrefix (options: { | |||
259 | 264 | ||
260 | const s3Client = getClient() | 265 | const s3Client = getClient() |
261 | 266 | ||
262 | const commandPrefix = bucketInfo.PREFIX + prefix | 267 | const commandPrefix = buildKey(prefix, bucketInfo) |
263 | const listCommand = new ListObjectsV2Command({ | 268 | const listCommand = new ListObjectsV2Command({ |
264 | Bucket: bucketInfo.BUCKET_NAME, | 269 | Bucket: bucketInfo.BUCKET_NAME, |
265 | Prefix: commandPrefix, | 270 | Prefix: commandPrefix, |
@@ -275,11 +280,11 @@ async function applyOnPrefix (options: { | |||
275 | throw new Error(message) | 280 | throw new Error(message) |
276 | } | 281 | } |
277 | 282 | ||
278 | for (const object of listedObjects.Contents) { | 283 | await map(listedObjects.Contents, object => { |
279 | const command = commandBuilder(object) | 284 | const command = commandBuilder(object) |
280 | 285 | ||
281 | await s3Client.send(command) | 286 | return s3Client.send(command) |
282 | } | 287 | }, { concurrency: 10 }) |
283 | 288 | ||
284 | // Repeat if not all objects could be listed at once (limit of 1000?) | 289 | // Repeat if not all objects could be listed at once (limit of 1000?) |
285 | if (listedObjects.IsTruncated) { | 290 | if (listedObjects.IsTruncated) { |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index c568075d8..9399712b8 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -785,9 +785,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
785 | 785 | ||
786 | // Do not wait video deletion because we could be in a transaction | 786 | // Do not wait video deletion because we could be in a transaction |
787 | Promise.all(tasks) | 787 | Promise.all(tasks) |
788 | .catch(err => { | 788 | .then(() => logger.info('Removed files of video %s.', instance.url)) |
789 | logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err }) | 789 | .catch(err => logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err })) |
790 | }) | ||
791 | 790 | ||
792 | return undefined | 791 | return undefined |
793 | } | 792 | } |
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts index 981bbaa16..c6d7a1a2c 100644 --- a/server/tests/api/object-storage/video-static-file-privacy.ts +++ b/server/tests/api/object-storage/video-static-file-privacy.ts | |||
@@ -192,16 +192,6 @@ describe('Object storage for video static file privacy', function () { | |||
192 | 192 | ||
193 | await checkPublicFiles(publicVideoUUID) | 193 | await checkPublicFiles(publicVideoUUID) |
194 | }) | 194 | }) |
195 | |||
196 | after(async function () { | ||
197 | this.timeout(30000) | ||
198 | |||
199 | if (privateVideoUUID) await server.videos.remove({ id: privateVideoUUID }) | ||
200 | if (publicVideoUUID) await server.videos.remove({ id: publicVideoUUID }) | ||
201 | if (userPrivateVideoUUID) await server.videos.remove({ id: userPrivateVideoUUID }) | ||
202 | |||
203 | await waitJobs([ server ]) | ||
204 | }) | ||
205 | }) | 195 | }) |
206 | 196 | ||
207 | describe('Live', function () { | 197 | describe('Live', function () { |
@@ -331,6 +321,18 @@ describe('Object storage for video static file privacy', function () { | |||
331 | }) | 321 | }) |
332 | 322 | ||
333 | after(async function () { | 323 | after(async function () { |
324 | this.timeout(60000) | ||
325 | |||
326 | const { data } = await server.videos.listAllForAdmin() | ||
327 | |||
328 | for (const v of data) { | ||
329 | await server.videos.remove({ id: v.uuid }) | ||
330 | } | ||
331 | |||
332 | for (const v of data) { | ||
333 | await server.servers.waitUntilLog('Removed files of video ' + v.url, 1, true) | ||
334 | } | ||
335 | |||
334 | await cleanupTests([ server ]) | 336 | await cleanupTests([ server ]) |
335 | }) | 337 | }) |
336 | }) | 338 | }) |