aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts15
-rw-r--r--server/models/video/video.ts5
-rw-r--r--server/tests/api/object-storage/video-static-file-privacy.ts22
-rw-r--r--shared/server-commands/videos/videos-command.ts19
4 files changed, 42 insertions, 19 deletions
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index 05b52f412..d13c25798 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -1,3 +1,4 @@
1import { map } from 'bluebird'
1import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' 2import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra'
2import { dirname } from 'path' 3import { dirname } from 'path'
3import { Readable } from 'stream' 4import { Readable } from 'stream'
@@ -93,6 +94,8 @@ function updatePrefixACL (options: {
93 prefix, 94 prefix,
94 bucketInfo, 95 bucketInfo,
95 commandBuilder: obj => { 96 commandBuilder: obj => {
97 logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
98
96 return new PutObjectAclCommand({ 99 return new PutObjectAclCommand({
97 Bucket: bucketInfo.BUCKET_NAME, 100 Bucket: bucketInfo.BUCKET_NAME,
98 Key: obj.Key, 101 Key: obj.Key,
@@ -117,7 +120,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
117 return getClient().send(command) 120 return getClient().send(command)
118} 121}
119 122
120function removePrefix (prefix: string, bucketInfo: BucketInfo) { 123async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
121 // FIXME: use bulk delete when s3ninja will support this operation 124 // FIXME: use bulk delete when s3ninja will support this operation
122 125
123 logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) 126 logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
@@ -126,6 +129,8 @@ function removePrefix (prefix: string, bucketInfo: BucketInfo) {
126 prefix, 129 prefix,
127 bucketInfo, 130 bucketInfo,
128 commandBuilder: obj => { 131 commandBuilder: obj => {
132 logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
133
129 return new DeleteObjectCommand({ 134 return new DeleteObjectCommand({
130 Bucket: bucketInfo.BUCKET_NAME, 135 Bucket: bucketInfo.BUCKET_NAME,
131 Key: obj.Key 136 Key: obj.Key
@@ -259,7 +264,7 @@ async function applyOnPrefix (options: {
259 264
260 const s3Client = getClient() 265 const s3Client = getClient()
261 266
262 const commandPrefix = bucketInfo.PREFIX + prefix 267 const commandPrefix = buildKey(prefix, bucketInfo)
263 const listCommand = new ListObjectsV2Command({ 268 const listCommand = new ListObjectsV2Command({
264 Bucket: bucketInfo.BUCKET_NAME, 269 Bucket: bucketInfo.BUCKET_NAME,
265 Prefix: commandPrefix, 270 Prefix: commandPrefix,
@@ -275,11 +280,11 @@ async function applyOnPrefix (options: {
275 throw new Error(message) 280 throw new Error(message)
276 } 281 }
277 282
278 for (const object of listedObjects.Contents) { 283 await map(listedObjects.Contents, object => {
279 const command = commandBuilder(object) 284 const command = commandBuilder(object)
280 285
281 await s3Client.send(command) 286 return s3Client.send(command)
282 } 287 }, { concurrency: 10 })
283 288
284 // Repeat if not all objects could be listed at once (limit of 1000?) 289 // Repeat if not all objects could be listed at once (limit of 1000?)
285 if (listedObjects.IsTruncated) { 290 if (listedObjects.IsTruncated) {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index c568075d8..9399712b8 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -785,9 +785,8 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
785 785
786 // Do not wait video deletion because we could be in a transaction 786 // Do not wait video deletion because we could be in a transaction
787 Promise.all(tasks) 787 Promise.all(tasks)
788 .catch(err => { 788 .then(() => logger.info('Removed files of video %s.', instance.url))
789 logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err }) 789 .catch(err => logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err }))
790 })
791 790
792 return undefined 791 return undefined
793 } 792 }
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts
index 981bbaa16..c6d7a1a2c 100644
--- a/server/tests/api/object-storage/video-static-file-privacy.ts
+++ b/server/tests/api/object-storage/video-static-file-privacy.ts
@@ -192,16 +192,6 @@ describe('Object storage for video static file privacy', function () {
192 192
193 await checkPublicFiles(publicVideoUUID) 193 await checkPublicFiles(publicVideoUUID)
194 }) 194 })
195
196 after(async function () {
197 this.timeout(30000)
198
199 if (privateVideoUUID) await server.videos.remove({ id: privateVideoUUID })
200 if (publicVideoUUID) await server.videos.remove({ id: publicVideoUUID })
201 if (userPrivateVideoUUID) await server.videos.remove({ id: userPrivateVideoUUID })
202
203 await waitJobs([ server ])
204 })
205 }) 195 })
206 196
207 describe('Live', function () { 197 describe('Live', function () {
@@ -331,6 +321,18 @@ describe('Object storage for video static file privacy', function () {
331 }) 321 })
332 322
333 after(async function () { 323 after(async function () {
324 this.timeout(60000)
325
326 const { data } = await server.videos.listAllForAdmin()
327
328 for (const v of data) {
329 await server.videos.remove({ id: v.uuid })
330 }
331
332 for (const v of data) {
333 await server.servers.waitUntilLog('Removed files of video ' + v.url, 1, true)
334 }
335
334 await cleanupTests([ server ]) 336 await cleanupTests([ server ])
335 }) 337 })
336}) 338})
diff --git a/shared/server-commands/videos/videos-command.ts b/shared/server-commands/videos/videos-command.ts
index 1d6cad351..590244484 100644
--- a/shared/server-commands/videos/videos-command.ts
+++ b/shared/server-commands/videos/videos-command.ts
@@ -4,7 +4,7 @@ import { expect } from 'chai'
4import { createReadStream, stat } from 'fs-extra' 4import { createReadStream, stat } from 'fs-extra'
5import got, { Response as GotResponse } from 'got' 5import got, { Response as GotResponse } from 'got'
6import validator from 'validator' 6import validator from 'validator'
7import { buildAbsoluteFixturePath, omit, pick, wait } from '@shared/core-utils' 7import { buildAbsoluteFixturePath, getAllPrivacies, omit, pick, wait } from '@shared/core-utils'
8import { buildUUID } from '@shared/extra-utils' 8import { buildUUID } from '@shared/extra-utils'
9import { 9import {
10 HttpStatusCode, 10 HttpStatusCode,
@@ -15,6 +15,7 @@ import {
15 VideoCreateResult, 15 VideoCreateResult,
16 VideoDetails, 16 VideoDetails,
17 VideoFileMetadata, 17 VideoFileMetadata,
18 VideoInclude,
18 VideoPrivacy, 19 VideoPrivacy,
19 VideosCommonQuery, 20 VideosCommonQuery,
20 VideoTranscodingCreate 21 VideoTranscodingCreate
@@ -234,6 +235,22 @@ export class VideosCommand extends AbstractCommand {
234 }) 235 })
235 } 236 }
236 237
238 listAllForAdmin (options: OverrideCommandOptions & VideosCommonQuery = {}) {
239 const include = VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLACKLISTED | VideoInclude.BLOCKED_OWNER
240 const nsfw = 'both'
241 const privacyOneOf = getAllPrivacies()
242
243 return this.list({
244 ...options,
245
246 include,
247 nsfw,
248 privacyOneOf,
249
250 token: this.buildCommonRequestToken({ ...options, implicitToken: true })
251 })
252 }
253
237 listByAccount (options: OverrideCommandOptions & VideosCommonQuery & { 254 listByAccount (options: OverrideCommandOptions & VideosCommonQuery & {
238 handle: string 255 handle: string
239 }) { 256 }) {