diff options
author | Jelle Besseling <jelle@pingiun.com> | 2021-08-17 08:26:20 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-08-17 08:26:20 +0200 |
commit | 0305db28c98fd6cf43a3c50ba92c76215e99d512 (patch) | |
tree | 33b753a19728d9f453c1aa4f19b36ac797e5fe80 | |
parent | f88ae8f5bc223579313b28582de9101944a4a814 (diff) | |
download | PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.gz PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.zst PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.zip |
Add support for saving video files to object storage (#4290)
* Add support for saving video files to object storage
* Add support for custom url generation on s3 stored files
Uses two config keys to support url generation that doesn't directly go
to (compatible s3). Can be used to generate urls to any cache server or
CDN.
* Upload files to s3 concurrently and delete originals afterwards
* Only publish after move to object storage is complete
* Use base url instead of url template
* Fix mistyped config field
* Add rudenmentary way to download before transcode
* Implement Chocobozzz suggestions
https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478
The remarks in question:
Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names
Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket
Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET)
I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs)
https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer
I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs
* Import correct function
* Support multipart upload
* Remove import of node 15.0 module stream/promises
* Extend maximum upload job length
Using the same value as for redundancy downloading seems logical
* Use dynamic part size for really large uploads
Also adds very small part size for local testing
* Fix decreasePendingMove query
* Resolve various PR comments
* Move to object storage after optimize
* Make upload size configurable and increase default
* Prune webtorrent files that are stored in object storage
* Move files after transcoding jobs
* Fix federation
* Add video path manager
* Support move to external storage job in client
* Fix live object storage tests
Co-authored-by: Chocobozzz <me@florianbigard.com>
86 files changed, 3585 insertions, 573 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5bbd9e2c..35c91bf85 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml | |||
@@ -31,6 +31,11 @@ jobs: | |||
31 | ports: | 31 | ports: |
32 | - 10389:10389 | 32 | - 10389:10389 |
33 | 33 | ||
34 | s3ninja: | ||
35 | image: scireum/s3-ninja | ||
36 | ports: | ||
37 | - 9444:9000 | ||
38 | |||
34 | strategy: | 39 | strategy: |
35 | fail-fast: false | 40 | fail-fast: false |
36 | matrix: | 41 | matrix: |
@@ -40,6 +45,7 @@ jobs: | |||
40 | PGUSER: peertube | 45 | PGUSER: peertube |
41 | PGHOST: localhost | 46 | PGHOST: localhost |
42 | NODE_PENDING_JOB_WAIT: 250 | 47 | NODE_PENDING_JOB_WAIT: 250 |
48 | ENABLE_OBJECT_STORAGE_TESTS: true | ||
43 | 49 | ||
44 | steps: | 50 | steps: |
45 | - uses: actions/checkout@v2 | 51 | - uses: actions/checkout@v2 |
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 29ba95c5c..4b02e1bc1 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts | |||
@@ -36,7 +36,8 @@ export class JobsComponent extends RestTable implements OnInit { | |||
36 | 'video-live-ending', | 36 | 'video-live-ending', |
37 | 'video-redundancy', | 37 | 'video-redundancy', |
38 | 'video-transcoding', | 38 | 'video-transcoding', |
39 | 'videos-views' | 39 | 'videos-views', |
40 | 'move-to-object-storage' | ||
40 | ] | 41 | ] |
41 | 42 | ||
42 | jobs: Job[] = [] | 43 | jobs: Job[] = [] |
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html index 3480d3656..e2dd44bf7 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html | |||
@@ -6,6 +6,10 @@ | |||
6 | The video is being transcoded, it may not work properly. | 6 | The video is being transcoded, it may not work properly. |
7 | </div> | 7 | </div> |
8 | 8 | ||
9 | <div i18n class="alert alert-warning" *ngIf="isVideoToMoveToExternalStorage()"> | ||
10 | The video is being moved to an external server, it may not work properly. | ||
11 | </div> | ||
12 | |||
9 | <div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()"> | 13 | <div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()"> |
10 | This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}. | 14 | This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}. |
11 | </div> | 15 | </div> |
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts index 8a46ba0d5..0072492ac 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts | |||
@@ -18,6 +18,10 @@ export class VideoAlertComponent { | |||
18 | return this.video && this.video.state.id === VideoState.TO_IMPORT | 18 | return this.video && this.video.state.id === VideoState.TO_IMPORT |
19 | } | 19 | } |
20 | 20 | ||
21 | isVideoToMoveToExternalStorage () { | ||
22 | return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | ||
23 | } | ||
24 | |||
21 | hasVideoScheduledPublication () { | 25 | hasVideoScheduledPublication () { |
22 | return this.video && this.video.scheduledUpdate !== undefined | 26 | return this.video && this.video.scheduledUpdate !== undefined |
23 | } | 27 | } |
diff --git a/config/default.yaml b/config/default.yaml index 3d0ae6e87..3865ab5cf 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -95,6 +95,39 @@ storage: | |||
95 | # If not, peertube will fallback to the default fil | 95 | # If not, peertube will fallback to the default fil |
96 | client_overrides: 'storage/client-overrides/' | 96 | client_overrides: 'storage/client-overrides/' |
97 | 97 | ||
98 | object_storage: | ||
99 | enabled: false | ||
100 | |||
101 | # Without protocol, will default to HTTPS | ||
102 | endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example | ||
103 | |||
104 | region: 'us-east-1' | ||
105 | |||
106 | credentials: | ||
107 | # You can also use AWS_ACCESS_KEY_ID env variable | ||
108 | access_key_id: '' | ||
109 | # You can also use AWS_SECRET_ACCESS_KEY env variable | ||
110 | secret_access_key: '' | ||
111 | |||
112 | # Maximum amount to upload in one request to object storage | ||
113 | max_upload_part: 2GB | ||
114 | |||
115 | streaming_playlists: | ||
116 | bucket_name: 'streaming-playlists' | ||
117 | |||
118 | # Allows setting all buckets to the same value but with a different prefix | ||
119 | prefix: '' # Example: 'streaming-playlists:' | ||
120 | |||
121 | # Base url for object URL generation, scheme and host will be replaced by this URL | ||
122 | # Useful when you want to use a CDN/external proxy | ||
123 | base_url: '' # Example: 'https://mirror.example.com' | ||
124 | |||
125 | # Same settings but for webtorrent videos | ||
126 | videos: | ||
127 | bucket_name: 'videos' | ||
128 | prefix: '' | ||
129 | base_url: '' | ||
130 | |||
98 | log: | 131 | log: |
99 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' | 132 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' |
100 | rotation: | 133 | rotation: |
diff --git a/config/production.yaml.example b/config/production.yaml.example index 514ab99a4..94238fad0 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example | |||
@@ -93,6 +93,39 @@ storage: | |||
93 | # If not, peertube will fallback to the default file | 93 | # If not, peertube will fallback to the default file |
94 | client_overrides: '/var/www/peertube/storage/client-overrides/' | 94 | client_overrides: '/var/www/peertube/storage/client-overrides/' |
95 | 95 | ||
96 | object_storage: | ||
97 | enabled: false | ||
98 | |||
99 | # Without protocol, will default to HTTPS | ||
100 | endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example | ||
101 | |||
102 | region: 'us-east-1' | ||
103 | |||
104 | credentials: | ||
105 | # You can also use AWS_ACCESS_KEY_ID env variable | ||
106 | access_key_id: '' | ||
107 | # You can also use AWS_SECRET_ACCESS_KEY env variable | ||
108 | secret_access_key: '' | ||
109 | |||
110 | # Maximum amount to upload in one request to object storage | ||
111 | max_upload_part: 2GB | ||
112 | |||
113 | streaming_playlists: | ||
114 | bucket_name: 'streaming-playlists' | ||
115 | |||
116 | # Allows setting all buckets to the same value but with a different prefix | ||
117 | prefix: '' # Example: 'streaming-playlists:' | ||
118 | |||
119 | # Base url for object URL generation, scheme and host will be replaced by this URL | ||
120 | # Useful when you want to use a CDN/external proxy | ||
121 | base_url: '' # Example: 'https://mirror.example.com' | ||
122 | |||
123 | # Same settings but for webtorrent videos | ||
124 | videos: | ||
125 | bucket_name: 'videos' | ||
126 | prefix: '' | ||
127 | base_url: '' | ||
128 | |||
96 | log: | 129 | log: |
97 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' | 130 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' |
98 | rotation: | 131 | rotation: |
diff --git a/package.json b/package.json index 807a90742..d2b1e1245 100644 --- a/package.json +++ b/package.json | |||
@@ -73,6 +73,7 @@ | |||
73 | "swagger-cli": "swagger-cli" | 73 | "swagger-cli": "swagger-cli" |
74 | }, | 74 | }, |
75 | "dependencies": { | 75 | "dependencies": { |
76 | "@aws-sdk/client-s3": "^3.23.0", | ||
76 | "@uploadx/core": "^4.4.0", | 77 | "@uploadx/core": "^4.4.0", |
77 | "async": "^3.0.1", | 78 | "async": "^3.0.1", |
78 | "async-lru": "^1.1.1", | 79 | "async-lru": "^1.1.1", |
diff --git a/scripts/ci.sh b/scripts/ci.sh index 71b1be53b..f49dbe6ad 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh | |||
@@ -89,9 +89,10 @@ elif [ "$1" = "api-4" ]; then | |||
89 | 89 | ||
90 | moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation) | 90 | moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation) |
91 | redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy) | 91 | redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy) |
92 | objectStorageFiles=$(findTestFiles ./dist/server/tests/api/object-storage) | ||
92 | activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub) | 93 | activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub) |
93 | 94 | ||
94 | MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles | 95 | MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles |
95 | elif [ "$1" = "external-plugins" ]; then | 96 | elif [ "$1" = "external-plugins" ]; then |
96 | npm run build:server | 97 | npm run build:server |
97 | 98 | ||
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index 3a552c19a..0bb9bfeab 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts | |||
@@ -6,9 +6,10 @@ import { VideoModel } from '../server/models/video/video' | |||
6 | import { initDatabaseModels } from '../server/initializers/database' | 6 | import { initDatabaseModels } from '../server/initializers/database' |
7 | import { JobQueue } from '../server/lib/job-queue' | 7 | import { JobQueue } from '../server/lib/job-queue' |
8 | import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils' | 8 | import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils' |
9 | import { VideoTranscodingPayload } from '@shared/models' | 9 | import { VideoState, VideoTranscodingPayload } from '@shared/models' |
10 | import { CONFIG } from '@server/initializers/config' | 10 | import { CONFIG } from '@server/initializers/config' |
11 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' | 11 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' |
12 | import { addTranscodingJob } from '@server/lib/video' | ||
12 | 13 | ||
13 | program | 14 | program |
14 | .option('-v, --video [videoUUID]', 'Video UUID') | 15 | .option('-v, --video [videoUUID]', 'Video UUID') |
@@ -47,7 +48,7 @@ async function run () { | |||
47 | if (!video) throw new Error('Video not found.') | 48 | if (!video) throw new Error('Video not found.') |
48 | 49 | ||
49 | const dataInput: VideoTranscodingPayload[] = [] | 50 | const dataInput: VideoTranscodingPayload[] = [] |
50 | const { resolution } = await video.getMaxQualityResolution() | 51 | const resolution = video.getMaxQualityFile().resolution |
51 | 52 | ||
52 | // Generate HLS files | 53 | // Generate HLS files |
53 | if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { | 54 | if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { |
@@ -62,6 +63,7 @@ async function run () { | |||
62 | resolution, | 63 | resolution, |
63 | isPortraitMode: false, | 64 | isPortraitMode: false, |
64 | copyCodecs: false, | 65 | copyCodecs: false, |
66 | isNewVideo: false, | ||
65 | isMaxQuality: false | 67 | isMaxQuality: false |
66 | }) | 68 | }) |
67 | } | 69 | } |
@@ -87,10 +89,13 @@ async function run () { | |||
87 | } | 89 | } |
88 | } | 90 | } |
89 | 91 | ||
90 | await JobQueue.Instance.init() | 92 | JobQueue.Instance.init() |
93 | |||
94 | video.state = VideoState.TO_TRANSCODE | ||
95 | await video.save() | ||
91 | 96 | ||
92 | for (const d of dataInput) { | 97 | for (const d of dataInput) { |
93 | await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: d }) | 98 | await addTranscodingJob(d, {}) |
94 | console.log('Transcoding job for video %s created.', video.uuid) | 99 | console.log('Transcoding job for video %s created.', video.uuid) |
95 | } | 100 | } |
96 | } | 101 | } |
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index 9e66105dd..245e4cf28 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts | |||
@@ -1,15 +1,18 @@ | |||
1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' | 1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' |
2 | registerTSPaths() | 2 | registerTSPaths() |
3 | 3 | ||
4 | import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils' | ||
5 | import { VideoModel } from '../server/models/video/video' | ||
6 | import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding' | ||
7 | import { initDatabaseModels } from '../server/initializers/database' | ||
8 | import { basename, dirname } from 'path' | ||
9 | import { copy, move, remove } from 'fs-extra' | 4 | import { copy, move, remove } from 'fs-extra' |
5 | import { basename, dirname } from 'path' | ||
10 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
11 | import { getVideoFilePath } from '@server/lib/video-paths' | 7 | import { CONFIG } from '@server/initializers/config' |
8 | import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage' | ||
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
12 | import { getMaxBitrate } from '@shared/core-utils' | 10 | import { getMaxBitrate } from '@shared/core-utils' |
11 | import { MoveObjectStoragePayload } from '@shared/models' | ||
12 | import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils' | ||
13 | import { initDatabaseModels } from '../server/initializers/database' | ||
14 | import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding' | ||
15 | import { VideoModel } from '../server/models/video/video' | ||
13 | 16 | ||
14 | run() | 17 | run() |
15 | .then(() => process.exit(0)) | 18 | .then(() => process.exit(0)) |
@@ -39,43 +42,49 @@ async function run () { | |||
39 | currentVideoId = video.id | 42 | currentVideoId = video.id |
40 | 43 | ||
41 | for (const file of video.VideoFiles) { | 44 | for (const file of video.VideoFiles) { |
42 | currentFilePath = getVideoFilePath(video, file) | 45 | await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => { |
43 | 46 | currentFilePath = path | |
44 | const [ videoBitrate, fps, dataResolution ] = await Promise.all([ | 47 | |
45 | getVideoFileBitrate(currentFilePath), | 48 | const [ videoBitrate, fps, dataResolution ] = await Promise.all([ |
46 | getVideoFileFPS(currentFilePath), | 49 | getVideoFileBitrate(currentFilePath), |
47 | getVideoFileResolution(currentFilePath) | 50 | getVideoFileFPS(currentFilePath), |
48 | ]) | 51 | getVideoFileResolution(currentFilePath) |
49 | 52 | ]) | |
50 | const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) | 53 | |
51 | const isMaxBitrateExceeded = videoBitrate > maxBitrate | 54 | const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) |
52 | if (isMaxBitrateExceeded) { | 55 | const isMaxBitrateExceeded = videoBitrate > maxBitrate |
53 | console.log( | 56 | if (isMaxBitrateExceeded) { |
54 | 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', | 57 | console.log( |
55 | basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 | 58 | 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', |
56 | ) | 59 | basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 |
57 | 60 | ) | |
58 | const backupFile = `${currentFilePath}_backup` | 61 | |
59 | await copy(currentFilePath, backupFile) | 62 | const backupFile = `${currentFilePath}_backup` |
60 | 63 | await copy(currentFilePath, backupFile) | |
61 | await optimizeOriginalVideofile(video, file) | 64 | |
62 | // Update file path, the video filename changed | 65 | await optimizeOriginalVideofile(video, file) |
63 | currentFilePath = getVideoFilePath(video, file) | 66 | // Update file path, the video filename changed |
64 | 67 | currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) | |
65 | const originalDuration = await getDurationFromVideoFile(backupFile) | 68 | |
66 | const newDuration = await getDurationFromVideoFile(currentFilePath) | 69 | const originalDuration = await getDurationFromVideoFile(backupFile) |
67 | 70 | const newDuration = await getDurationFromVideoFile(currentFilePath) | |
68 | if (originalDuration === newDuration) { | 71 | |
69 | console.log('Finished optimizing %s', basename(currentFilePath)) | 72 | if (originalDuration === newDuration) { |
70 | await remove(backupFile) | 73 | console.log('Finished optimizing %s', basename(currentFilePath)) |
71 | continue | 74 | await remove(backupFile) |
75 | return | ||
76 | } | ||
77 | |||
78 | console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) | ||
79 | await move(backupFile, currentFilePath, { overwrite: true }) | ||
80 | await createTorrentAndSetInfoHash(video, file) | ||
81 | await file.save() | ||
72 | } | 82 | } |
83 | }) | ||
84 | } | ||
73 | 85 | ||
74 | console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) | 86 | if (CONFIG.OBJECT_STORAGE.ENABLED === true) { |
75 | await move(backupFile, currentFilePath, { overwrite: true }) | 87 | await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any) |
76 | await createTorrentAndSetInfoHash(video, file) | ||
77 | await file.save() | ||
78 | } | ||
79 | } | 88 | } |
80 | } | 89 | } |
81 | 90 | ||
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 89f50714d..5c740c041 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts | |||
@@ -1,12 +1,21 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { move } from 'fs-extra' | 2 | import { move } from 'fs-extra' |
3 | import { basename } from 'path' | ||
3 | import { getLowercaseExtension } from '@server/helpers/core-utils' | 4 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' | 5 | import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' |
5 | import { uuidToShort } from '@server/helpers/uuid' | 6 | import { uuidToShort } from '@server/helpers/uuid' |
6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
7 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 8 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
8 | import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' | 9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' |
9 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 10 | import { |
11 | addMoveToObjectStorageJob, | ||
12 | addOptimizeOrMergeAudioJob, | ||
13 | buildLocalVideoFromReq, | ||
14 | buildVideoThumbnailsFromReq, | ||
15 | setVideoTags | ||
16 | } from '@server/lib/video' | ||
17 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
18 | import { buildNextVideoState } from '@server/lib/video-state' | ||
10 | import { openapiOperationDoc } from '@server/middlewares/doc' | 19 | import { openapiOperationDoc } from '@server/middlewares/doc' |
11 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 20 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
12 | import { uploadx } from '@uploadx/core' | 21 | import { uploadx } from '@uploadx/core' |
@@ -139,23 +148,20 @@ async function addVideo (options: { | |||
139 | 148 | ||
140 | const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) | 149 | const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) |
141 | 150 | ||
142 | videoData.state = CONFIG.TRANSCODING.ENABLED | 151 | videoData.state = buildNextVideoState() |
143 | ? VideoState.TO_TRANSCODE | ||
144 | : VideoState.PUBLISHED | ||
145 | |||
146 | videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware | 152 | videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware |
147 | 153 | ||
148 | const video = new VideoModel(videoData) as MVideoFullLight | 154 | const video = new VideoModel(videoData) as MVideoFullLight |
149 | video.VideoChannel = videoChannel | 155 | video.VideoChannel = videoChannel |
150 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object | 156 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object |
151 | 157 | ||
152 | const videoFile = await buildNewFile(video, videoPhysicalFile) | 158 | const videoFile = await buildNewFile(videoPhysicalFile) |
153 | 159 | ||
154 | // Move physical file | 160 | // Move physical file |
155 | const destination = getVideoFilePath(video, videoFile) | 161 | const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) |
156 | await move(videoPhysicalFile.path, destination) | 162 | await move(videoPhysicalFile.path, destination) |
157 | // This is important in case if there is another attempt in the retry process | 163 | // This is important in case if there is another attempt in the retry process |
158 | videoPhysicalFile.filename = getVideoFilePath(video, videoFile) | 164 | videoPhysicalFile.filename = basename(destination) |
159 | videoPhysicalFile.path = destination | 165 | videoPhysicalFile.path = destination |
160 | 166 | ||
161 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ | 167 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ |
@@ -210,9 +216,13 @@ async function addVideo (options: { | |||
210 | 216 | ||
211 | createTorrentFederate(video, videoFile) | 217 | createTorrentFederate(video, videoFile) |
212 | .then(() => { | 218 | .then(() => { |
213 | if (video.state !== VideoState.TO_TRANSCODE) return | 219 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { |
220 | return addMoveToObjectStorageJob(video) | ||
221 | } | ||
214 | 222 | ||
215 | return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) | 223 | if (video.state === VideoState.TO_TRANSCODE) { |
224 | return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) | ||
225 | } | ||
216 | }) | 226 | }) |
217 | .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) | 227 | .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) |
218 | 228 | ||
@@ -227,7 +237,7 @@ async function addVideo (options: { | |||
227 | }) | 237 | }) |
228 | } | 238 | } |
229 | 239 | ||
230 | async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { | 240 | async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) { |
231 | const videoFile = new VideoFileModel({ | 241 | const videoFile = new VideoFileModel({ |
232 | extname: getLowercaseExtension(videoPhysicalFile.filename), | 242 | extname: getLowercaseExtension(videoPhysicalFile.filename), |
233 | size: videoPhysicalFile.size, | 243 | size: videoPhysicalFile.size, |
diff --git a/server/controllers/download.ts b/server/controllers/download.ts index ddacc1b68..ffe40d57e 100644 --- a/server/controllers/download.ts +++ b/server/controllers/download.ts | |||
@@ -3,9 +3,9 @@ import * as express from 'express' | |||
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' | 4 | import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' |
5 | import { Hooks } from '@server/lib/plugins/hooks' | 5 | import { Hooks } from '@server/lib/plugins/hooks' |
6 | import { getVideoFilePath } from '@server/lib/video-paths' | 6 | import { VideoPathManager } from '@server/lib/video-path-manager' |
7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
8 | import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' | 8 | import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' |
9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' | 9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' |
10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' | 10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' |
11 | 11 | ||
@@ -81,7 +81,15 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { | |||
81 | 81 | ||
82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
83 | 83 | ||
84 | return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) | 84 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
85 | return res.redirect(videoFile.getObjectStorageUrl()) | ||
86 | } | ||
87 | |||
88 | await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => { | ||
89 | const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}` | ||
90 | |||
91 | return res.download(path, filename) | ||
92 | }) | ||
85 | } | 93 | } |
86 | 94 | ||
87 | async function downloadHLSVideoFile (req: express.Request, res: express.Response) { | 95 | async function downloadHLSVideoFile (req: express.Request, res: express.Response) { |
@@ -107,8 +115,15 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response | |||
107 | 115 | ||
108 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 116 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
109 | 117 | ||
110 | const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` | 118 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
111 | return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename) | 119 | return res.redirect(videoFile.getObjectStorageUrl()) |
120 | } | ||
121 | |||
122 | await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => { | ||
123 | const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` | ||
124 | |||
125 | return res.download(path, filename) | ||
126 | }) | ||
112 | } | 127 | } |
113 | 128 | ||
114 | function getVideoFile (req: express.Request, files: MVideoFile[]) { | 129 | function getVideoFile (req: express.Request, files: MVideoFile[]) { |
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index ecf63e93e..c84376304 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -6,7 +6,8 @@ import { dirname, join } from 'path' | |||
6 | import * as WebTorrent from 'webtorrent' | 6 | import * as WebTorrent from 'webtorrent' |
7 | import { isArray } from '@server/helpers/custom-validators/misc' | 7 | import { isArray } from '@server/helpers/custom-validators/misc' |
8 | import { WEBSERVER } from '@server/initializers/constants' | 8 | import { WEBSERVER } from '@server/initializers/constants' |
9 | import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' | 9 | import { generateTorrentFileName } from '@server/lib/paths' |
10 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
10 | import { MVideo } from '@server/types/models/video/video' | 11 | import { MVideo } from '@server/types/models/video/video' |
11 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' | 12 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' |
12 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' | 13 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' |
@@ -78,7 +79,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName | |||
78 | }) | 79 | }) |
79 | } | 80 | } |
80 | 81 | ||
81 | async function createTorrentAndSetInfoHash ( | 82 | function createTorrentAndSetInfoHash ( |
82 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | 83 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, |
83 | videoFile: MVideoFile | 84 | videoFile: MVideoFile |
84 | ) { | 85 | ) { |
@@ -95,22 +96,24 @@ async function createTorrentAndSetInfoHash ( | |||
95 | urlList: [ videoFile.getFileUrl(video) ] | 96 | urlList: [ videoFile.getFileUrl(video) ] |
96 | } | 97 | } |
97 | 98 | ||
98 | const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) | 99 | return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { |
100 | const torrent = await createTorrentPromise(videoPath, options) | ||
99 | 101 | ||
100 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) | 102 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) |
101 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) | 103 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) |
102 | logger.info('Creating torrent %s.', torrentPath) | 104 | logger.info('Creating torrent %s.', torrentPath) |
103 | 105 | ||
104 | await writeFile(torrentPath, torrent) | 106 | await writeFile(torrentPath, torrent) |
105 | 107 | ||
106 | // Remove old torrent file if it existed | 108 | // Remove old torrent file if it existed |
107 | if (videoFile.hasTorrent()) { | 109 | if (videoFile.hasTorrent()) { |
108 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) | 110 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) |
109 | } | 111 | } |
110 | 112 | ||
111 | const parsedTorrent = parseTorrent(torrent) | 113 | const parsedTorrent = parseTorrent(torrent) |
112 | videoFile.infoHash = parsedTorrent.infoHash | 114 | videoFile.infoHash = parsedTorrent.infoHash |
113 | videoFile.torrentFilename = torrentFilename | 115 | videoFile.torrentFilename = torrentFilename |
116 | }) | ||
114 | } | 117 | } |
115 | 118 | ||
116 | function generateMagnetUri ( | 119 | function generateMagnetUri ( |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index 911734fa0..09f587274 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -153,6 +153,29 @@ function checkConfig () { | |||
153 | } | 153 | } |
154 | } | 154 | } |
155 | 155 | ||
156 | // Object storage | ||
157 | if (CONFIG.OBJECT_STORAGE.ENABLED === true) { | ||
158 | |||
159 | if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) { | ||
160 | return 'videos_bucket should be set when object storage support is enabled.' | ||
161 | } | ||
162 | |||
163 | if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { | ||
164 | return 'streaming_playlists_bucket should be set when object storage support is enabled.' | ||
165 | } | ||
166 | |||
167 | if ( | ||
168 | CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && | ||
169 | CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX | ||
170 | ) { | ||
171 | if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') { | ||
172 | return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.' | ||
173 | } else { | ||
174 | return 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' | ||
175 | } | ||
176 | } | ||
177 | } | ||
178 | |||
156 | return null | 179 | return null |
157 | } | 180 | } |
158 | 181 | ||
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 30a9823b9..0e684eef8 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -73,6 +73,26 @@ const CONFIG = { | |||
73 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), | 73 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), |
74 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) | 74 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) |
75 | }, | 75 | }, |
76 | OBJECT_STORAGE: { | ||
77 | ENABLED: config.get<boolean>('object_storage.enabled'), | ||
78 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), | ||
79 | ENDPOINT: config.get<string>('object_storage.endpoint'), | ||
80 | REGION: config.get<string>('object_storage.region'), | ||
81 | CREDENTIALS: { | ||
82 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), | ||
83 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') | ||
84 | }, | ||
85 | VIDEOS: { | ||
86 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), | ||
87 | PREFIX: config.get<string>('object_storage.videos.prefix'), | ||
88 | BASE_URL: config.get<string>('object_storage.videos.base_url') | ||
89 | }, | ||
90 | STREAMING_PLAYLISTS: { | ||
91 | BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'), | ||
92 | PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'), | ||
93 | BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url') | ||
94 | } | ||
95 | }, | ||
76 | WEBSERVER: { | 96 | WEBSERVER: { |
77 | SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', | 97 | SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', |
78 | WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', | 98 | WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5f121d9a4..8a1526ae8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
24 | 24 | ||
25 | // --------------------------------------------------------------------------- | 25 | // --------------------------------------------------------------------------- |
26 | 26 | ||
27 | const LAST_MIGRATION_VERSION = 655 | 27 | const LAST_MIGRATION_VERSION = 660 |
28 | 28 | ||
29 | // --------------------------------------------------------------------------- | 29 | // --------------------------------------------------------------------------- |
30 | 30 | ||
@@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
147 | 'videos-views': 1, | 147 | 'videos-views': 1, |
148 | 'activitypub-refresher': 1, | 148 | 'activitypub-refresher': 1, |
149 | 'video-redundancy': 1, | 149 | 'video-redundancy': 1, |
150 | 'video-live-ending': 1 | 150 | 'video-live-ending': 1, |
151 | 'move-to-object-storage': 3 | ||
151 | } | 152 | } |
152 | // Excluded keys are jobs that can be configured by admins | 153 | // Excluded keys are jobs that can be configured by admins |
153 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { | 154 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { |
@@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im | |||
162 | 'videos-views': 1, | 163 | 'videos-views': 1, |
163 | 'activitypub-refresher': 1, | 164 | 'activitypub-refresher': 1, |
164 | 'video-redundancy': 1, | 165 | 'video-redundancy': 1, |
165 | 'video-live-ending': 10 | 166 | 'video-live-ending': 10, |
167 | 'move-to-object-storage': 1 | ||
166 | } | 168 | } |
167 | const JOB_TTL: { [id in JobType]: number } = { | 169 | const JOB_TTL: { [id in JobType]: number } = { |
168 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | 170 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes |
@@ -178,7 +180,8 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
178 | 'videos-views': undefined, // Unlimited | 180 | 'videos-views': undefined, // Unlimited |
179 | 'activitypub-refresher': 60000 * 10, // 10 minutes | 181 | 'activitypub-refresher': 60000 * 10, // 10 minutes |
180 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours | 182 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours |
181 | 'video-live-ending': 1000 * 60 * 10 // 10 minutes | 183 | 'video-live-ending': 1000 * 60 * 10, // 10 minutes |
184 | 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours | ||
182 | } | 185 | } |
183 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { | 186 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { |
184 | 'videos-views': { | 187 | 'videos-views': { |
@@ -412,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = { | |||
412 | [VideoState.TO_TRANSCODE]: 'To transcode', | 415 | [VideoState.TO_TRANSCODE]: 'To transcode', |
413 | [VideoState.TO_IMPORT]: 'To import', | 416 | [VideoState.TO_IMPORT]: 'To import', |
414 | [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', | 417 | [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', |
415 | [VideoState.LIVE_ENDED]: 'Livestream ended' | 418 | [VideoState.LIVE_ENDED]: 'Livestream ended', |
419 | [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage' | ||
416 | } | 420 | } |
417 | 421 | ||
418 | const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { | 422 | const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 38e7a76d0..0e690f6ae 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -45,6 +45,7 @@ import { VideoTagModel } from '../models/video/video-tag' | |||
45 | import { VideoViewModel } from '../models/video/video-view' | 45 | import { VideoViewModel } from '../models/video/video-view' |
46 | import { CONFIG } from './config' | 46 | import { CONFIG } from './config' |
47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | 47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' |
48 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
48 | 49 | ||
49 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 50 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
50 | 51 | ||
@@ -143,7 +144,8 @@ async function initDatabaseModels (silent: boolean) { | |||
143 | TrackerModel, | 144 | TrackerModel, |
144 | VideoTrackerModel, | 145 | VideoTrackerModel, |
145 | PluginModel, | 146 | PluginModel, |
146 | ActorCustomPageModel | 147 | ActorCustomPageModel, |
148 | VideoJobInfoModel | ||
147 | ]) | 149 | ]) |
148 | 150 | ||
149 | // Check extensions exist in the database | 151 | // Check extensions exist in the database |
diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts index 1aeb27f2d..ac952a98c 100644 --- a/server/initializers/migrations/0065-video-file-size.ts +++ b/server/initializers/migrations/0065-video-file-size.ts | |||
@@ -1,7 +1,4 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import { stat } from 'fs-extra' | ||
3 | import { VideoModel } from '../../models/video/video' | ||
4 | import { getVideoFilePath } from '@server/lib/video-paths' | ||
5 | 2 | ||
6 | function up (utils: { | 3 | function up (utils: { |
7 | transaction: Sequelize.Transaction | 4 | transaction: Sequelize.Transaction |
@@ -9,30 +6,7 @@ function up (utils: { | |||
9 | sequelize: Sequelize.Sequelize | 6 | sequelize: Sequelize.Sequelize |
10 | db: any | 7 | db: any |
11 | }): Promise<void> { | 8 | }): Promise<void> { |
12 | return utils.db.Video.listOwnedAndPopulateAuthorAndTags() | 9 | throw new Error('Removed, please upgrade from a previous version first.') |
13 | .then((videos: VideoModel[]) => { | ||
14 | const tasks: Promise<any>[] = [] | ||
15 | |||
16 | videos.forEach(video => { | ||
17 | video.VideoFiles.forEach(videoFile => { | ||
18 | const p = new Promise((res, rej) => { | ||
19 | stat(getVideoFilePath(video, videoFile), (err, stats) => { | ||
20 | if (err) return rej(err) | ||
21 | |||
22 | videoFile.size = stats.size | ||
23 | videoFile.save().then(res).catch(rej) | ||
24 | }) | ||
25 | }) | ||
26 | |||
27 | tasks.push(p) | ||
28 | }) | ||
29 | }) | ||
30 | |||
31 | return tasks | ||
32 | }) | ||
33 | .then((tasks: Promise<any>[]) => { | ||
34 | return Promise.all(tasks) | ||
35 | }) | ||
36 | } | 10 | } |
37 | 11 | ||
38 | function down (options) { | 12 | function down (options) { |
diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts new file mode 100644 index 000000000..c815c71c6 --- /dev/null +++ b/server/initializers/migrations/0660-object-storage.ts | |||
@@ -0,0 +1,58 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { VideoStorage } from '@shared/models' | ||
3 | |||
4 | async function up (utils: { | ||
5 | transaction: Sequelize.Transaction | ||
6 | queryInterface: Sequelize.QueryInterface | ||
7 | sequelize: Sequelize.Sequelize | ||
8 | db: any | ||
9 | }): Promise<void> { | ||
10 | { | ||
11 | const query = ` | ||
12 | CREATE TABLE IF NOT EXISTS "videoJobInfo" ( | ||
13 | "id" serial, | ||
14 | "pendingMove" INTEGER NOT NULL, | ||
15 | "pendingTranscode" INTEGER NOT NULL, | ||
16 | "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
17 | "createdAt" timestamp WITH time zone NOT NULL, | ||
18 | "updatedAt" timestamp WITH time zone NOT NULL, | ||
19 | PRIMARY KEY ("id") | ||
20 | ); | ||
21 | ` | ||
22 | |||
23 | await utils.sequelize.query(query) | ||
24 | } | ||
25 | |||
26 | { | ||
27 | await utils.queryInterface.addColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: true }) | ||
28 | } | ||
29 | { | ||
30 | await utils.sequelize.query( | ||
31 | `UPDATE "videoFile" SET "storage" = ${VideoStorage.FILE_SYSTEM}` | ||
32 | ) | ||
33 | } | ||
34 | { | ||
35 | await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false }) | ||
36 | } | ||
37 | |||
38 | { | ||
39 | await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: true }) | ||
40 | } | ||
41 | { | ||
42 | await utils.sequelize.query( | ||
43 | `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.FILE_SYSTEM}` | ||
44 | ) | ||
45 | } | ||
46 | { | ||
47 | await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: false }) | ||
48 | } | ||
49 | } | ||
50 | |||
51 | function down (options) { | ||
52 | throw new Error('Not implemented.') | ||
53 | } | ||
54 | |||
55 | export { | ||
56 | up, | ||
57 | down | ||
58 | } | ||
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index 1fa16295d..bd9ed45a9 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -6,7 +6,7 @@ import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/vide | |||
6 | import { logger } from '@server/helpers/logger' | 6 | import { logger } from '@server/helpers/logger' |
7 | import { getExtFromMimetype } from '@server/helpers/video' | 7 | import { getExtFromMimetype } from '@server/helpers/video' |
8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' | 8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' |
9 | import { generateTorrentFileName } from '@server/lib/video-paths' | 9 | import { generateTorrentFileName } from '@server/lib/paths' |
10 | import { VideoCaptionModel } from '@server/models/video/video-caption' | 10 | import { VideoCaptionModel } from '@server/models/video/video-caption' |
11 | import { VideoFileModel } from '@server/models/video/video-file' | 11 | import { VideoFileModel } from '@server/models/video/video-file' |
12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 0e77ab9fa..0828a2d0f 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
2 | import { flatten, uniq } from 'lodash' | 2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | 3 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' | 4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' |
@@ -8,11 +8,12 @@ import { logger } from '../helpers/logger' | |||
8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | 8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
9 | import { generateRandomString } from '../helpers/utils' | 9 | import { generateRandomString } from '../helpers/utils' |
10 | import { CONFIG } from '../initializers/config' | 10 | import { CONFIG } from '../initializers/config' |
11 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' | 11 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' |
12 | import { sequelizeTypescript } from '../initializers/database' | 12 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { VideoFileModel } from '../models/video/video-file' | 13 | import { VideoFileModel } from '../models/video/video-file' |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
15 | import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' | 15 | import { getHlsResolutionPlaylistFilename } from './paths' |
16 | import { VideoPathManager } from './video-path-manager' | ||
16 | 17 | ||
17 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | 18 | async function updateStreamingPlaylistsInfohashesIfNeeded () { |
18 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | 19 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() |
@@ -31,75 +32,66 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { | |||
31 | } | 32 | } |
32 | 33 | ||
33 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 34 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
34 | const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
35 | |||
36 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | 35 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
37 | 36 | ||
38 | const masterPlaylistPath = join(directory, playlist.playlistFilename) | ||
39 | |||
40 | for (const file of playlist.VideoFiles) { | 37 | for (const file of playlist.VideoFiles) { |
41 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | 38 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) |
42 | 39 | ||
43 | // If we did not generated a playlist for this resolution, skip | 40 | await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => { |
44 | const filePlaylistPath = join(directory, playlistFilename) | 41 | const size = await getVideoStreamSize(videoFilePath) |
45 | if (await pathExists(filePlaylistPath) === false) continue | ||
46 | |||
47 | const videoFilePath = getVideoFilePath(playlist, file) | ||
48 | 42 | ||
49 | const size = await getVideoStreamSize(videoFilePath) | 43 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) |
44 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
50 | 45 | ||
51 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | 46 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` |
52 | const resolution = `RESOLUTION=${size.width}x${size.height}` | 47 | if (file.fps) line += ',FRAME-RATE=' + file.fps |
53 | 48 | ||
54 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | 49 | const codecs = await Promise.all([ |
55 | if (file.fps) line += ',FRAME-RATE=' + file.fps | 50 | getVideoStreamCodec(videoFilePath), |
51 | getAudioStreamCodec(videoFilePath) | ||
52 | ]) | ||
56 | 53 | ||
57 | const codecs = await Promise.all([ | 54 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
58 | getVideoStreamCodec(videoFilePath), | ||
59 | getAudioStreamCodec(videoFilePath) | ||
60 | ]) | ||
61 | 55 | ||
62 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` | 56 | masterPlaylists.push(line) |
63 | 57 | masterPlaylists.push(playlistFilename) | |
64 | masterPlaylists.push(line) | 58 | }) |
65 | masterPlaylists.push(playlistFilename) | ||
66 | } | 59 | } |
67 | 60 | ||
68 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 61 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { |
62 | return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
63 | }) | ||
69 | } | 64 | } |
70 | 65 | ||
71 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 66 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
72 | const json: { [filename: string]: { [range: string]: string } } = {} | 67 | const json: { [filename: string]: { [range: string]: string } } = {} |
73 | 68 | ||
74 | const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
75 | |||
76 | // For all the resolutions available for this video | 69 | // For all the resolutions available for this video |
77 | for (const file of playlist.VideoFiles) { | 70 | for (const file of playlist.VideoFiles) { |
78 | const rangeHashes: { [range: string]: string } = {} | 71 | const rangeHashes: { [range: string]: string } = {} |
79 | 72 | ||
80 | const videoPath = getVideoFilePath(playlist, file) | 73 | await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => { |
81 | const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename)) | ||
82 | |||
83 | // Maybe the playlist is not generated for this resolution yet | ||
84 | if (!await pathExists(resolutionPlaylistPath)) continue | ||
85 | 74 | ||
86 | const playlistContent = await readFile(resolutionPlaylistPath) | 75 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => { |
87 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | 76 | const playlistContent = await readFile(resolutionPlaylistPath) |
77 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
88 | 78 | ||
89 | const fd = await open(videoPath, 'r') | 79 | const fd = await open(videoPath, 'r') |
90 | for (const range of ranges) { | 80 | for (const range of ranges) { |
91 | const buf = Buffer.alloc(range.length) | 81 | const buf = Buffer.alloc(range.length) |
92 | await read(fd, buf, 0, range.length, range.offset) | 82 | await read(fd, buf, 0, range.length, range.offset) |
93 | 83 | ||
94 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | 84 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) |
95 | } | 85 | } |
96 | await close(fd) | 86 | await close(fd) |
97 | 87 | ||
98 | const videoFilename = file.filename | 88 | const videoFilename = file.filename |
99 | json[videoFilename] = rangeHashes | 89 | json[videoFilename] = rangeHashes |
90 | }) | ||
91 | }) | ||
100 | } | 92 | } |
101 | 93 | ||
102 | const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) | 94 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
103 | await outputJSON(outputPath, json) | 95 | await outputJSON(outputPath, json) |
104 | } | 96 | } |
105 | 97 | ||
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts new file mode 100644 index 000000000..a0c58d211 --- /dev/null +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -0,0 +1,114 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { remove } from 'fs-extra' | ||
3 | import { join } from 'path' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
6 | import { CONFIG } from '@server/initializers/config' | ||
7 | import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' | ||
8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
9 | import { moveToNextState } from '@server/lib/video-state' | ||
10 | import { VideoModel } from '@server/models/video/video' | ||
11 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
12 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' | ||
13 | import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared' | ||
14 | |||
15 | export async function processMoveToObjectStorage (job: Bull.Job) { | ||
16 | const payload = job.data as MoveObjectStoragePayload | ||
17 | logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) | ||
18 | |||
19 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | ||
20 | // No video, maybe deleted? | ||
21 | if (!video) { | ||
22 | logger.info('Can\'t process job %d, video does not exist.', job.id) | ||
23 | return undefined | ||
24 | } | ||
25 | |||
26 | if (video.VideoFiles) { | ||
27 | await moveWebTorrentFiles(video) | ||
28 | } | ||
29 | |||
30 | if (video.VideoStreamingPlaylists) { | ||
31 | await moveHLSFiles(video) | ||
32 | } | ||
33 | |||
34 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') | ||
35 | if (pendingMove === 0) { | ||
36 | logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) | ||
37 | await doAfterLastJob(video, payload.isNewVideo) | ||
38 | } | ||
39 | |||
40 | return payload.videoUUID | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | ||
46 | for (const file of video.VideoFiles) { | ||
47 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
48 | |||
49 | const fileUrl = await storeWebTorrentFile(file.filename) | ||
50 | |||
51 | const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) | ||
52 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | ||
53 | } | ||
54 | } | ||
55 | |||
56 | async function moveHLSFiles (video: MVideoWithAllFiles) { | ||
57 | for (const playlist of video.VideoStreamingPlaylists) { | ||
58 | |||
59 | for (const file of playlist.VideoFiles) { | ||
60 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
61 | |||
62 | // Resolution playlist | ||
63 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | ||
64 | await storeHLSFile(playlist, video, playlistFilename) | ||
65 | |||
66 | // Resolution fragmented file | ||
67 | const fileUrl = await storeHLSFile(playlist, video, file.filename) | ||
68 | |||
69 | const oldPath = join(getHLSDirectory(video), file.filename) | ||
70 | |||
71 | await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) | ||
72 | } | ||
73 | } | ||
74 | } | ||
75 | |||
76 | async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) { | ||
77 | for (const playlist of video.VideoStreamingPlaylists) { | ||
78 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue | ||
79 | |||
80 | // Master playlist | ||
81 | playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename) | ||
82 | // Sha256 segments file | ||
83 | playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename) | ||
84 | |||
85 | playlist.storage = VideoStorage.OBJECT_STORAGE | ||
86 | |||
87 | await playlist.save() | ||
88 | } | ||
89 | |||
90 | // Remove empty hls video directory | ||
91 | if (video.VideoStreamingPlaylists) { | ||
92 | await remove(getHLSDirectory(video)) | ||
93 | } | ||
94 | |||
95 | await moveToNextState(video, isNewVideo) | ||
96 | } | ||
97 | |||
98 | async function onFileMoved (options: { | ||
99 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo | ||
100 | file: MVideoFile | ||
101 | fileUrl: string | ||
102 | oldPath: string | ||
103 | }) { | ||
104 | const { videoOrPlaylist, file, fileUrl, oldPath } = options | ||
105 | |||
106 | file.fileUrl = fileUrl | ||
107 | file.storage = VideoStorage.OBJECT_STORAGE | ||
108 | |||
109 | await createTorrentAndSetInfoHash(videoOrPlaylist, file) | ||
110 | await file.save() | ||
111 | |||
112 | logger.debug('Removing %s because it\'s now on object storage', oldPath) | ||
113 | await remove(oldPath) | ||
114 | } | ||
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 2f4abf730..e8ee1f759 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -2,15 +2,19 @@ import * as Bull from 'bull' | |||
2 | import { copy, stat } from 'fs-extra' | 2 | import { copy, stat } from 'fs-extra' |
3 | import { getLowercaseExtension } from '@server/helpers/core-utils' | 3 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
5 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 5 | import { CONFIG } from '@server/initializers/config' |
6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
8 | import { addMoveToObjectStorageJob } from '@server/lib/video' | ||
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
6 | import { UserModel } from '@server/models/user/user' | 10 | import { UserModel } from '@server/models/user/user' |
7 | import { MVideoFullLight } from '@server/types/models' | 11 | import { MVideoFullLight } from '@server/types/models' |
8 | import { VideoFileImportPayload } from '@shared/models' | 12 | import { VideoFileImportPayload, VideoStorage } from '@shared/models' |
9 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' | 13 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' |
10 | import { logger } from '../../../helpers/logger' | 14 | import { logger } from '../../../helpers/logger' |
11 | import { VideoModel } from '../../../models/video/video' | 15 | import { VideoModel } from '../../../models/video/video' |
12 | import { VideoFileModel } from '../../../models/video/video-file' | 16 | import { VideoFileModel } from '../../../models/video/video-file' |
13 | import { onNewWebTorrentFileResolution } from './video-transcoding' | 17 | import { createHlsJobIfEnabled } from './video-transcoding' |
14 | 18 | ||
15 | async function processVideoFileImport (job: Bull.Job) { | 19 | async function processVideoFileImport (job: Bull.Job) { |
16 | const payload = job.data as VideoFileImportPayload | 20 | const payload = job.data as VideoFileImportPayload |
@@ -29,15 +33,19 @@ async function processVideoFileImport (job: Bull.Job) { | |||
29 | 33 | ||
30 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) | 34 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) |
31 | 35 | ||
32 | const newResolutionPayload = { | 36 | await createHlsJobIfEnabled(user, { |
33 | type: 'new-resolution-to-webtorrent' as 'new-resolution-to-webtorrent', | ||
34 | videoUUID: video.uuid, | 37 | videoUUID: video.uuid, |
35 | resolution: data.resolution, | 38 | resolution: data.resolution, |
36 | isPortraitMode: data.isPortraitMode, | 39 | isPortraitMode: data.isPortraitMode, |
37 | copyCodecs: false, | 40 | copyCodecs: true, |
38 | isNewVideo: false | 41 | isMaxQuality: false |
42 | }) | ||
43 | |||
44 | if (CONFIG.OBJECT_STORAGE.ENABLED) { | ||
45 | await addMoveToObjectStorageJob(video) | ||
46 | } else { | ||
47 | await federateVideoIfNeeded(video, false) | ||
39 | } | 48 | } |
40 | await onNewWebTorrentFileResolution(video, user, newResolutionPayload) | ||
41 | 49 | ||
42 | return video | 50 | return video |
43 | } | 51 | } |
@@ -72,12 +80,13 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
72 | resolution, | 80 | resolution, |
73 | extname: fileExt, | 81 | extname: fileExt, |
74 | filename: generateWebTorrentVideoFilename(resolution, fileExt), | 82 | filename: generateWebTorrentVideoFilename(resolution, fileExt), |
83 | storage: VideoStorage.FILE_SYSTEM, | ||
75 | size, | 84 | size, |
76 | fps, | 85 | fps, |
77 | videoId: video.id | 86 | videoId: video.id |
78 | }) | 87 | }) |
79 | 88 | ||
80 | const outputPath = getVideoFilePath(video, newVideoFile) | 89 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
81 | await copy(inputFilePath, outputPath) | 90 | await copy(inputFilePath, outputPath) |
82 | 91 | ||
83 | video.VideoFiles.push(newVideoFile) | 92 | video.VideoFiles.push(newVideoFile) |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index fec553f2b..a5fa204f5 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -4,11 +4,13 @@ import { getLowercaseExtension } from '@server/helpers/core-utils' | |||
4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | 4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
5 | import { YoutubeDL } from '@server/helpers/youtube-dl' | 5 | import { YoutubeDL } from '@server/helpers/youtube-dl' |
6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | 6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' |
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
7 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
8 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
9 | import { isAbleToUploadVideo } from '@server/lib/user' | 10 | import { isAbleToUploadVideo } from '@server/lib/user' |
10 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' | 11 | import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video' |
11 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 12 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { buildNextVideoState } from '@server/lib/video-state' | ||
12 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 14 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
13 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | 15 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' |
14 | import { | 16 | import { |
@@ -25,7 +27,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro | |||
25 | import { logger } from '../../../helpers/logger' | 27 | import { logger } from '../../../helpers/logger' |
26 | import { getSecureTorrentName } from '../../../helpers/utils' | 28 | import { getSecureTorrentName } from '../../../helpers/utils' |
27 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 29 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
28 | import { CONFIG } from '../../../initializers/config' | ||
29 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' | 30 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' |
30 | import { sequelizeTypescript } from '../../../initializers/database' | 31 | import { sequelizeTypescript } from '../../../initializers/database' |
31 | import { VideoModel } from '../../../models/video/video' | 32 | import { VideoModel } from '../../../models/video/video' |
@@ -100,7 +101,6 @@ type ProcessFileOptions = { | |||
100 | } | 101 | } |
101 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { | 102 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { |
102 | let tempVideoPath: string | 103 | let tempVideoPath: string |
103 | let videoDestFile: string | ||
104 | let videoFile: VideoFileModel | 104 | let videoFile: VideoFileModel |
105 | 105 | ||
106 | try { | 106 | try { |
@@ -159,7 +159,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
159 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) | 159 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) |
160 | 160 | ||
161 | // Move file | 161 | // Move file |
162 | videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) | 162 | const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) |
163 | await move(tempVideoPath, videoDestFile) | 163 | await move(tempVideoPath, videoDestFile) |
164 | tempVideoPath = null // This path is not used anymore | 164 | tempVideoPath = null // This path is not used anymore |
165 | 165 | ||
@@ -204,7 +204,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
204 | 204 | ||
205 | // Update video DB object | 205 | // Update video DB object |
206 | video.duration = duration | 206 | video.duration = duration |
207 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED | 207 | video.state = buildNextVideoState(video.state) |
208 | await video.save({ transaction: t }) | 208 | await video.save({ transaction: t }) |
209 | 209 | ||
210 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) | 210 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) |
@@ -245,6 +245,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
245 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 245 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) |
246 | } | 246 | } |
247 | 247 | ||
248 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
249 | return addMoveToObjectStorageJob(videoImportUpdated.Video) | ||
250 | } | ||
251 | |||
248 | // Create transcoding jobs? | 252 | // Create transcoding jobs? |
249 | if (video.state === VideoState.TO_TRANSCODE) { | 253 | if (video.state === VideoState.TO_TRANSCODE) { |
250 | await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) | 254 | await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) |
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index aa5bd573a..9ccf724c2 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -4,10 +4,11 @@ import { join } from 'path' | |||
4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | 4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' |
5 | import { VIDEO_LIVE } from '@server/initializers/constants' | 5 | import { VIDEO_LIVE } from '@server/initializers/constants' |
6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' | 6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' |
7 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory } from '@server/lib/paths' | ||
7 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 8 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
8 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' | 9 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' |
9 | import { publishAndFederateIfNeeded } from '@server/lib/video' | 10 | import { VideoPathManager } from '@server/lib/video-path-manager' |
10 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' | 11 | import { moveToNextState } from '@server/lib/video-state' |
11 | import { VideoModel } from '@server/models/video/video' | 12 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 13 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { VideoLiveModel } from '@server/models/video/video-live' | 14 | import { VideoLiveModel } from '@server/models/video/video-live' |
@@ -55,16 +56,15 @@ export { | |||
55 | // --------------------------------------------------------------------------- | 56 | // --------------------------------------------------------------------------- |
56 | 57 | ||
57 | async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { | 58 | async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { |
58 | const hlsDirectory = getHLSDirectory(video, false) | 59 | const replayDirectory = VideoPathManager.Instance.getFSHLSOutputPath(video, VIDEO_LIVE.REPLAY_DIRECTORY) |
59 | const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
60 | 60 | ||
61 | const rootFiles = await readdir(hlsDirectory) | 61 | const rootFiles = await readdir(getLiveDirectory(video)) |
62 | 62 | ||
63 | const playlistFiles = rootFiles.filter(file => { | 63 | const playlistFiles = rootFiles.filter(file => { |
64 | return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename | 64 | return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename |
65 | }) | 65 | }) |
66 | 66 | ||
67 | await cleanupLiveFiles(hlsDirectory) | 67 | await cleanupTMPLiveFiles(getLiveDirectory(video)) |
68 | 68 | ||
69 | await live.destroy() | 69 | await live.destroy() |
70 | 70 | ||
@@ -98,7 +98,7 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt | |||
98 | 98 | ||
99 | const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) | 99 | const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) |
100 | 100 | ||
101 | const outputPath = await generateHlsPlaylistResolutionFromTS({ | 101 | const { resolutionPlaylistPath: outputPath } = await generateHlsPlaylistResolutionFromTS({ |
102 | video: videoWithFiles, | 102 | video: videoWithFiles, |
103 | concatenatedTsFilePath, | 103 | concatenatedTsFilePath, |
104 | resolution, | 104 | resolution, |
@@ -133,10 +133,10 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt | |||
133 | }) | 133 | }) |
134 | } | 134 | } |
135 | 135 | ||
136 | await publishAndFederateIfNeeded(videoWithFiles, true) | 136 | await moveToNextState(videoWithFiles, false) |
137 | } | 137 | } |
138 | 138 | ||
139 | async function cleanupLiveFiles (hlsDirectory: string) { | 139 | async function cleanupTMPLiveFiles (hlsDirectory: string) { |
140 | if (!await pathExists(hlsDirectory)) return | 140 | if (!await pathExists(hlsDirectory)) return |
141 | 141 | ||
142 | const files = await readdir(hlsDirectory) | 142 | const files = await readdir(hlsDirectory) |
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 876d1460c..b3149dde8 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -1,9 +1,11 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' | 2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' |
3 | import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' | 3 | import { addTranscodingJob, getTranscodingJobPriority } from '@server/lib/video' |
4 | import { getVideoFilePath } from '@server/lib/video-paths' | 4 | import { VideoPathManager } from '@server/lib/video-path-manager' |
5 | import { moveToNextState } from '@server/lib/video-state' | ||
5 | import { UserModel } from '@server/models/user/user' | 6 | import { UserModel } from '@server/models/user/user' |
6 | import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' | 7 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
8 | import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models' | ||
7 | import { | 9 | import { |
8 | HLSTranscodingPayload, | 10 | HLSTranscodingPayload, |
9 | MergeAudioTranscodingPayload, | 11 | MergeAudioTranscodingPayload, |
@@ -16,17 +18,14 @@ import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' | |||
16 | import { logger } from '../../../helpers/logger' | 18 | import { logger } from '../../../helpers/logger' |
17 | import { CONFIG } from '../../../initializers/config' | 19 | import { CONFIG } from '../../../initializers/config' |
18 | import { VideoModel } from '../../../models/video/video' | 20 | import { VideoModel } from '../../../models/video/video' |
19 | import { federateVideoIfNeeded } from '../../activitypub/videos' | ||
20 | import { Notifier } from '../../notifier' | ||
21 | import { | 21 | import { |
22 | generateHlsPlaylistResolution, | 22 | generateHlsPlaylistResolution, |
23 | mergeAudioVideofile, | 23 | mergeAudioVideofile, |
24 | optimizeOriginalVideofile, | 24 | optimizeOriginalVideofile, |
25 | transcodeNewWebTorrentResolution | 25 | transcodeNewWebTorrentResolution |
26 | } from '../../transcoding/video-transcoding' | 26 | } from '../../transcoding/video-transcoding' |
27 | import { JobQueue } from '../job-queue' | ||
28 | 27 | ||
29 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> | 28 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void> |
30 | 29 | ||
31 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { | 30 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { |
32 | 'new-resolution-to-hls': handleHLSJob, | 31 | 'new-resolution-to-hls': handleHLSJob, |
@@ -69,15 +68,16 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide | |||
69 | : video.getMaxQualityFile() | 68 | : video.getMaxQualityFile() |
70 | 69 | ||
71 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | 70 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() |
72 | const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput) | ||
73 | 71 | ||
74 | await generateHlsPlaylistResolution({ | 72 | await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => { |
75 | video, | 73 | return generateHlsPlaylistResolution({ |
76 | videoInputPath, | 74 | video, |
77 | resolution: payload.resolution, | 75 | videoInputPath, |
78 | copyCodecs: payload.copyCodecs, | 76 | resolution: payload.resolution, |
79 | isPortraitMode: payload.isPortraitMode || false, | 77 | copyCodecs: payload.copyCodecs, |
80 | job | 78 | isPortraitMode: payload.isPortraitMode || false, |
79 | job | ||
80 | }) | ||
81 | }) | 81 | }) |
82 | 82 | ||
83 | await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) | 83 | await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) |
@@ -101,7 +101,7 @@ async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudio | |||
101 | } | 101 | } |
102 | 102 | ||
103 | async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 103 | async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
104 | const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) | 104 | const { transcodeType } = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) |
105 | 105 | ||
106 | await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) | 106 | await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) |
107 | } | 107 | } |
@@ -121,10 +121,18 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay | |||
121 | video.VideoFiles = [] | 121 | video.VideoFiles = [] |
122 | 122 | ||
123 | // Create HLS new resolution jobs | 123 | // Create HLS new resolution jobs |
124 | await createLowerResolutionsJobs(video, user, payload.resolution, payload.isPortraitMode, 'hls') | 124 | await createLowerResolutionsJobs({ |
125 | video, | ||
126 | user, | ||
127 | videoFileResolution: payload.resolution, | ||
128 | isPortraitMode: payload.isPortraitMode, | ||
129 | isNewVideo: payload.isNewVideo ?? true, | ||
130 | type: 'hls' | ||
131 | }) | ||
125 | } | 132 | } |
126 | 133 | ||
127 | return publishAndFederateIfNeeded(video) | 134 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') |
135 | await moveToNextState(video, payload.isNewVideo) | ||
128 | } | 136 | } |
129 | 137 | ||
130 | async function onVideoFileOptimizer ( | 138 | async function onVideoFileOptimizer ( |
@@ -143,58 +151,54 @@ async function onVideoFileOptimizer ( | |||
143 | // Video does not exist anymore | 151 | // Video does not exist anymore |
144 | if (!videoDatabase) return undefined | 152 | if (!videoDatabase) return undefined |
145 | 153 | ||
146 | let videoPublished = false | ||
147 | |||
148 | // Generate HLS version of the original file | 154 | // Generate HLS version of the original file |
149 | const originalFileHLSPayload = Object.assign({}, payload, { | 155 | const originalFileHLSPayload = { |
156 | ...payload, | ||
157 | |||
150 | isPortraitMode, | 158 | isPortraitMode, |
151 | resolution: videoDatabase.getMaxQualityFile().resolution, | 159 | resolution: videoDatabase.getMaxQualityFile().resolution, |
152 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | 160 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues |
153 | copyCodecs: transcodeType !== 'quick-transcode', | 161 | copyCodecs: transcodeType !== 'quick-transcode', |
154 | isMaxQuality: true | 162 | isMaxQuality: true |
155 | }) | 163 | } |
156 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | 164 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) |
165 | const hasNewResolutions = await createLowerResolutionsJobs({ | ||
166 | video: videoDatabase, | ||
167 | user, | ||
168 | videoFileResolution: resolution, | ||
169 | isPortraitMode, | ||
170 | type: 'webtorrent', | ||
171 | isNewVideo: payload.isNewVideo ?? true | ||
172 | }) | ||
157 | 173 | ||
158 | const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, resolution, isPortraitMode, 'webtorrent') | 174 | await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') |
159 | 175 | ||
176 | // Move to next state if there are no other resolutions to generate | ||
160 | if (!hasHls && !hasNewResolutions) { | 177 | if (!hasHls && !hasNewResolutions) { |
161 | // No transcoding to do, it's now published | 178 | await moveToNextState(videoDatabase, payload.isNewVideo) |
162 | videoPublished = await videoDatabase.publishIfNeededAndSave(undefined) | ||
163 | } | 179 | } |
164 | |||
165 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo) | ||
166 | |||
167 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) | ||
168 | if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) | ||
169 | } | 180 | } |
170 | 181 | ||
171 | async function onNewWebTorrentFileResolution ( | 182 | async function onNewWebTorrentFileResolution ( |
172 | video: MVideoUUID, | 183 | video: MVideo, |
173 | user: MUserId, | 184 | user: MUserId, |
174 | payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload | 185 | payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload |
175 | ) { | 186 | ) { |
176 | await publishAndFederateIfNeeded(video) | 187 | await createHlsJobIfEnabled(user, { ...payload, copyCodecs: true, isMaxQuality: false }) |
188 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
177 | 189 | ||
178 | await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true, isMaxQuality: false })) | 190 | await moveToNextState(video, payload.isNewVideo) |
179 | } | 191 | } |
180 | 192 | ||
181 | // --------------------------------------------------------------------------- | ||
182 | |||
183 | export { | ||
184 | processVideoTranscoding, | ||
185 | onNewWebTorrentFileResolution | ||
186 | } | ||
187 | |||
188 | // --------------------------------------------------------------------------- | ||
189 | |||
190 | async function createHlsJobIfEnabled (user: MUserId, payload: { | 193 | async function createHlsJobIfEnabled (user: MUserId, payload: { |
191 | videoUUID: string | 194 | videoUUID: string |
192 | resolution: number | 195 | resolution: number |
193 | isPortraitMode?: boolean | 196 | isPortraitMode?: boolean |
194 | copyCodecs: boolean | 197 | copyCodecs: boolean |
195 | isMaxQuality: boolean | 198 | isMaxQuality: boolean |
199 | isNewVideo?: boolean | ||
196 | }) { | 200 | }) { |
197 | if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false | 201 | if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false |
198 | 202 | ||
199 | const jobOptions = { | 203 | const jobOptions = { |
200 | priority: await getTranscodingJobPriority(user) | 204 | priority: await getTranscodingJobPriority(user) |
@@ -206,21 +210,35 @@ async function createHlsJobIfEnabled (user: MUserId, payload: { | |||
206 | resolution: payload.resolution, | 210 | resolution: payload.resolution, |
207 | isPortraitMode: payload.isPortraitMode, | 211 | isPortraitMode: payload.isPortraitMode, |
208 | copyCodecs: payload.copyCodecs, | 212 | copyCodecs: payload.copyCodecs, |
209 | isMaxQuality: payload.isMaxQuality | 213 | isMaxQuality: payload.isMaxQuality, |
214 | isNewVideo: payload.isNewVideo | ||
210 | } | 215 | } |
211 | 216 | ||
212 | JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions) | 217 | await addTranscodingJob(hlsTranscodingPayload, jobOptions) |
213 | 218 | ||
214 | return true | 219 | return true |
215 | } | 220 | } |
216 | 221 | ||
217 | async function createLowerResolutionsJobs ( | 222 | // --------------------------------------------------------------------------- |
218 | video: MVideoFullLight, | 223 | |
219 | user: MUserId, | 224 | export { |
220 | videoFileResolution: number, | 225 | processVideoTranscoding, |
221 | isPortraitMode: boolean, | 226 | createHlsJobIfEnabled, |
227 | onNewWebTorrentFileResolution | ||
228 | } | ||
229 | |||
230 | // --------------------------------------------------------------------------- | ||
231 | |||
232 | async function createLowerResolutionsJobs (options: { | ||
233 | video: MVideoFullLight | ||
234 | user: MUserId | ||
235 | videoFileResolution: number | ||
236 | isPortraitMode: boolean | ||
237 | isNewVideo: boolean | ||
222 | type: 'hls' | 'webtorrent' | 238 | type: 'hls' | 'webtorrent' |
223 | ) { | 239 | }) { |
240 | const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options | ||
241 | |||
224 | // Create transcoding jobs if there are enabled resolutions | 242 | // Create transcoding jobs if there are enabled resolutions |
225 | const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') | 243 | const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') |
226 | const resolutionCreated: number[] = [] | 244 | const resolutionCreated: number[] = [] |
@@ -234,7 +252,8 @@ async function createLowerResolutionsJobs ( | |||
234 | type: 'new-resolution-to-webtorrent', | 252 | type: 'new-resolution-to-webtorrent', |
235 | videoUUID: video.uuid, | 253 | videoUUID: video.uuid, |
236 | resolution, | 254 | resolution, |
237 | isPortraitMode | 255 | isPortraitMode, |
256 | isNewVideo | ||
238 | } | 257 | } |
239 | } | 258 | } |
240 | 259 | ||
@@ -245,7 +264,8 @@ async function createLowerResolutionsJobs ( | |||
245 | resolution, | 264 | resolution, |
246 | isPortraitMode, | 265 | isPortraitMode, |
247 | copyCodecs: false, | 266 | copyCodecs: false, |
248 | isMaxQuality: false | 267 | isMaxQuality: false, |
268 | isNewVideo | ||
249 | } | 269 | } |
250 | } | 270 | } |
251 | 271 | ||
@@ -257,7 +277,7 @@ async function createLowerResolutionsJobs ( | |||
257 | priority: await getTranscodingJobPriority(user) | 277 | priority: await getTranscodingJobPriority(user) |
258 | } | 278 | } |
259 | 279 | ||
260 | JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions) | 280 | await addTranscodingJob(dataInput, jobOptions) |
261 | } | 281 | } |
262 | 282 | ||
263 | if (resolutionCreated.length === 0) { | 283 | if (resolutionCreated.length === 0) { |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 42e8347b1..7a3a1bf82 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -11,6 +11,7 @@ import { | |||
11 | EmailPayload, | 11 | EmailPayload, |
12 | JobState, | 12 | JobState, |
13 | JobType, | 13 | JobType, |
14 | MoveObjectStoragePayload, | ||
14 | RefreshPayload, | 15 | RefreshPayload, |
15 | VideoFileImportPayload, | 16 | VideoFileImportPayload, |
16 | VideoImportPayload, | 17 | VideoImportPayload, |
@@ -34,6 +35,7 @@ import { processVideoImport } from './handlers/video-import' | |||
34 | import { processVideoLiveEnding } from './handlers/video-live-ending' | 35 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
35 | import { processVideoTranscoding } from './handlers/video-transcoding' | 36 | import { processVideoTranscoding } from './handlers/video-transcoding' |
36 | import { processVideosViews } from './handlers/video-views' | 37 | import { processVideosViews } from './handlers/video-views' |
38 | import { processMoveToObjectStorage } from './handlers/move-to-object-storage' | ||
37 | 39 | ||
38 | type CreateJobArgument = | 40 | type CreateJobArgument = |
39 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 41 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -49,9 +51,10 @@ type CreateJobArgument = | |||
49 | { type: 'videos-views', payload: {} } | | 51 | { type: 'videos-views', payload: {} } | |
50 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | | 52 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | |
51 | { type: 'actor-keys', payload: ActorKeysPayload } | | 53 | { type: 'actor-keys', payload: ActorKeysPayload } | |
52 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | 54 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | |
55 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | ||
53 | 56 | ||
54 | type CreateJobOptions = { | 57 | export type CreateJobOptions = { |
55 | delay?: number | 58 | delay?: number |
56 | priority?: number | 59 | priority?: number |
57 | } | 60 | } |
@@ -70,7 +73,8 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = { | |||
70 | 'activitypub-refresher': refreshAPObject, | 73 | 'activitypub-refresher': refreshAPObject, |
71 | 'video-live-ending': processVideoLiveEnding, | 74 | 'video-live-ending': processVideoLiveEnding, |
72 | 'actor-keys': processActorKeys, | 75 | 'actor-keys': processActorKeys, |
73 | 'video-redundancy': processVideoRedundancy | 76 | 'video-redundancy': processVideoRedundancy, |
77 | 'move-to-object-storage': processMoveToObjectStorage | ||
74 | } | 78 | } |
75 | 79 | ||
76 | const jobTypes: JobType[] = [ | 80 | const jobTypes: JobType[] = [ |
@@ -87,7 +91,8 @@ const jobTypes: JobType[] = [ | |||
87 | 'activitypub-refresher', | 91 | 'activitypub-refresher', |
88 | 'video-redundancy', | 92 | 'video-redundancy', |
89 | 'actor-keys', | 93 | 'actor-keys', |
90 | 'video-live-ending' | 94 | 'video-live-ending', |
95 | 'move-to-object-storage' | ||
91 | ] | 96 | ] |
92 | 97 | ||
93 | class JobQueue { | 98 | class JobQueue { |
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 2a429fb33..d7dc841d9 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts | |||
@@ -20,7 +20,7 @@ import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | |||
20 | import { federateVideoIfNeeded } from '../activitypub/videos' | 20 | import { federateVideoIfNeeded } from '../activitypub/videos' |
21 | import { JobQueue } from '../job-queue' | 21 | import { JobQueue } from '../job-queue' |
22 | import { PeerTubeSocket } from '../peertube-socket' | 22 | import { PeerTubeSocket } from '../peertube-socket' |
23 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' | 23 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../paths' |
24 | import { LiveQuotaStore } from './live-quota-store' | 24 | import { LiveQuotaStore } from './live-quota-store' |
25 | import { LiveSegmentShaStore } from './live-segment-sha-store' | 25 | import { LiveSegmentShaStore } from './live-segment-sha-store' |
26 | import { cleanupLive } from './live-utils' | 26 | import { cleanupLive } from './live-utils' |
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index e4526c7a5..3bf723b98 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
2 | import { basename } from 'path' | 2 | import { basename } from 'path' |
3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
4 | import { getHLSDirectory } from '../video-paths' | 4 | import { getLiveDirectory } from '../paths' |
5 | 5 | ||
6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | 6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { |
7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | 7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) |
@@ -10,7 +10,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) { | |||
10 | } | 10 | } |
11 | 11 | ||
12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
13 | const hlsDirectory = getHLSDirectory(video) | 13 | const hlsDirectory = getLiveDirectory(video) |
14 | 14 | ||
15 | await remove(hlsDirectory) | 15 | await remove(hlsDirectory) |
16 | 16 | ||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index a80abc843..9b5b6c4fc 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -11,9 +11,9 @@ import { CONFIG } from '@server/initializers/config' | |||
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | 11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 12 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | 13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' |
14 | import { getLiveDirectory } from '../../paths' | ||
14 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' | 15 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' |
15 | import { isAbleToUploadVideo } from '../../user' | 16 | import { isAbleToUploadVideo } from '../../user' |
16 | import { getHLSDirectory } from '../../video-paths' | ||
17 | import { LiveQuotaStore } from '../live-quota-store' | 17 | import { LiveQuotaStore } from '../live-quota-store' |
18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | 18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' |
19 | import { buildConcatenatedName } from '../live-utils' | 19 | import { buildConcatenatedName } from '../live-utils' |
@@ -282,7 +282,7 @@ class MuxingSession extends EventEmitter { | |||
282 | } | 282 | } |
283 | 283 | ||
284 | private async prepareDirectories () { | 284 | private async prepareDirectories () { |
285 | const outPath = getHLSDirectory(this.videoLive.Video) | 285 | const outPath = getLiveDirectory(this.videoLive.Video) |
286 | await ensureDir(outPath) | 286 | await ensureDir(outPath) |
287 | 287 | ||
288 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | 288 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) |
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts new file mode 100644 index 000000000..8b413a40e --- /dev/null +++ b/server/lib/object-storage/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './keys' | ||
2 | export * from './urls' | ||
3 | export * from './videos' | ||
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts new file mode 100644 index 000000000..519474775 --- /dev/null +++ b/server/lib/object-storage/keys.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import { join } from 'path' | ||
2 | import { MStreamingPlaylist, MVideoUUID } from '@server/types/models' | ||
3 | |||
4 | function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { | ||
5 | return join(generateHLSObjectBaseStorageKey(playlist, video), filename) | ||
6 | } | ||
7 | |||
8 | function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { | ||
9 | return playlist.getStringType() + '_' + video.uuid | ||
10 | } | ||
11 | |||
12 | function generateWebTorrentObjectStorageKey (filename: string) { | ||
13 | return filename | ||
14 | } | ||
15 | |||
16 | export { | ||
17 | generateHLSObjectStorageKey, | ||
18 | generateHLSObjectBaseStorageKey, | ||
19 | generateWebTorrentObjectStorageKey | ||
20 | } | ||
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts new file mode 100644 index 000000000..c9a614593 --- /dev/null +++ b/server/lib/object-storage/shared/client.ts | |||
@@ -0,0 +1,56 @@ | |||
1 | import { S3Client } from '@aws-sdk/client-s3' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { lTags } from './logger' | ||
5 | |||
6 | let endpointParsed: URL | ||
7 | function getEndpointParsed () { | ||
8 | if (endpointParsed) return endpointParsed | ||
9 | |||
10 | endpointParsed = new URL(getEndpoint()) | ||
11 | |||
12 | return endpointParsed | ||
13 | } | ||
14 | |||
15 | let s3Client: S3Client | ||
16 | function getClient () { | ||
17 | if (s3Client) return s3Client | ||
18 | |||
19 | const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE | ||
20 | |||
21 | s3Client = new S3Client({ | ||
22 | endpoint: getEndpoint(), | ||
23 | region: OBJECT_STORAGE.REGION, | ||
24 | credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID | ||
25 | ? { | ||
26 | accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, | ||
27 | secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY | ||
28 | } | ||
29 | : undefined | ||
30 | }) | ||
31 | |||
32 | logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) | ||
33 | |||
34 | return s3Client | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | export { | ||
40 | getEndpointParsed, | ||
41 | getClient | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | let endpoint: string | ||
47 | function getEndpoint () { | ||
48 | if (endpoint) return endpoint | ||
49 | |||
50 | const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT | ||
51 | endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') | ||
52 | ? CONFIG.OBJECT_STORAGE.ENDPOINT | ||
53 | : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT | ||
54 | |||
55 | return endpoint | ||
56 | } | ||
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts new file mode 100644 index 000000000..11e10aa9f --- /dev/null +++ b/server/lib/object-storage/shared/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './client' | ||
2 | export * from './logger' | ||
3 | export * from './object-storage-helpers' | ||
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts new file mode 100644 index 000000000..8ab7cbd71 --- /dev/null +++ b/server/lib/object-storage/shared/logger.ts | |||
@@ -0,0 +1,7 @@ | |||
1 | import { loggerTagsFactory } from '@server/helpers/logger' | ||
2 | |||
3 | const lTags = loggerTagsFactory('object-storage') | ||
4 | |||
5 | export { | ||
6 | lTags | ||
7 | } | ||
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts new file mode 100644 index 000000000..e23216907 --- /dev/null +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -0,0 +1,229 @@ | |||
1 | import { close, createReadStream, createWriteStream, ensureDir, open, ReadStream, stat } from 'fs-extra' | ||
2 | import { min } from 'lodash' | ||
3 | import { dirname } from 'path' | ||
4 | import { Readable } from 'stream' | ||
5 | import { | ||
6 | CompletedPart, | ||
7 | CompleteMultipartUploadCommand, | ||
8 | CreateMultipartUploadCommand, | ||
9 | DeleteObjectCommand, | ||
10 | GetObjectCommand, | ||
11 | ListObjectsV2Command, | ||
12 | PutObjectCommand, | ||
13 | UploadPartCommand | ||
14 | } from '@aws-sdk/client-s3' | ||
15 | import { pipelinePromise } from '@server/helpers/core-utils' | ||
16 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
17 | import { logger } from '@server/helpers/logger' | ||
18 | import { CONFIG } from '@server/initializers/config' | ||
19 | import { getPrivateUrl } from '../urls' | ||
20 | import { getClient } from './client' | ||
21 | import { lTags } from './logger' | ||
22 | |||
23 | type BucketInfo = { | ||
24 | BUCKET_NAME: string | ||
25 | PREFIX?: string | ||
26 | } | ||
27 | |||
28 | async function storeObject (options: { | ||
29 | inputPath: string | ||
30 | objectStorageKey: string | ||
31 | bucketInfo: BucketInfo | ||
32 | }): Promise<string> { | ||
33 | const { inputPath, objectStorageKey, bucketInfo } = options | ||
34 | |||
35 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | ||
36 | |||
37 | const stats = await stat(inputPath) | ||
38 | |||
39 | // If bigger than max allowed size we do a multipart upload | ||
40 | if (stats.size > CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART) { | ||
41 | return multiPartUpload({ inputPath, objectStorageKey, bucketInfo }) | ||
42 | } | ||
43 | |||
44 | const fileStream = createReadStream(inputPath) | ||
45 | return objectStoragePut({ objectStorageKey, content: fileStream, bucketInfo }) | ||
46 | } | ||
47 | |||
48 | async function removeObject (filename: string, bucketInfo: BucketInfo) { | ||
49 | const command = new DeleteObjectCommand({ | ||
50 | Bucket: bucketInfo.BUCKET_NAME, | ||
51 | Key: buildKey(filename, bucketInfo) | ||
52 | }) | ||
53 | |||
54 | return getClient().send(command) | ||
55 | } | ||
56 | |||
57 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | ||
58 | const s3Client = getClient() | ||
59 | |||
60 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
61 | const listCommand = new ListObjectsV2Command({ | ||
62 | Bucket: bucketInfo.BUCKET_NAME, | ||
63 | Prefix: commandPrefix | ||
64 | }) | ||
65 | |||
66 | const listedObjects = await s3Client.send(listCommand) | ||
67 | |||
68 | // FIXME: use bulk delete when s3ninja will support this operation | ||
69 | // const deleteParams = { | ||
70 | // Bucket: bucketInfo.BUCKET_NAME, | ||
71 | // Delete: { Objects: [] } | ||
72 | // } | ||
73 | |||
74 | if (isArray(listedObjects.Contents) !== true) { | ||
75 | const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
76 | |||
77 | logger.error(message, { response: listedObjects, ...lTags() }) | ||
78 | throw new Error(message) | ||
79 | } | ||
80 | |||
81 | for (const object of listedObjects.Contents) { | ||
82 | const command = new DeleteObjectCommand({ | ||
83 | Bucket: bucketInfo.BUCKET_NAME, | ||
84 | Key: object.Key | ||
85 | }) | ||
86 | |||
87 | await s3Client.send(command) | ||
88 | |||
89 | // FIXME: use bulk delete when s3ninja will support this operation | ||
90 | // deleteParams.Delete.Objects.push({ Key: object.Key }) | ||
91 | } | ||
92 | |||
93 | // FIXME: use bulk delete when s3ninja will support this operation | ||
94 | // const deleteCommand = new DeleteObjectsCommand(deleteParams) | ||
95 | // await s3Client.send(deleteCommand) | ||
96 | |||
97 | // Repeat if not all objects could be listed at once (limit of 1000?) | ||
98 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) | ||
99 | } | ||
100 | |||
101 | async function makeAvailable (options: { | ||
102 | key: string | ||
103 | destination: string | ||
104 | bucketInfo: BucketInfo | ||
105 | }) { | ||
106 | const { key, destination, bucketInfo } = options | ||
107 | |||
108 | await ensureDir(dirname(options.destination)) | ||
109 | |||
110 | const command = new GetObjectCommand({ | ||
111 | Bucket: bucketInfo.BUCKET_NAME, | ||
112 | Key: buildKey(key, bucketInfo) | ||
113 | }) | ||
114 | const response = await getClient().send(command) | ||
115 | |||
116 | const file = createWriteStream(destination) | ||
117 | await pipelinePromise(response.Body as Readable, file) | ||
118 | |||
119 | file.close() | ||
120 | } | ||
121 | |||
122 | function buildKey (key: string, bucketInfo: BucketInfo) { | ||
123 | return bucketInfo.PREFIX + key | ||
124 | } | ||
125 | |||
126 | // --------------------------------------------------------------------------- | ||
127 | |||
128 | export { | ||
129 | BucketInfo, | ||
130 | buildKey, | ||
131 | storeObject, | ||
132 | removeObject, | ||
133 | removePrefix, | ||
134 | makeAvailable | ||
135 | } | ||
136 | |||
137 | // --------------------------------------------------------------------------- | ||
138 | |||
139 | async function objectStoragePut (options: { | ||
140 | objectStorageKey: string | ||
141 | content: ReadStream | ||
142 | bucketInfo: BucketInfo | ||
143 | }) { | ||
144 | const { objectStorageKey, content, bucketInfo } = options | ||
145 | |||
146 | const command = new PutObjectCommand({ | ||
147 | Bucket: bucketInfo.BUCKET_NAME, | ||
148 | Key: buildKey(objectStorageKey, bucketInfo), | ||
149 | Body: content | ||
150 | }) | ||
151 | |||
152 | await getClient().send(command) | ||
153 | |||
154 | return getPrivateUrl(bucketInfo, objectStorageKey) | ||
155 | } | ||
156 | |||
157 | async function multiPartUpload (options: { | ||
158 | inputPath: string | ||
159 | objectStorageKey: string | ||
160 | bucketInfo: BucketInfo | ||
161 | }) { | ||
162 | const { objectStorageKey, inputPath, bucketInfo } = options | ||
163 | |||
164 | const key = buildKey(objectStorageKey, bucketInfo) | ||
165 | const s3Client = getClient() | ||
166 | |||
167 | const statResult = await stat(inputPath) | ||
168 | |||
169 | const createMultipartCommand = new CreateMultipartUploadCommand({ | ||
170 | Bucket: bucketInfo.BUCKET_NAME, | ||
171 | Key: key | ||
172 | }) | ||
173 | const createResponse = await s3Client.send(createMultipartCommand) | ||
174 | |||
175 | const fd = await open(inputPath, 'r') | ||
176 | let partNumber = 1 | ||
177 | const parts: CompletedPart[] = [] | ||
178 | const partSize = CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART | ||
179 | |||
180 | for (let start = 0; start < statResult.size; start += partSize) { | ||
181 | logger.debug( | ||
182 | 'Uploading part %d of file to %s%s in bucket %s', | ||
183 | partNumber, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() | ||
184 | ) | ||
185 | |||
186 | // FIXME: Remove when https://github.com/aws/aws-sdk-js-v3/pull/2637 is released | ||
187 | // The s3 sdk needs to know the length of the http body beforehand, but doesn't support | ||
188 | // streams with start and end set, so it just tries to stat the file in stream.path. | ||
189 | // This fails for us because we only want to send part of the file. The stream type | ||
190 | // is modified so we can set the byteLength here, which s3 detects because array buffers | ||
191 | // have this field set | ||
192 | const stream: ReadStream & { byteLength: number } = | ||
193 | createReadStream( | ||
194 | inputPath, | ||
195 | { fd, autoClose: false, start, end: (start + partSize) - 1 } | ||
196 | ) as ReadStream & { byteLength: number } | ||
197 | |||
198 | // Calculate if the part size is more than what's left over, and in that case use left over bytes for byteLength | ||
199 | stream.byteLength = min([ statResult.size - start, partSize ]) | ||
200 | |||
201 | const uploadPartCommand = new UploadPartCommand({ | ||
202 | Bucket: bucketInfo.BUCKET_NAME, | ||
203 | Key: key, | ||
204 | UploadId: createResponse.UploadId, | ||
205 | PartNumber: partNumber, | ||
206 | Body: stream | ||
207 | }) | ||
208 | const uploadResponse = await s3Client.send(uploadPartCommand) | ||
209 | |||
210 | parts.push({ ETag: uploadResponse.ETag, PartNumber: partNumber }) | ||
211 | partNumber += 1 | ||
212 | } | ||
213 | await close(fd) | ||
214 | |||
215 | const completeUploadCommand = new CompleteMultipartUploadCommand({ | ||
216 | Bucket: bucketInfo.BUCKET_NAME, | ||
217 | Key: objectStorageKey, | ||
218 | UploadId: createResponse.UploadId, | ||
219 | MultipartUpload: { Parts: parts } | ||
220 | }) | ||
221 | await s3Client.send(completeUploadCommand) | ||
222 | |||
223 | logger.debug( | ||
224 | 'Completed %s%s in bucket %s in %d parts', | ||
225 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, partNumber - 1, lTags() | ||
226 | ) | ||
227 | |||
228 | return getPrivateUrl(bucketInfo, objectStorageKey) | ||
229 | } | ||
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts new file mode 100644 index 000000000..2a889190b --- /dev/null +++ b/server/lib/object-storage/urls.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' | ||
3 | |||
4 | function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { | ||
5 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) | ||
6 | } | ||
7 | |||
8 | function getWebTorrentPublicFileUrl (fileUrl: string) { | ||
9 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL | ||
10 | if (!baseUrl) return fileUrl | ||
11 | |||
12 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
13 | } | ||
14 | |||
15 | function getHLSPublicFileUrl (fileUrl: string) { | ||
16 | const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL | ||
17 | if (!baseUrl) return fileUrl | ||
18 | |||
19 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
20 | } | ||
21 | |||
22 | export { | ||
23 | getPrivateUrl, | ||
24 | getWebTorrentPublicFileUrl, | ||
25 | replaceByBaseUrl, | ||
26 | getHLSPublicFileUrl | ||
27 | } | ||
28 | |||
29 | // --------------------------------------------------------------------------- | ||
30 | |||
31 | function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { | ||
32 | if (baseUrl) return baseUrl | ||
33 | |||
34 | return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` | ||
35 | } | ||
36 | |||
37 | const regex = new RegExp('https?://[^/]+') | ||
38 | function replaceByBaseUrl (fileUrl: string, baseUrl: string) { | ||
39 | return fileUrl.replace(regex, baseUrl) | ||
40 | } | ||
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts new file mode 100644 index 000000000..15b8f58d5 --- /dev/null +++ b/server/lib/object-storage/videos.ts | |||
@@ -0,0 +1,72 @@ | |||
1 | import { join } from 'path' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models' | ||
5 | import { getHLSDirectory } from '../paths' | ||
6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | ||
7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | ||
8 | |||
9 | function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { | ||
10 | const baseHlsDirectory = getHLSDirectory(video) | ||
11 | |||
12 | return storeObject({ | ||
13 | inputPath: join(baseHlsDirectory, filename), | ||
14 | objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename), | ||
15 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
16 | }) | ||
17 | } | ||
18 | |||
19 | function storeWebTorrentFile (filename: string) { | ||
20 | return storeObject({ | ||
21 | inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), | ||
22 | objectStorageKey: generateWebTorrentObjectStorageKey(filename), | ||
23 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) { | ||
28 | return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
29 | } | ||
30 | |||
31 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { | ||
32 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) | ||
33 | } | ||
34 | |||
35 | async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) { | ||
36 | const key = generateHLSObjectStorageKey(playlist, video, filename) | ||
37 | |||
38 | logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) | ||
39 | |||
40 | await makeAvailable({ | ||
41 | key, | ||
42 | destination, | ||
43 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
44 | }) | ||
45 | |||
46 | return destination | ||
47 | } | ||
48 | |||
49 | async function makeWebTorrentFileAvailable (filename: string, destination: string) { | ||
50 | const key = generateWebTorrentObjectStorageKey(filename) | ||
51 | |||
52 | logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags()) | ||
53 | |||
54 | await makeAvailable({ | ||
55 | key, | ||
56 | destination, | ||
57 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | ||
58 | }) | ||
59 | |||
60 | return destination | ||
61 | } | ||
62 | |||
63 | export { | ||
64 | storeWebTorrentFile, | ||
65 | storeHLSFile, | ||
66 | |||
67 | removeHLSObjectStorage, | ||
68 | removeWebTorrentObjectStorage, | ||
69 | |||
70 | makeWebTorrentFileAvailable, | ||
71 | makeHLSFileAvailable | ||
72 | } | ||
diff --git a/server/lib/video-paths.ts b/server/lib/paths.ts index 1e4382108..434e637c6 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/paths.ts | |||
@@ -1,9 +1,8 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { extractVideo } from '@server/helpers/video' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
5 | import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
6 | import { buildUUID } from '@server/helpers/uuid' | 2 | import { buildUUID } from '@server/helpers/uuid' |
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' | ||
5 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
7 | import { removeFragmentedMP4Ext } from '@shared/core-utils' | 6 | import { removeFragmentedMP4Ext } from '@shared/core-utils' |
8 | 7 | ||
9 | // ################## Video file name ################## | 8 | // ################## Video file name ################## |
@@ -16,39 +15,18 @@ function generateHLSVideoFilename (resolution: number) { | |||
16 | return `${buildUUID()}-${resolution}-fragmented.mp4` | 15 | return `${buildUUID()}-${resolution}-fragmented.mp4` |
17 | } | 16 | } |
18 | 17 | ||
19 | function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { | 18 | // ################## Streaming playlist ################## |
20 | if (videoFile.isHLS()) { | ||
21 | const video = extractVideo(videoOrPlaylist) | ||
22 | |||
23 | return join(getHLSDirectory(video), videoFile.filename) | ||
24 | } | ||
25 | |||
26 | const baseDir = isRedundancy | ||
27 | ? CONFIG.STORAGE.REDUNDANCY_DIR | ||
28 | : CONFIG.STORAGE.VIDEOS_DIR | ||
29 | |||
30 | return join(baseDir, videoFile.filename) | ||
31 | } | ||
32 | |||
33 | // ################## Redundancy ################## | ||
34 | 19 | ||
35 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | 20 | function getLiveDirectory (video: MVideoUUID) { |
36 | // Base URL used by our HLS player | 21 | return getHLSDirectory(video) |
37 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
38 | } | 22 | } |
39 | 23 | ||
40 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | 24 | function getHLSDirectory (video: MVideoUUID) { |
41 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | 25 | return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) |
42 | } | 26 | } |
43 | 27 | ||
44 | // ################## Streaming playlist ################## | 28 | function getHLSRedundancyDirectory (video: MVideoUUID) { |
45 | 29 | return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | |
46 | function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { | ||
47 | const baseDir = isRedundancy | ||
48 | ? HLS_REDUNDANCY_DIRECTORY | ||
49 | : HLS_STREAMING_PLAYLIST_DIRECTORY | ||
50 | |||
51 | return join(baseDir, video.uuid) | ||
52 | } | 30 | } |
53 | 31 | ||
54 | function getHlsResolutionPlaylistFilename (videoFilename: string) { | 32 | function getHlsResolutionPlaylistFilename (videoFilename: string) { |
@@ -81,36 +59,24 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi | |||
81 | return uuid + '-' + resolution + extension | 59 | return uuid + '-' + resolution + extension |
82 | } | 60 | } |
83 | 61 | ||
84 | function getTorrentFilePath (videoFile: MVideoFile) { | 62 | function getFSTorrentFilePath (videoFile: MVideoFile) { |
85 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) | 63 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) |
86 | } | 64 | } |
87 | 65 | ||
88 | // ################## Meta data ################## | ||
89 | |||
90 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
91 | const path = '/api/v1/videos/' | ||
92 | |||
93 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
94 | } | ||
95 | |||
96 | // --------------------------------------------------------------------------- | 66 | // --------------------------------------------------------------------------- |
97 | 67 | ||
98 | export { | 68 | export { |
99 | generateHLSVideoFilename, | 69 | generateHLSVideoFilename, |
100 | generateWebTorrentVideoFilename, | 70 | generateWebTorrentVideoFilename, |
101 | 71 | ||
102 | getVideoFilePath, | ||
103 | |||
104 | generateTorrentFileName, | 72 | generateTorrentFileName, |
105 | getTorrentFilePath, | 73 | getFSTorrentFilePath, |
106 | 74 | ||
107 | getHLSDirectory, | 75 | getHLSDirectory, |
76 | getLiveDirectory, | ||
77 | getHLSRedundancyDirectory, | ||
78 | |||
108 | generateHLSMasterPlaylistFilename, | 79 | generateHLSMasterPlaylistFilename, |
109 | generateHlsSha256SegmentsFilename, | 80 | generateHlsSha256SegmentsFilename, |
110 | getHlsResolutionPlaylistFilename, | 81 | getHlsResolutionPlaylistFilename |
111 | |||
112 | getLocalVideoFileMetadataUrl, | ||
113 | |||
114 | generateWebTorrentRedundancyUrl, | ||
115 | generateHLSRedundancyUrl | ||
116 | } | 82 | } |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 137ae53a0..ebfd015b5 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -24,7 +24,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli | |||
24 | import { getOrCreateAPVideo } from '../activitypub/videos' | 24 | import { getOrCreateAPVideo } from '../activitypub/videos' |
25 | import { downloadPlaylistSegments } from '../hls' | 25 | import { downloadPlaylistSegments } from '../hls' |
26 | import { removeVideoRedundancy } from '../redundancy' | 26 | import { removeVideoRedundancy } from '../redundancy' |
27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' | 27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls' |
28 | import { AbstractScheduler } from './abstract-scheduler' | 28 | import { AbstractScheduler } from './abstract-scheduler' |
29 | 29 | ||
30 | type CandidateToDuplicate = { | 30 | type CandidateToDuplicate = { |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index c08523988..d2384f53c 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | |||
3 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' | 2 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' |
4 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' | 3 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' |
5 | import { generateImageFilename, processImage } from '../helpers/image-utils' | 4 | import { generateImageFilename, processImage } from '../helpers/image-utils' |
@@ -10,7 +9,7 @@ import { ThumbnailModel } from '../models/video/thumbnail' | |||
10 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' | 9 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' |
11 | import { MThumbnail } from '../types/models/video/thumbnail' | 10 | import { MThumbnail } from '../types/models/video/thumbnail' |
12 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | 11 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' |
13 | import { getVideoFilePath } from './video-paths' | 12 | import { VideoPathManager } from './video-path-manager' |
14 | 13 | ||
15 | type ImageSize = { height?: number, width?: number } | 14 | type ImageSize = { height?: number, width?: number } |
16 | 15 | ||
@@ -116,21 +115,22 @@ function generateVideoMiniature (options: { | |||
116 | }) { | 115 | }) { |
117 | const { video, videoFile, type } = options | 116 | const { video, videoFile, type } = options |
118 | 117 | ||
119 | const input = getVideoFilePath(video, videoFile) | 118 | return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => { |
119 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | ||
120 | 120 | ||
121 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | 121 | const thumbnailCreator = videoFile.isAudio() |
122 | const thumbnailCreator = videoFile.isAudio() | 122 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) |
123 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) | 123 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) |
124 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | ||
125 | 124 | ||
126 | return updateThumbnailFromFunction({ | 125 | return updateThumbnailFromFunction({ |
127 | thumbnailCreator, | 126 | thumbnailCreator, |
128 | filename, | 127 | filename, |
129 | height, | 128 | height, |
130 | width, | 129 | width, |
131 | type, | 130 | type, |
132 | automaticallyGenerated: true, | 131 | automaticallyGenerated: true, |
133 | existingThumbnail | 132 | existingThumbnail |
133 | }) | ||
134 | }) | 134 | }) |
135 | } | 135 | } |
136 | 136 | ||
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index d2a556360..ee228c011 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts | |||
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path' | |||
4 | import { toEven } from '@server/helpers/core-utils' | 4 | import { toEven } from '@server/helpers/core-utils' |
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
7 | import { VideoResolution } from '../../../shared/models/videos' | 7 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' |
8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' | 9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' |
10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' | 10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' |
11 | import { logger } from '../../helpers/logger' | 11 | import { logger } from '../../helpers/logger' |
12 | import { CONFIG } from '../../initializers/config' | 12 | import { CONFIG } from '../../initializers/config' |
13 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' | 13 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' |
14 | import { VideoFileModel } from '../../models/video/video-file' | 14 | import { VideoFileModel } from '../../models/video/video-file' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | 15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' |
16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' | 16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' |
@@ -19,9 +19,9 @@ import { | |||
19 | generateHlsSha256SegmentsFilename, | 19 | generateHlsSha256SegmentsFilename, |
20 | generateHLSVideoFilename, | 20 | generateHLSVideoFilename, |
21 | generateWebTorrentVideoFilename, | 21 | generateWebTorrentVideoFilename, |
22 | getHlsResolutionPlaylistFilename, | 22 | getHlsResolutionPlaylistFilename |
23 | getVideoFilePath | 23 | } from '../paths' |
24 | } from '../video-paths' | 24 | import { VideoPathManager } from '../video-path-manager' |
25 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | 25 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' |
26 | 26 | ||
27 | /** | 27 | /** |
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | |||
32 | */ | 32 | */ |
33 | 33 | ||
34 | // Optimize the original video file and replace it. The resolution is not changed. | 34 | // Optimize the original video file and replace it. The resolution is not changed. |
35 | async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { | 35 | function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { |
36 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 36 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
37 | const newExtname = '.mp4' | 37 | const newExtname = '.mp4' |
38 | 38 | ||
39 | const videoInputPath = getVideoFilePath(video, inputVideoFile) | 39 | return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => { |
40 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 40 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
41 | 41 | ||
42 | const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) | 42 | const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) |
43 | ? 'quick-transcode' | 43 | ? 'quick-transcode' |
44 | : 'video' | 44 | : 'video' |
45 | 45 | ||
46 | const resolution = toEven(inputVideoFile.resolution) | 46 | const resolution = toEven(inputVideoFile.resolution) |
47 | 47 | ||
48 | const transcodeOptions: TranscodeOptions = { | 48 | const transcodeOptions: TranscodeOptions = { |
49 | type: transcodeType, | 49 | type: transcodeType, |
50 | 50 | ||
51 | inputPath: videoInputPath, | 51 | inputPath: videoInputPath, |
52 | outputPath: videoTranscodedPath, | 52 | outputPath: videoTranscodedPath, |
53 | 53 | ||
54 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 54 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
55 | profile: CONFIG.TRANSCODING.PROFILE, | 55 | profile: CONFIG.TRANSCODING.PROFILE, |
56 | 56 | ||
57 | resolution, | 57 | resolution, |
58 | 58 | ||
59 | job | 59 | job |
60 | } | 60 | } |
61 | 61 | ||
62 | // Could be very long! | 62 | // Could be very long! |
63 | await transcode(transcodeOptions) | 63 | await transcode(transcodeOptions) |
64 | 64 | ||
65 | try { | 65 | try { |
66 | await remove(videoInputPath) | 66 | await remove(videoInputPath) |
67 | 67 | ||
68 | // Important to do this before getVideoFilename() to take in account the new filename | 68 | // Important to do this before getVideoFilename() to take in account the new filename |
69 | inputVideoFile.extname = newExtname | 69 | inputVideoFile.extname = newExtname |
70 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | 70 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) |
71 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
71 | 72 | ||
72 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 73 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) |
73 | 74 | ||
74 | await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 75 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) |
75 | 76 | ||
76 | return transcodeType | 77 | return { transcodeType, videoFile } |
77 | } catch (err) { | 78 | } catch (err) { |
78 | // Auto destruction... | 79 | // Auto destruction... |
79 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) | 80 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) |
80 | 81 | ||
81 | throw err | 82 | throw err |
82 | } | 83 | } |
84 | }) | ||
83 | } | 85 | } |
84 | 86 | ||
85 | // Transcode the original video file to a lower resolution. | 87 | // Transcode the original video file to a lower resolution |
86 | async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { | 88 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed |
89 | function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { | ||
87 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 90 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
88 | const extname = '.mp4' | 91 | const extname = '.mp4' |
89 | 92 | ||
90 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed | 93 | return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => { |
91 | const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) | 94 | const newVideoFile = new VideoFileModel({ |
95 | resolution, | ||
96 | extname, | ||
97 | filename: generateWebTorrentVideoFilename(resolution, extname), | ||
98 | size: 0, | ||
99 | videoId: video.id | ||
100 | }) | ||
92 | 101 | ||
93 | const newVideoFile = new VideoFileModel({ | 102 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
94 | resolution, | 103 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) |
95 | extname, | ||
96 | filename: generateWebTorrentVideoFilename(resolution, extname), | ||
97 | size: 0, | ||
98 | videoId: video.id | ||
99 | }) | ||
100 | 104 | ||
101 | const videoOutputPath = getVideoFilePath(video, newVideoFile) | 105 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO |
102 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) | 106 | ? { |
107 | type: 'only-audio' as 'only-audio', | ||
103 | 108 | ||
104 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO | 109 | inputPath: videoInputPath, |
105 | ? { | 110 | outputPath: videoTranscodedPath, |
106 | type: 'only-audio' as 'only-audio', | ||
107 | 111 | ||
108 | inputPath: videoInputPath, | 112 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
109 | outputPath: videoTranscodedPath, | 113 | profile: CONFIG.TRANSCODING.PROFILE, |
110 | 114 | ||
111 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 115 | resolution, |
112 | profile: CONFIG.TRANSCODING.PROFILE, | ||
113 | 116 | ||
114 | resolution, | 117 | job |
118 | } | ||
119 | : { | ||
120 | type: 'video' as 'video', | ||
121 | inputPath: videoInputPath, | ||
122 | outputPath: videoTranscodedPath, | ||
115 | 123 | ||
116 | job | 124 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
117 | } | 125 | profile: CONFIG.TRANSCODING.PROFILE, |
118 | : { | ||
119 | type: 'video' as 'video', | ||
120 | inputPath: videoInputPath, | ||
121 | outputPath: videoTranscodedPath, | ||
122 | 126 | ||
123 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 127 | resolution, |
124 | profile: CONFIG.TRANSCODING.PROFILE, | 128 | isPortraitMode: isPortrait, |
125 | 129 | ||
126 | resolution, | 130 | job |
127 | isPortraitMode: isPortrait, | 131 | } |
128 | 132 | ||
129 | job | 133 | await transcode(transcodeOptions) |
130 | } | ||
131 | |||
132 | await transcode(transcodeOptions) | ||
133 | 134 | ||
134 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) | 135 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) |
136 | }) | ||
135 | } | 137 | } |
136 | 138 | ||
137 | // Merge an image with an audio file to create a video | 139 | // Merge an image with an audio file to create a video |
138 | async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { | 140 | function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { |
139 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 141 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
140 | const newExtname = '.mp4' | 142 | const newExtname = '.mp4' |
141 | 143 | ||
142 | const inputVideoFile = video.getMinQualityFile() | 144 | const inputVideoFile = video.getMinQualityFile() |
143 | 145 | ||
144 | const audioInputPath = getVideoFilePath(video, inputVideoFile) | 146 | return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => { |
145 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 147 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
146 | 148 | ||
147 | // If the user updates the video preview during transcoding | 149 | // If the user updates the video preview during transcoding |
148 | const previewPath = video.getPreview().getPath() | 150 | const previewPath = video.getPreview().getPath() |
149 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | 151 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) |
150 | await copyFile(previewPath, tmpPreviewPath) | 152 | await copyFile(previewPath, tmpPreviewPath) |
151 | 153 | ||
152 | const transcodeOptions = { | 154 | const transcodeOptions = { |
153 | type: 'merge-audio' as 'merge-audio', | 155 | type: 'merge-audio' as 'merge-audio', |
154 | 156 | ||
155 | inputPath: tmpPreviewPath, | 157 | inputPath: tmpPreviewPath, |
156 | outputPath: videoTranscodedPath, | 158 | outputPath: videoTranscodedPath, |
157 | 159 | ||
158 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 160 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
159 | profile: CONFIG.TRANSCODING.PROFILE, | 161 | profile: CONFIG.TRANSCODING.PROFILE, |
160 | 162 | ||
161 | audioPath: audioInputPath, | 163 | audioPath: audioInputPath, |
162 | resolution, | 164 | resolution, |
163 | 165 | ||
164 | job | 166 | job |
165 | } | 167 | } |
166 | 168 | ||
167 | try { | 169 | try { |
168 | await transcode(transcodeOptions) | 170 | await transcode(transcodeOptions) |
169 | 171 | ||
170 | await remove(audioInputPath) | 172 | await remove(audioInputPath) |
171 | await remove(tmpPreviewPath) | 173 | await remove(tmpPreviewPath) |
172 | } catch (err) { | 174 | } catch (err) { |
173 | await remove(tmpPreviewPath) | 175 | await remove(tmpPreviewPath) |
174 | throw err | 176 | throw err |
175 | } | 177 | } |
176 | 178 | ||
177 | // Important to do this before getVideoFilename() to take in account the new file extension | 179 | // Important to do this before getVideoFilename() to take in account the new file extension |
178 | inputVideoFile.extname = newExtname | 180 | inputVideoFile.extname = newExtname |
179 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | 181 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) |
180 | 182 | ||
181 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 183 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) |
182 | // ffmpeg generated a new video file, so update the video duration | 184 | // ffmpeg generated a new video file, so update the video duration |
183 | // See https://trac.ffmpeg.org/ticket/5456 | 185 | // See https://trac.ffmpeg.org/ticket/5456 |
184 | video.duration = await getDurationFromVideoFile(videoTranscodedPath) | 186 | video.duration = await getDurationFromVideoFile(videoTranscodedPath) |
185 | await video.save() | 187 | await video.save() |
186 | 188 | ||
187 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 189 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) |
190 | }) | ||
188 | } | 191 | } |
189 | 192 | ||
190 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | 193 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist |
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding ( | |||
258 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | 261 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) |
259 | video.VideoFiles = await video.$get('VideoFiles') | 262 | video.VideoFiles = await video.$get('VideoFiles') |
260 | 263 | ||
261 | return video | 264 | return { video, videoFile } |
262 | } | 265 | } |
263 | 266 | ||
264 | async function generateHlsPlaylistCommon (options: { | 267 | async function generateHlsPlaylistCommon (options: { |
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: { | |||
335 | videoStreamingPlaylistId: playlist.id | 338 | videoStreamingPlaylistId: playlist.id |
336 | }) | 339 | }) |
337 | 340 | ||
338 | const videoFilePath = getVideoFilePath(playlist, newVideoFile) | 341 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) |
339 | 342 | ||
340 | // Move files from tmp transcoded directory to the appropriate place | 343 | // Move files from tmp transcoded directory to the appropriate place |
341 | const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | 344 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) |
342 | await ensureDir(baseHlsDirectory) | ||
343 | 345 | ||
344 | // Move playlist file | 346 | // Move playlist file |
345 | const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) | 347 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) |
346 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | 348 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) |
347 | // Move video file | 349 | // Move video file |
348 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | 350 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) |
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: { | |||
355 | 357 | ||
356 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | 358 | await createTorrentAndSetInfoHash(playlist, newVideoFile) |
357 | 359 | ||
358 | await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | 360 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) |
359 | 361 | ||
360 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo | 362 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo |
361 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') | 363 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') |
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: { | |||
368 | await updateMasterHLSPlaylist(video, playlistWithFiles) | 370 | await updateMasterHLSPlaylist(video, playlistWithFiles) |
369 | await updateSha256VODSegments(video, playlistWithFiles) | 371 | await updateSha256VODSegments(video, playlistWithFiles) |
370 | 372 | ||
371 | return resolutionPlaylistPath | 373 | return { resolutionPlaylistPath, videoFile: savedVideoFile } |
372 | } | 374 | } |
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts new file mode 100644 index 000000000..4c5d0c89d --- /dev/null +++ b/server/lib/video-path-manager.ts | |||
@@ -0,0 +1,139 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { extname, join } from 'path' | ||
3 | import { buildUUID } from '@server/helpers/uuid' | ||
4 | import { extractVideo } from '@server/helpers/video' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
7 | import { VideoStorage } from '@shared/models' | ||
8 | import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' | ||
9 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' | ||
10 | |||
11 | type MakeAvailableCB <T> = (path: string) => Promise<T> | T | ||
12 | |||
13 | class VideoPathManager { | ||
14 | |||
15 | private static instance: VideoPathManager | ||
16 | |||
17 | private constructor () {} | ||
18 | |||
19 | getFSHLSOutputPath (video: MVideoUUID, filename?: string) { | ||
20 | const base = getHLSDirectory(video) | ||
21 | if (!filename) return base | ||
22 | |||
23 | return join(base, filename) | ||
24 | } | ||
25 | |||
26 | getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
27 | if (videoFile.isHLS()) { | ||
28 | const video = extractVideo(videoOrPlaylist) | ||
29 | |||
30 | return join(getHLSRedundancyDirectory(video), videoFile.filename) | ||
31 | } | ||
32 | |||
33 | return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) | ||
34 | } | ||
35 | |||
36 | getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
37 | if (videoFile.isHLS()) { | ||
38 | const video = extractVideo(videoOrPlaylist) | ||
39 | |||
40 | return join(getHLSDirectory(video), videoFile.filename) | ||
41 | } | ||
42 | |||
43 | return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) | ||
44 | } | ||
45 | |||
46 | async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { | ||
47 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
48 | return this.makeAvailableFactory( | ||
49 | () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile), | ||
50 | false, | ||
51 | cb | ||
52 | ) | ||
53 | } | ||
54 | |||
55 | const destination = this.buildTMPDestination(videoFile.filename) | ||
56 | |||
57 | if (videoFile.isHLS()) { | ||
58 | const video = extractVideo(videoOrPlaylist) | ||
59 | |||
60 | return this.makeAvailableFactory( | ||
61 | () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination), | ||
62 | true, | ||
63 | cb | ||
64 | ) | ||
65 | } | ||
66 | |||
67 | return this.makeAvailableFactory( | ||
68 | () => makeWebTorrentFileAvailable(videoFile.filename, destination), | ||
69 | true, | ||
70 | cb | ||
71 | ) | ||
72 | } | ||
73 | |||
74 | async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { | ||
75 | const filename = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
76 | |||
77 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
78 | return this.makeAvailableFactory( | ||
79 | () => join(getHLSDirectory(playlist.Video), filename), | ||
80 | false, | ||
81 | cb | ||
82 | ) | ||
83 | } | ||
84 | |||
85 | return this.makeAvailableFactory( | ||
86 | () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), | ||
87 | true, | ||
88 | cb | ||
89 | ) | ||
90 | } | ||
91 | |||
92 | async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) { | ||
93 | if (playlist.storage === VideoStorage.FILE_SYSTEM) { | ||
94 | return this.makeAvailableFactory( | ||
95 | () => join(getHLSDirectory(playlist.Video), filename), | ||
96 | false, | ||
97 | cb | ||
98 | ) | ||
99 | } | ||
100 | |||
101 | return this.makeAvailableFactory( | ||
102 | () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), | ||
103 | true, | ||
104 | cb | ||
105 | ) | ||
106 | } | ||
107 | |||
108 | private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { | ||
109 | let result: T | ||
110 | |||
111 | const destination = await method() | ||
112 | |||
113 | try { | ||
114 | result = await cb(destination) | ||
115 | } catch (err) { | ||
116 | if (destination && clean) await remove(destination) | ||
117 | throw err | ||
118 | } | ||
119 | |||
120 | if (clean) await remove(destination) | ||
121 | |||
122 | return result | ||
123 | } | ||
124 | |||
125 | private buildTMPDestination (filename: string) { | ||
126 | return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) | ||
127 | |||
128 | } | ||
129 | |||
130 | static get Instance () { | ||
131 | return this.instance || (this.instance = new this()) | ||
132 | } | ||
133 | } | ||
134 | |||
135 | // --------------------------------------------------------------------------- | ||
136 | |||
137 | export { | ||
138 | VideoPathManager | ||
139 | } | ||
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts new file mode 100644 index 000000000..0613d94bf --- /dev/null +++ b/server/lib/video-state.ts | |||
@@ -0,0 +1,99 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
7 | import { MVideoFullLight, MVideoUUID } from '@server/types/models' | ||
8 | import { VideoState } from '@shared/models' | ||
9 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
10 | import { Notifier } from './notifier' | ||
11 | import { addMoveToObjectStorageJob } from './video' | ||
12 | |||
13 | function buildNextVideoState (currentState?: VideoState) { | ||
14 | if (currentState === VideoState.PUBLISHED) { | ||
15 | throw new Error('Video is already in its final state') | ||
16 | } | ||
17 | |||
18 | if ( | ||
19 | currentState !== VideoState.TO_TRANSCODE && | ||
20 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
21 | CONFIG.TRANSCODING.ENABLED | ||
22 | ) { | ||
23 | return VideoState.TO_TRANSCODE | ||
24 | } | ||
25 | |||
26 | if ( | ||
27 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
28 | CONFIG.OBJECT_STORAGE.ENABLED | ||
29 | ) { | ||
30 | return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | ||
31 | } | ||
32 | |||
33 | return VideoState.PUBLISHED | ||
34 | } | ||
35 | |||
36 | function moveToNextState (video: MVideoUUID, isNewVideo = true) { | ||
37 | return sequelizeTypescript.transaction(async t => { | ||
38 | // Maybe the video changed in database, refresh it | ||
39 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
40 | // Video does not exist anymore | ||
41 | if (!videoDatabase) return undefined | ||
42 | |||
43 | // Already in its final state | ||
44 | if (videoDatabase.state === VideoState.PUBLISHED) { | ||
45 | return federateVideoIfNeeded(videoDatabase, false, t) | ||
46 | } | ||
47 | |||
48 | const newState = buildNextVideoState(videoDatabase.state) | ||
49 | |||
50 | if (newState === VideoState.PUBLISHED) { | ||
51 | return moveToPublishedState(videoDatabase, isNewVideo, t) | ||
52 | } | ||
53 | |||
54 | if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
55 | return moveToExternalStorageState(videoDatabase, isNewVideo, t) | ||
56 | } | ||
57 | }) | ||
58 | } | ||
59 | |||
60 | // --------------------------------------------------------------------------- | ||
61 | |||
62 | export { | ||
63 | buildNextVideoState, | ||
64 | moveToNextState | ||
65 | } | ||
66 | |||
67 | // --------------------------------------------------------------------------- | ||
68 | |||
69 | async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
70 | logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
71 | |||
72 | const previousState = video.state | ||
73 | await video.setNewState(VideoState.PUBLISHED, transaction) | ||
74 | |||
75 | // If the video was not published, we consider it is a new one for other instances | ||
76 | // Live videos are always federated, so it's not a new video | ||
77 | await federateVideoIfNeeded(video, isNewVideo, transaction) | ||
78 | |||
79 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
80 | |||
81 | if (previousState === VideoState.TO_TRANSCODE) { | ||
82 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) | ||
83 | } | ||
84 | } | ||
85 | |||
86 | async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
87 | const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) | ||
88 | const pendingTranscode = videoJobInfo?.pendingTranscode || 0 | ||
89 | |||
90 | // We want to wait all transcoding jobs before moving the video on an external storage | ||
91 | if (pendingTranscode !== 0) return | ||
92 | |||
93 | await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction) | ||
94 | |||
95 | logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
96 | |||
97 | addMoveToObjectStorageJob(video, isNewVideo) | ||
98 | .catch(err => logger.error('Cannot add move to object storage job', { err })) | ||
99 | } | ||
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts new file mode 100644 index 000000000..64c2c9bf9 --- /dev/null +++ b/server/lib/video-urls.ts | |||
@@ -0,0 +1,31 @@ | |||
1 | |||
2 | import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
3 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
4 | |||
5 | // ################## Redundancy ################## | ||
6 | |||
7 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | ||
8 | // Base URL used by our HLS player | ||
9 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
10 | } | ||
11 | |||
12 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | ||
13 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | ||
14 | } | ||
15 | |||
16 | // ################## Meta data ################## | ||
17 | |||
18 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
19 | const path = '/api/v1/videos/' | ||
20 | |||
21 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
22 | } | ||
23 | |||
24 | // --------------------------------------------------------------------------- | ||
25 | |||
26 | export { | ||
27 | getLocalVideoFileMetadataUrl, | ||
28 | |||
29 | generateWebTorrentRedundancyUrl, | ||
30 | generateHLSRedundancyUrl | ||
31 | } | ||
diff --git a/server/lib/video.ts b/server/lib/video.ts index 61fee4949..0a2b93cc0 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -1,15 +1,13 @@ | |||
1 | import { UploadFiles } from 'express' | 1 | import { UploadFiles } from 'express' |
2 | import { Transaction } from 'sequelize/types' | 2 | import { Transaction } from 'sequelize/types' |
3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' | 3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' |
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { TagModel } from '@server/models/video/tag' | 4 | import { TagModel } from '@server/models/video/tag' |
6 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
7 | import { FilteredModelAttributes } from '@server/types' | 7 | import { FilteredModelAttributes } from '@server/types' |
8 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | 8 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' |
9 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' | 9 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' |
10 | import { federateVideoIfNeeded } from './activitypub/videos' | 10 | import { CreateJobOptions, JobQueue } from './job-queue/job-queue' |
11 | import { JobQueue } from './job-queue/job-queue' | ||
12 | import { Notifier } from './notifier' | ||
13 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 11 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
14 | 12 | ||
15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 13 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
@@ -82,29 +80,6 @@ async function setVideoTags (options: { | |||
82 | video.Tags = tagInstances | 80 | video.Tags = tagInstances |
83 | } | 81 | } |
84 | 82 | ||
85 | async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) { | ||
86 | const result = await sequelizeTypescript.transaction(async t => { | ||
87 | // Maybe the video changed in database, refresh it | ||
88 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
89 | // Video does not exist anymore | ||
90 | if (!videoDatabase) return undefined | ||
91 | |||
92 | // We transcoded the video file in another format, now we can publish it | ||
93 | const videoPublished = await videoDatabase.publishIfNeededAndSave(t) | ||
94 | |||
95 | // If the video was not published, we consider it is a new one for other instances | ||
96 | // Live videos are always federated, so it's not a new video | ||
97 | await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t) | ||
98 | |||
99 | return { videoDatabase, videoPublished } | ||
100 | }) | ||
101 | |||
102 | if (result?.videoPublished) { | ||
103 | Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase) | ||
104 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase) | ||
105 | } | ||
106 | } | ||
107 | |||
108 | async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { | 83 | async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { |
109 | let dataInput: VideoTranscodingPayload | 84 | let dataInput: VideoTranscodingPayload |
110 | 85 | ||
@@ -127,7 +102,20 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF | |||
127 | priority: await getTranscodingJobPriority(user) | 102 | priority: await getTranscodingJobPriority(user) |
128 | } | 103 | } |
129 | 104 | ||
130 | return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions) | 105 | return addTranscodingJob(dataInput, jobOptions) |
106 | } | ||
107 | |||
108 | async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) { | ||
109 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
110 | |||
111 | return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options) | ||
112 | } | ||
113 | |||
114 | async function addMoveToObjectStorageJob (video: MVideoUUID, isNewVideo = true) { | ||
115 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') | ||
116 | |||
117 | const dataInput = { videoUUID: video.uuid, isNewVideo } | ||
118 | return JobQueue.Instance.createJobWithPromise({ type: 'move-to-object-storage', payload: dataInput }) | ||
131 | } | 119 | } |
132 | 120 | ||
133 | async function getTranscodingJobPriority (user: MUserId) { | 121 | async function getTranscodingJobPriority (user: MUserId) { |
@@ -143,9 +131,10 @@ async function getTranscodingJobPriority (user: MUserId) { | |||
143 | 131 | ||
144 | export { | 132 | export { |
145 | buildLocalVideoFromReq, | 133 | buildLocalVideoFromReq, |
146 | publishAndFederateIfNeeded, | ||
147 | buildVideoThumbnailsFromReq, | 134 | buildVideoThumbnailsFromReq, |
148 | setVideoTags, | 135 | setVideoTags, |
149 | addOptimizeOrMergeAudioJob, | 136 | addOptimizeOrMergeAudioJob, |
137 | addTranscodingJob, | ||
138 | addMoveToObjectStorageJob, | ||
150 | getTranscodingJobPriority | 139 | getTranscodingJobPriority |
151 | } | 140 | } |
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 8a54de3b0..b3c4f390d 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { uuidToShort } from '@server/helpers/uuid' | 1 | import { uuidToShort } from '@server/helpers/uuid' |
2 | import { generateMagnetUri } from '@server/helpers/webtorrent' | 2 | import { generateMagnetUri } from '@server/helpers/webtorrent' |
3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' | 3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' |
4 | import { VideoFile } from '@shared/models/videos/video-file.model' | 4 | import { VideoFile } from '@shared/models/videos/video-file.model' |
5 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' | 5 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' |
6 | import { Video, VideoDetails } from '../../../../shared/models/videos' | 6 | import { Video, VideoDetails } from '../../../../shared/models/videos' |
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts index 742d19099..75823864d 100644 --- a/server/models/video/sql/shared/video-tables.ts +++ b/server/models/video/sql/shared/video-tables.ts | |||
@@ -87,7 +87,8 @@ export class VideoTables { | |||
87 | 'fps', | 87 | 'fps', |
88 | 'metadataUrl', | 88 | 'metadataUrl', |
89 | 'videoStreamingPlaylistId', | 89 | 'videoStreamingPlaylistId', |
90 | 'videoId' | 90 | 'videoId', |
91 | 'storage' | ||
91 | ] | 92 | ] |
92 | } | 93 | } |
93 | 94 | ||
@@ -102,7 +103,8 @@ export class VideoTables { | |||
102 | 'segmentsSha256Url', | 103 | 'segmentsSha256Url', |
103 | 'videoId', | 104 | 'videoId', |
104 | 'createdAt', | 105 | 'createdAt', |
105 | 'updatedAt' | 106 | 'updatedAt', |
107 | 'storage' | ||
106 | ]) | 108 | ]) |
107 | } | 109 | } |
108 | 110 | ||
@@ -258,7 +260,8 @@ export class VideoTables { | |||
258 | 'originallyPublishedAt', | 260 | 'originallyPublishedAt', |
259 | 'channelId', | 261 | 'channelId', |
260 | 'createdAt', | 262 | 'createdAt', |
261 | 'updatedAt' | 263 | 'updatedAt', |
264 | 'moveJobsRunning' | ||
262 | ] | 265 | ] |
263 | } | 266 | } |
264 | } | 267 | } |
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 09fc5288b..627c95763 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -23,9 +23,11 @@ import validator from 'validator' | |||
23 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' | 23 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' |
24 | import { logger } from '@server/helpers/logger' | 24 | import { logger } from '@server/helpers/logger' |
25 | import { extractVideo } from '@server/helpers/video' | 25 | import { extractVideo } from '@server/helpers/video' |
26 | import { getTorrentFilePath } from '@server/lib/video-paths' | 26 | import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' |
27 | import { getFSTorrentFilePath } from '@server/lib/paths' | ||
27 | import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' | 28 | import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' |
28 | import { AttributesOnly } from '@shared/core-utils' | 29 | import { AttributesOnly } from '@shared/core-utils' |
30 | import { VideoStorage } from '@shared/models' | ||
29 | import { | 31 | import { |
30 | isVideoFileExtnameValid, | 32 | isVideoFileExtnameValid, |
31 | isVideoFileInfoHashValid, | 33 | isVideoFileInfoHashValid, |
@@ -214,6 +216,11 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
214 | @Column | 216 | @Column |
215 | videoId: number | 217 | videoId: number |
216 | 218 | ||
219 | @AllowNull(false) | ||
220 | @Default(VideoStorage.FILE_SYSTEM) | ||
221 | @Column | ||
222 | storage: VideoStorage | ||
223 | |||
217 | @BelongsTo(() => VideoModel, { | 224 | @BelongsTo(() => VideoModel, { |
218 | foreignKey: { | 225 | foreignKey: { |
219 | allowNull: true | 226 | allowNull: true |
@@ -273,7 +280,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
273 | 280 | ||
274 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { | 281 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { |
275 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + | 282 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + |
276 | 'WHERE "filename" = $filename LIMIT 1' | 283 | `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` |
277 | 284 | ||
278 | return doesExist(query, { filename }) | 285 | return doesExist(query, { filename }) |
279 | } | 286 | } |
@@ -450,9 +457,20 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
450 | return !!this.videoStreamingPlaylistId | 457 | return !!this.videoStreamingPlaylistId |
451 | } | 458 | } |
452 | 459 | ||
460 | getObjectStorageUrl () { | ||
461 | if (this.isHLS()) { | ||
462 | return getHLSPublicFileUrl(this.fileUrl) | ||
463 | } | ||
464 | |||
465 | return getWebTorrentPublicFileUrl(this.fileUrl) | ||
466 | } | ||
467 | |||
453 | getFileUrl (video: MVideo) { | 468 | getFileUrl (video: MVideo) { |
454 | if (!this.Video) this.Video = video as VideoModel | 469 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
470 | return this.getObjectStorageUrl() | ||
471 | } | ||
455 | 472 | ||
473 | if (!this.Video) this.Video = video as VideoModel | ||
456 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) | 474 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) |
457 | 475 | ||
458 | return this.fileUrl | 476 | return this.fileUrl |
@@ -503,7 +521,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
503 | removeTorrent () { | 521 | removeTorrent () { |
504 | if (!this.torrentFilename) return null | 522 | if (!this.torrentFilename) return null |
505 | 523 | ||
506 | const torrentPath = getTorrentFilePath(this) | 524 | const torrentPath = getFSTorrentFilePath(this) |
507 | return remove(torrentPath) | 525 | return remove(torrentPath) |
508 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | 526 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) |
509 | } | 527 | } |
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts new file mode 100644 index 000000000..7c1fe6734 --- /dev/null +++ b/server/models/video/video-job-info.ts | |||
@@ -0,0 +1,100 @@ | |||
1 | import { Op, QueryTypes, Transaction } from 'sequelize' | ||
2 | import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' | ||
3 | import { AttributesOnly } from '@shared/core-utils' | ||
4 | import { VideoModel } from './video' | ||
5 | |||
6 | @Table({ | ||
7 | tableName: 'videoJobInfo', | ||
8 | indexes: [ | ||
9 | { | ||
10 | fields: [ 'videoId' ], | ||
11 | where: { | ||
12 | videoId: { | ||
13 | [Op.ne]: null | ||
14 | } | ||
15 | } | ||
16 | } | ||
17 | ] | ||
18 | }) | ||
19 | |||
20 | export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfoModel>>> { | ||
21 | @CreatedAt | ||
22 | createdAt: Date | ||
23 | |||
24 | @UpdatedAt | ||
25 | updatedAt: Date | ||
26 | |||
27 | @AllowNull(false) | ||
28 | @Default(0) | ||
29 | @IsInt | ||
30 | @Column | ||
31 | pendingMove: number | ||
32 | |||
33 | @AllowNull(false) | ||
34 | @Default(0) | ||
35 | @IsInt | ||
36 | @Column | ||
37 | pendingTranscode: number | ||
38 | |||
39 | @ForeignKey(() => VideoModel) | ||
40 | @Unique | ||
41 | @Column | ||
42 | videoId: number | ||
43 | |||
44 | @BelongsTo(() => VideoModel, { | ||
45 | foreignKey: { | ||
46 | allowNull: false | ||
47 | }, | ||
48 | onDelete: 'cascade' | ||
49 | }) | ||
50 | Video: VideoModel | ||
51 | |||
52 | static load (videoId: number, transaction: Transaction) { | ||
53 | const where = { | ||
54 | videoId | ||
55 | } | ||
56 | |||
57 | return VideoJobInfoModel.findOne({ where, transaction }) | ||
58 | } | ||
59 | |||
60 | static async increaseOrCreate (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> { | ||
61 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | ||
62 | |||
63 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | ||
64 | INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") | ||
65 | SELECT | ||
66 | "video"."id" AS "videoId", 1, NOW(), NOW() | ||
67 | FROM | ||
68 | "video" | ||
69 | WHERE | ||
70 | "video"."uuid" = $videoUUID | ||
71 | ON CONFLICT ("videoId") DO UPDATE | ||
72 | SET | ||
73 | "${column}" = "videoJobInfo"."${column}" + 1, | ||
74 | "updatedAt" = NOW() | ||
75 | RETURNING | ||
76 | "${column}" | ||
77 | `, options) | ||
78 | |||
79 | return pendingMove | ||
80 | } | ||
81 | |||
82 | static async decrease (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> { | ||
83 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | ||
84 | |||
85 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | ||
86 | UPDATE | ||
87 | "videoJobInfo" | ||
88 | SET | ||
89 | "${column}" = "videoJobInfo"."${column}" - 1, | ||
90 | "updatedAt" = NOW() | ||
91 | FROM "video" | ||
92 | WHERE | ||
93 | "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID | ||
94 | RETURNING | ||
95 | "${column}"; | ||
96 | `, options) | ||
97 | |||
98 | return pendingMove | ||
99 | } | ||
100 | } | ||
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index d591a3134..3e9fd97c7 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -1,10 +1,25 @@ | |||
1 | import * as memoizee from 'memoizee' | 1 | import * as memoizee from 'memoizee' |
2 | import { join } from 'path' | 2 | import { join } from 'path' |
3 | import { Op } from 'sequelize' | 3 | import { Op } from 'sequelize' |
4 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | 4 | import { |
5 | AllowNull, | ||
6 | BelongsTo, | ||
7 | Column, | ||
8 | CreatedAt, | ||
9 | DataType, | ||
10 | Default, | ||
11 | ForeignKey, | ||
12 | HasMany, | ||
13 | Is, | ||
14 | Model, | ||
15 | Table, | ||
16 | UpdatedAt | ||
17 | } from 'sequelize-typescript' | ||
18 | import { getHLSPublicFileUrl } from '@server/lib/object-storage' | ||
5 | import { VideoFileModel } from '@server/models/video/video-file' | 19 | import { VideoFileModel } from '@server/models/video/video-file' |
6 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 20 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
7 | import { AttributesOnly } from '@shared/core-utils' | 21 | import { AttributesOnly } from '@shared/core-utils' |
22 | import { VideoStorage } from '@shared/models' | ||
8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 23 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
9 | import { sha1 } from '../../helpers/core-utils' | 24 | import { sha1 } from '../../helpers/core-utils' |
10 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 25 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
@@ -81,6 +96,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
81 | @Column | 96 | @Column |
82 | videoId: number | 97 | videoId: number |
83 | 98 | ||
99 | @AllowNull(false) | ||
100 | @Default(VideoStorage.FILE_SYSTEM) | ||
101 | @Column | ||
102 | storage: VideoStorage | ||
103 | |||
84 | @BelongsTo(() => VideoModel, { | 104 | @BelongsTo(() => VideoModel, { |
85 | foreignKey: { | 105 | foreignKey: { |
86 | allowNull: false | 106 | allowNull: false |
@@ -185,12 +205,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
185 | } | 205 | } |
186 | 206 | ||
187 | getMasterPlaylistUrl (video: MVideo) { | 207 | getMasterPlaylistUrl (video: MVideo) { |
208 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
209 | return getHLSPublicFileUrl(this.playlistUrl) | ||
210 | } | ||
211 | |||
188 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) | 212 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) |
189 | 213 | ||
190 | return this.playlistUrl | 214 | return this.playlistUrl |
191 | } | 215 | } |
192 | 216 | ||
193 | getSha256SegmentsUrl (video: MVideo) { | 217 | getSha256SegmentsUrl (video: MVideo) { |
218 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
219 | return getHLSPublicFileUrl(this.segmentsSha256Url) | ||
220 | } | ||
221 | |||
194 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) | 222 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) |
195 | 223 | ||
196 | return this.segmentsSha256Url | 224 | return this.segmentsSha256Url |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 56a5b0e18..874ad168a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -28,14 +28,16 @@ import { buildNSFWFilter } from '@server/helpers/express-utils' | |||
28 | import { uuidToShort } from '@server/helpers/uuid' | 28 | import { uuidToShort } from '@server/helpers/uuid' |
29 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' | 29 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' |
30 | import { LiveManager } from '@server/lib/live/live-manager' | 30 | import { LiveManager } from '@server/lib/live/live-manager' |
31 | import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' | 31 | import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' |
32 | import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths' | ||
33 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
32 | import { getServerActor } from '@server/models/application/application' | 34 | import { getServerActor } from '@server/models/application/application' |
33 | import { ModelCache } from '@server/models/model-cache' | 35 | import { ModelCache } from '@server/models/model-cache' |
34 | import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' | 36 | import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' |
35 | import { VideoFile } from '@shared/models/videos/video-file.model' | 37 | import { VideoFile } from '@shared/models/videos/video-file.model' |
36 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' | 38 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' |
37 | import { VideoObject } from '../../../shared/models/activitypub/objects' | 39 | import { VideoObject } from '../../../shared/models/activitypub/objects' |
38 | import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos' | 40 | import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos' |
39 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' | 41 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' |
40 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 42 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 43 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
@@ -114,6 +116,7 @@ import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel | |||
114 | import { VideoCommentModel } from './video-comment' | 116 | import { VideoCommentModel } from './video-comment' |
115 | import { VideoFileModel } from './video-file' | 117 | import { VideoFileModel } from './video-file' |
116 | import { VideoImportModel } from './video-import' | 118 | import { VideoImportModel } from './video-import' |
119 | import { VideoJobInfoModel } from './video-job-info' | ||
117 | import { VideoLiveModel } from './video-live' | 120 | import { VideoLiveModel } from './video-live' |
118 | import { VideoPlaylistElementModel } from './video-playlist-element' | 121 | import { VideoPlaylistElementModel } from './video-playlist-element' |
119 | import { VideoShareModel } from './video-share' | 122 | import { VideoShareModel } from './video-share' |
@@ -732,6 +735,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
732 | }) | 735 | }) |
733 | VideoCaptions: VideoCaptionModel[] | 736 | VideoCaptions: VideoCaptionModel[] |
734 | 737 | ||
738 | @HasOne(() => VideoJobInfoModel, { | ||
739 | foreignKey: { | ||
740 | name: 'videoId', | ||
741 | allowNull: false | ||
742 | }, | ||
743 | onDelete: 'cascade' | ||
744 | }) | ||
745 | VideoJobInfo: VideoJobInfoModel | ||
746 | |||
735 | @BeforeDestroy | 747 | @BeforeDestroy |
736 | static async sendDelete (instance: MVideoAccountLight, options) { | 748 | static async sendDelete (instance: MVideoAccountLight, options) { |
737 | if (!instance.isOwned()) return undefined | 749 | if (!instance.isOwned()) return undefined |
@@ -1641,9 +1653,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1641 | getMaxQualityResolution () { | 1653 | getMaxQualityResolution () { |
1642 | const file = this.getMaxQualityFile() | 1654 | const file = this.getMaxQualityFile() |
1643 | const videoOrPlaylist = file.getVideoOrStreamingPlaylist() | 1655 | const videoOrPlaylist = file.getVideoOrStreamingPlaylist() |
1644 | const originalFilePath = getVideoFilePath(videoOrPlaylist, file) | ||
1645 | 1656 | ||
1646 | return getVideoFileResolution(originalFilePath) | 1657 | return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => { |
1658 | return getVideoFileResolution(originalFilePath) | ||
1659 | }) | ||
1647 | } | 1660 | } |
1648 | 1661 | ||
1649 | getDescriptionAPIPath () { | 1662 | getDescriptionAPIPath () { |
@@ -1673,16 +1686,24 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1673 | } | 1686 | } |
1674 | 1687 | ||
1675 | removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { | 1688 | removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { |
1676 | const filePath = getVideoFilePath(this, videoFile, isRedundancy) | 1689 | const filePath = isRedundancy |
1690 | ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) | ||
1691 | : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) | ||
1677 | 1692 | ||
1678 | const promises: Promise<any>[] = [ remove(filePath) ] | 1693 | const promises: Promise<any>[] = [ remove(filePath) ] |
1679 | if (!isRedundancy) promises.push(videoFile.removeTorrent()) | 1694 | if (!isRedundancy) promises.push(videoFile.removeTorrent()) |
1680 | 1695 | ||
1696 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { | ||
1697 | promises.push(removeWebTorrentObjectStorage(videoFile)) | ||
1698 | } | ||
1699 | |||
1681 | return Promise.all(promises) | 1700 | return Promise.all(promises) |
1682 | } | 1701 | } |
1683 | 1702 | ||
1684 | async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { | 1703 | async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { |
1685 | const directoryPath = getHLSDirectory(this, isRedundancy) | 1704 | const directoryPath = isRedundancy |
1705 | ? getHLSRedundancyDirectory(this) | ||
1706 | : getHLSDirectory(this) | ||
1686 | 1707 | ||
1687 | await remove(directoryPath) | 1708 | await remove(directoryPath) |
1688 | 1709 | ||
@@ -1698,6 +1719,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1698 | await Promise.all( | 1719 | await Promise.all( |
1699 | streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) | 1720 | streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) |
1700 | ) | 1721 | ) |
1722 | |||
1723 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
1724 | await removeHLSObjectStorage(streamingPlaylist, this) | ||
1725 | } | ||
1701 | } | 1726 | } |
1702 | } | 1727 | } |
1703 | 1728 | ||
@@ -1741,16 +1766,16 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1741 | this.privacy === VideoPrivacy.INTERNAL | 1766 | this.privacy === VideoPrivacy.INTERNAL |
1742 | } | 1767 | } |
1743 | 1768 | ||
1744 | async publishIfNeededAndSave (t: Transaction) { | 1769 | async setNewState (newState: VideoState, transaction: Transaction) { |
1745 | if (this.state !== VideoState.PUBLISHED) { | 1770 | if (this.state === newState) throw new Error('Cannot use same state ' + newState) |
1746 | this.state = VideoState.PUBLISHED | 1771 | |
1747 | this.publishedAt = new Date() | 1772 | this.state = newState |
1748 | await this.save({ transaction: t }) | ||
1749 | 1773 | ||
1750 | return true | 1774 | if (this.state === VideoState.PUBLISHED) { |
1775 | this.publishedAt = new Date() | ||
1751 | } | 1776 | } |
1752 | 1777 | ||
1753 | return false | 1778 | await this.save({ transaction }) |
1754 | } | 1779 | } |
1755 | 1780 | ||
1756 | getBandwidthBits (videoFile: MVideoFile) { | 1781 | getBandwidthBits (videoFile: MVideoFile) { |
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index b62e2f5f7..19301c0b9 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts | |||
@@ -2,6 +2,7 @@ | |||
2 | import './activitypub' | 2 | import './activitypub' |
3 | import './check-params' | 3 | import './check-params' |
4 | import './moderation' | 4 | import './moderation' |
5 | import './object-storage' | ||
5 | import './notifications' | 6 | import './notifications' |
6 | import './redundancy' | 7 | import './redundancy' |
7 | import './search' | 8 | import './search' |
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index 8f1fb78a5..6c4ea90ca 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts | |||
@@ -15,7 +15,9 @@ import { | |||
15 | stopFfmpeg, | 15 | stopFfmpeg, |
16 | testFfmpegStreamError, | 16 | testFfmpegStreamError, |
17 | wait, | 17 | wait, |
18 | waitJobs | 18 | waitJobs, |
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveSavedOnAllServers | ||
19 | } from '@shared/extra-utils' | 21 | } from '@shared/extra-utils' |
20 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' | 22 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' |
21 | 23 | ||
@@ -66,18 +68,6 @@ describe('Save replay setting', function () { | |||
66 | } | 68 | } |
67 | } | 69 | } |
68 | 70 | ||
69 | async function waitUntilLivePublishedOnAllServers (videoId: string) { | ||
70 | for (const server of servers) { | ||
71 | await server.live.waitUntilPublished({ videoId }) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | async function waitUntilLiveSavedOnAllServers (videoId: string) { | ||
76 | for (const server of servers) { | ||
77 | await server.live.waitUntilSaved({ videoId }) | ||
78 | } | ||
79 | } | ||
80 | |||
81 | before(async function () { | 71 | before(async function () { |
82 | this.timeout(120000) | 72 | this.timeout(120000) |
83 | 73 | ||
@@ -127,7 +117,7 @@ describe('Save replay setting', function () { | |||
127 | 117 | ||
128 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 118 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
129 | 119 | ||
130 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 120 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
131 | 121 | ||
132 | await waitJobs(servers) | 122 | await waitJobs(servers) |
133 | 123 | ||
@@ -160,7 +150,7 @@ describe('Save replay setting', function () { | |||
160 | 150 | ||
161 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 151 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
162 | 152 | ||
163 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 153 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
164 | 154 | ||
165 | await waitJobs(servers) | 155 | await waitJobs(servers) |
166 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 156 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -189,7 +179,7 @@ describe('Save replay setting', function () { | |||
189 | 179 | ||
190 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 180 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
191 | 181 | ||
192 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 182 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
193 | 183 | ||
194 | await waitJobs(servers) | 184 | await waitJobs(servers) |
195 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 185 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -224,7 +214,7 @@ describe('Save replay setting', function () { | |||
224 | this.timeout(20000) | 214 | this.timeout(20000) |
225 | 215 | ||
226 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 216 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
227 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 217 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
228 | 218 | ||
229 | await waitJobs(servers) | 219 | await waitJobs(servers) |
230 | 220 | ||
@@ -237,7 +227,7 @@ describe('Save replay setting', function () { | |||
237 | 227 | ||
238 | await stopFfmpeg(ffmpegCommand) | 228 | await stopFfmpeg(ffmpegCommand) |
239 | 229 | ||
240 | await waitUntilLiveSavedOnAllServers(liveVideoUUID) | 230 | await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID) |
241 | await waitJobs(servers) | 231 | await waitJobs(servers) |
242 | 232 | ||
243 | // Live has been transcoded | 233 | // Live has been transcoded |
@@ -268,7 +258,7 @@ describe('Save replay setting', function () { | |||
268 | liveVideoUUID = await createLiveWrapper(true) | 258 | liveVideoUUID = await createLiveWrapper(true) |
269 | 259 | ||
270 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 260 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
271 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 261 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
272 | 262 | ||
273 | await waitJobs(servers) | 263 | await waitJobs(servers) |
274 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 264 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -296,7 +286,7 @@ describe('Save replay setting', function () { | |||
296 | liveVideoUUID = await createLiveWrapper(true) | 286 | liveVideoUUID = await createLiveWrapper(true) |
297 | 287 | ||
298 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 288 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
299 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 289 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
300 | 290 | ||
301 | await waitJobs(servers) | 291 | await waitJobs(servers) |
302 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 292 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts new file mode 100644 index 000000000..f319d6ef5 --- /dev/null +++ b/server/tests/api/object-storage/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './live' | ||
2 | export * from './video-imports' | ||
3 | export * from './videos' | ||
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts new file mode 100644 index 000000000..d3e6777f2 --- /dev/null +++ b/server/tests/api/object-storage/live.ts | |||
@@ -0,0 +1,136 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { | ||
7 | areObjectStorageTestsDisabled, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | killallServers, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
14 | PeerTubeServer, | ||
15 | setAccessTokensToServers, | ||
16 | setDefaultVideoChannel, | ||
17 | stopFfmpeg, | ||
18 | waitJobs, | ||
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveSavedOnAllServers | ||
21 | } from '@shared/extra-utils' | ||
22 | import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' | ||
23 | |||
24 | const expect = chai.expect | ||
25 | |||
26 | async function createLive (server: PeerTubeServer) { | ||
27 | const attributes: LiveVideoCreate = { | ||
28 | channelId: server.store.channel.id, | ||
29 | privacy: VideoPrivacy.PUBLIC, | ||
30 | name: 'my super live', | ||
31 | saveReplay: true | ||
32 | } | ||
33 | |||
34 | const { uuid } = await server.live.create({ fields: attributes }) | ||
35 | |||
36 | return uuid | ||
37 | } | ||
38 | |||
39 | async function checkFiles (files: VideoFile[]) { | ||
40 | for (const file of files) { | ||
41 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
42 | |||
43 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
44 | } | ||
45 | } | ||
46 | |||
47 | describe('Object storage for lives', function () { | ||
48 | if (areObjectStorageTestsDisabled()) return | ||
49 | |||
50 | let ffmpegCommand: FfmpegCommand | ||
51 | let servers: PeerTubeServer[] | ||
52 | let videoUUID: string | ||
53 | |||
54 | before(async function () { | ||
55 | this.timeout(120000) | ||
56 | |||
57 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
58 | |||
59 | servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) | ||
60 | |||
61 | await setAccessTokensToServers(servers) | ||
62 | await setDefaultVideoChannel(servers) | ||
63 | await doubleFollow(servers[0], servers[1]) | ||
64 | |||
65 | await servers[0].config.enableTranscoding() | ||
66 | }) | ||
67 | |||
68 | describe('Without live transcoding', async function () { | ||
69 | |||
70 | before(async function () { | ||
71 | await servers[0].config.enableLive({ transcoding: false }) | ||
72 | |||
73 | videoUUID = await createLive(servers[0]) | ||
74 | }) | ||
75 | |||
76 | it('Should create a live and save the replay on object storage', async function () { | ||
77 | this.timeout(220000) | ||
78 | |||
79 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
80 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
81 | |||
82 | await stopFfmpeg(ffmpegCommand) | ||
83 | |||
84 | await waitUntilLiveSavedOnAllServers(servers, videoUUID) | ||
85 | await waitJobs(servers) | ||
86 | |||
87 | for (const server of servers) { | ||
88 | const video = await server.videos.get({ id: videoUUID }) | ||
89 | |||
90 | expect(video.files).to.have.lengthOf(0) | ||
91 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
92 | |||
93 | const files = video.streamingPlaylists[0].files | ||
94 | |||
95 | await checkFiles(files) | ||
96 | } | ||
97 | }) | ||
98 | }) | ||
99 | |||
100 | describe('With live transcoding', async function () { | ||
101 | |||
102 | before(async function () { | ||
103 | await servers[0].config.enableLive({ transcoding: true }) | ||
104 | |||
105 | videoUUID = await createLive(servers[0]) | ||
106 | }) | ||
107 | |||
108 | it('Should import a video and have sent it to object storage', async function () { | ||
109 | this.timeout(240000) | ||
110 | |||
111 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
112 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
113 | |||
114 | await stopFfmpeg(ffmpegCommand) | ||
115 | |||
116 | await waitUntilLiveSavedOnAllServers(servers, videoUUID) | ||
117 | await waitJobs(servers) | ||
118 | |||
119 | for (const server of servers) { | ||
120 | const video = await server.videos.get({ id: videoUUID }) | ||
121 | |||
122 | expect(video.files).to.have.lengthOf(0) | ||
123 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
124 | |||
125 | const files = video.streamingPlaylists[0].files | ||
126 | expect(files).to.have.lengthOf(4) | ||
127 | |||
128 | await checkFiles(files) | ||
129 | } | ||
130 | }) | ||
131 | }) | ||
132 | |||
133 | after(async function () { | ||
134 | await killallServers(servers) | ||
135 | }) | ||
136 | }) | ||
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts new file mode 100644 index 000000000..efc01f550 --- /dev/null +++ b/server/tests/api/object-storage/video-imports.ts | |||
@@ -0,0 +1,112 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { | ||
6 | areObjectStorageTestsDisabled, | ||
7 | createSingleServer, | ||
8 | expectStartWith, | ||
9 | FIXTURE_URLS, | ||
10 | killallServers, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | setDefaultVideoChannel, | ||
16 | waitJobs | ||
17 | } from '@shared/extra-utils' | ||
18 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | ||
19 | |||
20 | const expect = chai.expect | ||
21 | |||
22 | async function importVideo (server: PeerTubeServer) { | ||
23 | const attributes = { | ||
24 | name: 'import 2', | ||
25 | privacy: VideoPrivacy.PUBLIC, | ||
26 | channelId: server.store.channel.id, | ||
27 | targetUrl: FIXTURE_URLS.goodVideo720 | ||
28 | } | ||
29 | |||
30 | const { video: { uuid } } = await server.imports.importVideo({ attributes }) | ||
31 | |||
32 | return uuid | ||
33 | } | ||
34 | |||
35 | describe('Object storage for video import', function () { | ||
36 | if (areObjectStorageTestsDisabled()) return | ||
37 | |||
38 | let server: PeerTubeServer | ||
39 | |||
40 | before(async function () { | ||
41 | this.timeout(120000) | ||
42 | |||
43 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
44 | |||
45 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) | ||
46 | |||
47 | await setAccessTokensToServers([ server ]) | ||
48 | await setDefaultVideoChannel([ server ]) | ||
49 | |||
50 | await server.config.enableImports() | ||
51 | }) | ||
52 | |||
53 | describe('Without transcoding', async function () { | ||
54 | |||
55 | before(async function () { | ||
56 | await server.config.disableTranscoding() | ||
57 | }) | ||
58 | |||
59 | it('Should import a video and have sent it to object storage', async function () { | ||
60 | this.timeout(120000) | ||
61 | |||
62 | const uuid = await importVideo(server) | ||
63 | await waitJobs(server) | ||
64 | |||
65 | const video = await server.videos.get({ id: uuid }) | ||
66 | |||
67 | expect(video.files).to.have.lengthOf(1) | ||
68 | expect(video.streamingPlaylists).to.have.lengthOf(0) | ||
69 | |||
70 | const fileUrl = video.files[0].fileUrl | ||
71 | expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
72 | |||
73 | await makeRawRequest(fileUrl, HttpStatusCode.OK_200) | ||
74 | }) | ||
75 | }) | ||
76 | |||
77 | describe('With transcoding', async function () { | ||
78 | |||
79 | before(async function () { | ||
80 | await server.config.enableTranscoding() | ||
81 | }) | ||
82 | |||
83 | it('Should import a video and have sent it to object storage', async function () { | ||
84 | this.timeout(120000) | ||
85 | |||
86 | const uuid = await importVideo(server) | ||
87 | await waitJobs(server) | ||
88 | |||
89 | const video = await server.videos.get({ id: uuid }) | ||
90 | |||
91 | expect(video.files).to.have.lengthOf(4) | ||
92 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
93 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(4) | ||
94 | |||
95 | for (const file of video.files) { | ||
96 | expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
97 | |||
98 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
99 | } | ||
100 | |||
101 | for (const file of video.streamingPlaylists[0].files) { | ||
102 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
103 | |||
104 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
105 | } | ||
106 | }) | ||
107 | }) | ||
108 | |||
109 | after(async function () { | ||
110 | await killallServers([ server ]) | ||
111 | }) | ||
112 | }) | ||
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts new file mode 100644 index 000000000..3958bd3d7 --- /dev/null +++ b/server/tests/api/object-storage/videos.ts | |||
@@ -0,0 +1,391 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { merge } from 'lodash' | ||
6 | import { | ||
7 | areObjectStorageTestsDisabled, | ||
8 | checkTmpIsEmpty, | ||
9 | cleanupTests, | ||
10 | createMultipleServers, | ||
11 | createSingleServer, | ||
12 | doubleFollow, | ||
13 | expectStartWith, | ||
14 | killallServers, | ||
15 | makeRawRequest, | ||
16 | MockObjectStorage, | ||
17 | ObjectStorageCommand, | ||
18 | PeerTubeServer, | ||
19 | setAccessTokensToServers, | ||
20 | waitJobs, | ||
21 | webtorrentAdd | ||
22 | } from '@shared/extra-utils' | ||
23 | import { HttpStatusCode, VideoDetails } from '@shared/models' | ||
24 | |||
25 | const expect = chai.expect | ||
26 | |||
27 | async function checkFiles (options: { | ||
28 | video: VideoDetails | ||
29 | |||
30 | baseMockUrl?: string | ||
31 | |||
32 | playlistBucket: string | ||
33 | playlistPrefix?: string | ||
34 | |||
35 | webtorrentBucket: string | ||
36 | webtorrentPrefix?: string | ||
37 | }) { | ||
38 | const { | ||
39 | video, | ||
40 | playlistBucket, | ||
41 | webtorrentBucket, | ||
42 | baseMockUrl, | ||
43 | playlistPrefix, | ||
44 | webtorrentPrefix | ||
45 | } = options | ||
46 | |||
47 | let allFiles = video.files | ||
48 | |||
49 | for (const file of video.files) { | ||
50 | const baseUrl = baseMockUrl | ||
51 | ? `${baseMockUrl}/${webtorrentBucket}/` | ||
52 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | ||
53 | |||
54 | const prefix = webtorrentPrefix || '' | ||
55 | const start = baseUrl + prefix | ||
56 | |||
57 | expectStartWith(file.fileUrl, start) | ||
58 | |||
59 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | ||
60 | const location = res.headers['location'] | ||
61 | expectStartWith(location, start) | ||
62 | |||
63 | await makeRawRequest(location, HttpStatusCode.OK_200) | ||
64 | } | ||
65 | |||
66 | const hls = video.streamingPlaylists[0] | ||
67 | |||
68 | if (hls) { | ||
69 | allFiles = allFiles.concat(hls.files) | ||
70 | |||
71 | const baseUrl = baseMockUrl | ||
72 | ? `${baseMockUrl}/${playlistBucket}/` | ||
73 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | ||
74 | |||
75 | const prefix = playlistPrefix || '' | ||
76 | const start = baseUrl + prefix | ||
77 | |||
78 | expectStartWith(hls.playlistUrl, start) | ||
79 | expectStartWith(hls.segmentsSha256Url, start) | ||
80 | |||
81 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | ||
82 | |||
83 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | ||
84 | expect(JSON.stringify(resSha.body)).to.not.throw | ||
85 | |||
86 | for (const file of hls.files) { | ||
87 | expectStartWith(file.fileUrl, start) | ||
88 | |||
89 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | ||
90 | const location = res.headers['location'] | ||
91 | expectStartWith(location, start) | ||
92 | |||
93 | await makeRawRequest(location, HttpStatusCode.OK_200) | ||
94 | } | ||
95 | } | ||
96 | |||
97 | for (const file of allFiles) { | ||
98 | const torrent = await webtorrentAdd(file.magnetUri, true) | ||
99 | |||
100 | expect(torrent.files).to.be.an('array') | ||
101 | expect(torrent.files.length).to.equal(1) | ||
102 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | ||
103 | |||
104 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
105 | expect(res.body).to.have.length.above(100) | ||
106 | } | ||
107 | |||
108 | return allFiles.map(f => f.fileUrl) | ||
109 | } | ||
110 | |||
111 | function runTestSuite (options: { | ||
112 | playlistBucket: string | ||
113 | playlistPrefix?: string | ||
114 | |||
115 | webtorrentBucket: string | ||
116 | webtorrentPrefix?: string | ||
117 | |||
118 | useMockBaseUrl?: boolean | ||
119 | |||
120 | maxUploadPart?: string | ||
121 | }) { | ||
122 | const mockObjectStorage = new MockObjectStorage() | ||
123 | let baseMockUrl: string | ||
124 | |||
125 | let servers: PeerTubeServer[] | ||
126 | |||
127 | let keptUrls: string[] = [] | ||
128 | |||
129 | const uuidsToDelete: string[] = [] | ||
130 | let deletedUrls: string[] = [] | ||
131 | |||
132 | before(async function () { | ||
133 | this.timeout(120000) | ||
134 | |||
135 | const port = await mockObjectStorage.initialize() | ||
136 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | ||
137 | |||
138 | await ObjectStorageCommand.createBucket(options.playlistBucket) | ||
139 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | ||
140 | |||
141 | const config = { | ||
142 | object_storage: { | ||
143 | enabled: true, | ||
144 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | ||
145 | region: ObjectStorageCommand.getRegion(), | ||
146 | |||
147 | credentials: ObjectStorageCommand.getCredentialsConfig(), | ||
148 | |||
149 | max_upload_part: options.maxUploadPart || '2MB', | ||
150 | |||
151 | streaming_playlists: { | ||
152 | bucket_name: options.playlistBucket, | ||
153 | prefix: options.playlistPrefix, | ||
154 | base_url: baseMockUrl | ||
155 | ? `${baseMockUrl}/${options.playlistBucket}` | ||
156 | : undefined | ||
157 | }, | ||
158 | |||
159 | videos: { | ||
160 | bucket_name: options.webtorrentBucket, | ||
161 | prefix: options.webtorrentPrefix, | ||
162 | base_url: baseMockUrl | ||
163 | ? `${baseMockUrl}/${options.webtorrentBucket}` | ||
164 | : undefined | ||
165 | } | ||
166 | } | ||
167 | } | ||
168 | |||
169 | servers = await createMultipleServers(2, config) | ||
170 | |||
171 | await setAccessTokensToServers(servers) | ||
172 | await doubleFollow(servers[0], servers[1]) | ||
173 | |||
174 | for (const server of servers) { | ||
175 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | ||
176 | await waitJobs(servers) | ||
177 | |||
178 | const files = await server.videos.listFiles({ id: uuid }) | ||
179 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | ||
180 | } | ||
181 | }) | ||
182 | |||
183 | it('Should upload a video and move it to the object storage without transcoding', async function () { | ||
184 | this.timeout(20000) | ||
185 | |||
186 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | ||
187 | uuidsToDelete.push(uuid) | ||
188 | |||
189 | await waitJobs(servers) | ||
190 | |||
191 | for (const server of servers) { | ||
192 | const video = await server.videos.get({ id: uuid }) | ||
193 | const files = await checkFiles({ ...options, video, baseMockUrl }) | ||
194 | |||
195 | deletedUrls = deletedUrls.concat(files) | ||
196 | } | ||
197 | }) | ||
198 | |||
199 | it('Should upload a video and move it to the object storage with transcoding', async function () { | ||
200 | this.timeout(40000) | ||
201 | |||
202 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' }) | ||
203 | uuidsToDelete.push(uuid) | ||
204 | |||
205 | await waitJobs(servers) | ||
206 | |||
207 | for (const server of servers) { | ||
208 | const video = await server.videos.get({ id: uuid }) | ||
209 | const files = await checkFiles({ ...options, video, baseMockUrl }) | ||
210 | |||
211 | deletedUrls = deletedUrls.concat(files) | ||
212 | } | ||
213 | }) | ||
214 | |||
215 | it('Should correctly delete the files', async function () { | ||
216 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | ||
217 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | ||
218 | |||
219 | await waitJobs(servers) | ||
220 | |||
221 | for (const url of deletedUrls) { | ||
222 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | ||
223 | } | ||
224 | }) | ||
225 | |||
226 | it('Should have kept other files', async function () { | ||
227 | for (const url of keptUrls) { | ||
228 | await makeRawRequest(url, HttpStatusCode.OK_200) | ||
229 | } | ||
230 | }) | ||
231 | |||
232 | it('Should have an empty tmp directory', async function () { | ||
233 | for (const server of servers) { | ||
234 | await checkTmpIsEmpty(server) | ||
235 | } | ||
236 | }) | ||
237 | |||
238 | after(async function () { | ||
239 | mockObjectStorage.terminate() | ||
240 | |||
241 | await cleanupTests(servers) | ||
242 | }) | ||
243 | } | ||
244 | |||
245 | describe('Object storage for videos', function () { | ||
246 | if (areObjectStorageTestsDisabled()) return | ||
247 | |||
248 | describe('Test config', function () { | ||
249 | let server: PeerTubeServer | ||
250 | |||
251 | const baseConfig = { | ||
252 | object_storage: { | ||
253 | enabled: true, | ||
254 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | ||
255 | region: ObjectStorageCommand.getRegion(), | ||
256 | |||
257 | credentials: ObjectStorageCommand.getCredentialsConfig(), | ||
258 | |||
259 | streaming_playlists: { | ||
260 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | ||
261 | }, | ||
262 | |||
263 | videos: { | ||
264 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | ||
265 | } | ||
266 | } | ||
267 | } | ||
268 | |||
269 | const badCredentials = { | ||
270 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
271 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
272 | } | ||
273 | |||
274 | it('Should fail with same bucket names without prefix', function (done) { | ||
275 | const config = merge({}, baseConfig, { | ||
276 | object_storage: { | ||
277 | streaming_playlists: { | ||
278 | bucket_name: 'aaa' | ||
279 | }, | ||
280 | |||
281 | videos: { | ||
282 | bucket_name: 'aaa' | ||
283 | } | ||
284 | } | ||
285 | }) | ||
286 | |||
287 | createSingleServer(1, config) | ||
288 | .then(() => done(new Error('Did not throw'))) | ||
289 | .catch(() => done()) | ||
290 | }) | ||
291 | |||
292 | it('Should fail with bad credentials', async function () { | ||
293 | this.timeout(60000) | ||
294 | |||
295 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
296 | |||
297 | const config = merge({}, baseConfig, { | ||
298 | object_storage: { | ||
299 | credentials: badCredentials | ||
300 | } | ||
301 | }) | ||
302 | |||
303 | server = await createSingleServer(1, config) | ||
304 | await setAccessTokensToServers([ server ]) | ||
305 | |||
306 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
307 | |||
308 | await waitJobs([ server ], true) | ||
309 | const video = await server.videos.get({ id: uuid }) | ||
310 | |||
311 | expectStartWith(video.files[0].fileUrl, server.url) | ||
312 | |||
313 | await killallServers([ server ]) | ||
314 | }) | ||
315 | |||
316 | it('Should succeed with credentials from env', async function () { | ||
317 | this.timeout(60000) | ||
318 | |||
319 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
320 | |||
321 | const config = merge({}, baseConfig, { | ||
322 | object_storage: { | ||
323 | credentials: { | ||
324 | access_key_id: '', | ||
325 | secret_access_key: '' | ||
326 | } | ||
327 | } | ||
328 | }) | ||
329 | |||
330 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | ||
331 | |||
332 | server = await createSingleServer(1, config, { | ||
333 | env: { | ||
334 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | ||
335 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | ||
336 | } | ||
337 | }) | ||
338 | |||
339 | await setAccessTokensToServers([ server ]) | ||
340 | |||
341 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
342 | |||
343 | await waitJobs([ server ], true) | ||
344 | const video = await server.videos.get({ id: uuid }) | ||
345 | |||
346 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
347 | }) | ||
348 | |||
349 | after(async function () { | ||
350 | await killallServers([ server ]) | ||
351 | }) | ||
352 | }) | ||
353 | |||
354 | describe('Test simple object storage', function () { | ||
355 | runTestSuite({ | ||
356 | playlistBucket: 'streaming-playlists', | ||
357 | webtorrentBucket: 'videos' | ||
358 | }) | ||
359 | }) | ||
360 | |||
361 | describe('Test object storage with prefix', function () { | ||
362 | runTestSuite({ | ||
363 | playlistBucket: 'mybucket', | ||
364 | webtorrentBucket: 'mybucket', | ||
365 | |||
366 | playlistPrefix: 'streaming-playlists_', | ||
367 | webtorrentPrefix: 'webtorrent_' | ||
368 | }) | ||
369 | }) | ||
370 | |||
371 | describe('Test object storage with prefix and base URL', function () { | ||
372 | runTestSuite({ | ||
373 | playlistBucket: 'mybucket', | ||
374 | webtorrentBucket: 'mybucket', | ||
375 | |||
376 | playlistPrefix: 'streaming-playlists_', | ||
377 | webtorrentPrefix: 'webtorrent_', | ||
378 | |||
379 | useMockBaseUrl: true | ||
380 | }) | ||
381 | }) | ||
382 | |||
383 | describe('Test object storage with small upload part', function () { | ||
384 | runTestSuite({ | ||
385 | playlistBucket: 'streaming-playlists', | ||
386 | webtorrentBucket: 'videos', | ||
387 | |||
388 | maxUploadPart: '5KB' | ||
389 | }) | ||
390 | }) | ||
391 | }) | ||
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index e1a12f5f8..3400b1d9a 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -207,14 +207,14 @@ async function check1PlaylistRedundancies (videoUUID?: string) { | |||
207 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) | 207 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) |
208 | } | 208 | } |
209 | 209 | ||
210 | const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' | 210 | const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID |
211 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls' | 211 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID |
212 | 212 | ||
213 | const video = await servers[0].videos.get({ id: videoUUID }) | 213 | const video = await servers[0].videos.get({ id: videoUUID }) |
214 | const hlsPlaylist = video.streamingPlaylists[0] | 214 | const hlsPlaylist = video.streamingPlaylists[0] |
215 | 215 | ||
216 | for (const resolution of [ 240, 360, 480, 720 ]) { | 216 | for (const resolution of [ 240, 360, 480, 720 ]) { |
217 | await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) | 217 | await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist }) |
218 | } | 218 | } |
219 | 219 | ||
220 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) | 220 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) |
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 961f0e617..2c829f532 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts | |||
@@ -5,6 +5,7 @@ import * as chai from 'chai' | |||
5 | import { basename, join } from 'path' | 5 | import { basename, join } from 'path' |
6 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' | 6 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' |
7 | import { | 7 | import { |
8 | areObjectStorageTestsDisabled, | ||
8 | checkDirectoryIsEmpty, | 9 | checkDirectoryIsEmpty, |
9 | checkResolutionsInMasterPlaylist, | 10 | checkResolutionsInMasterPlaylist, |
10 | checkSegmentHash, | 11 | checkSegmentHash, |
@@ -12,7 +13,9 @@ import { | |||
12 | cleanupTests, | 13 | cleanupTests, |
13 | createMultipleServers, | 14 | createMultipleServers, |
14 | doubleFollow, | 15 | doubleFollow, |
16 | expectStartWith, | ||
15 | makeRawRequest, | 17 | makeRawRequest, |
18 | ObjectStorageCommand, | ||
16 | PeerTubeServer, | 19 | PeerTubeServer, |
17 | setAccessTokensToServers, | 20 | setAccessTokensToServers, |
18 | waitJobs, | 21 | waitJobs, |
@@ -23,8 +26,19 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' | |||
23 | 26 | ||
24 | const expect = chai.expect | 27 | const expect = chai.expect |
25 | 28 | ||
26 | async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { | 29 | async function checkHlsPlaylist (options: { |
27 | for (const server of servers) { | 30 | servers: PeerTubeServer[] |
31 | videoUUID: string | ||
32 | hlsOnly: boolean | ||
33 | |||
34 | resolutions?: number[] | ||
35 | objectStorageBaseUrl: string | ||
36 | }) { | ||
37 | const { videoUUID, hlsOnly, objectStorageBaseUrl } = options | ||
38 | |||
39 | const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] | ||
40 | |||
41 | for (const server of options.servers) { | ||
28 | const videoDetails = await server.videos.get({ id: videoUUID }) | 42 | const videoDetails = await server.videos.get({ id: videoUUID }) |
29 | const baseUrl = `http://${videoDetails.account.host}` | 43 | const baseUrl = `http://${videoDetails.account.host}` |
30 | 44 | ||
@@ -48,9 +62,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
48 | expect(file.torrentUrl).to.match( | 62 | expect(file.torrentUrl).to.match( |
49 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) | 63 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) |
50 | ) | 64 | ) |
51 | expect(file.fileUrl).to.match( | 65 | |
52 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | 66 | if (objectStorageBaseUrl) { |
53 | ) | 67 | expectStartWith(file.fileUrl, objectStorageBaseUrl) |
68 | } else { | ||
69 | expect(file.fileUrl).to.match( | ||
70 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | ||
71 | ) | ||
72 | } | ||
73 | |||
54 | expect(file.resolution.label).to.equal(resolution + 'p') | 74 | expect(file.resolution.label).to.equal(resolution + 'p') |
55 | 75 | ||
56 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) | 76 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) |
@@ -80,9 +100,11 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
80 | const file = hlsFiles.find(f => f.resolution.id === resolution) | 100 | const file = hlsFiles.find(f => f.resolution.id === resolution) |
81 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' | 101 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' |
82 | 102 | ||
83 | const subPlaylist = await server.streamingPlaylists.get({ | 103 | const url = objectStorageBaseUrl |
84 | url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` | 104 | ? `${objectStorageBaseUrl}hls_${videoUUID}/${playlistName}` |
85 | }) | 105 | : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` |
106 | |||
107 | const subPlaylist = await server.streamingPlaylists.get({ url }) | ||
86 | 108 | ||
87 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) | 109 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) |
88 | expect(subPlaylist).to.contain(basename(file.fileUrl)) | 110 | expect(subPlaylist).to.contain(basename(file.fileUrl)) |
@@ -90,14 +112,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
90 | } | 112 | } |
91 | 113 | ||
92 | { | 114 | { |
93 | const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' | 115 | const baseUrlAndPath = objectStorageBaseUrl |
116 | ? objectStorageBaseUrl + 'hls_' + videoUUID | ||
117 | : baseUrl + '/static/streaming-playlists/hls/' + videoUUID | ||
94 | 118 | ||
95 | for (const resolution of resolutions) { | 119 | for (const resolution of resolutions) { |
96 | await checkSegmentHash({ | 120 | await checkSegmentHash({ |
97 | server, | 121 | server, |
98 | baseUrlPlaylist: baseUrlAndPath, | 122 | baseUrlPlaylist: baseUrlAndPath, |
99 | baseUrlSegment: baseUrlAndPath, | 123 | baseUrlSegment: baseUrlAndPath, |
100 | videoUUID, | ||
101 | resolution, | 124 | resolution, |
102 | hlsPlaylist | 125 | hlsPlaylist |
103 | }) | 126 | }) |
@@ -111,7 +134,7 @@ describe('Test HLS videos', function () { | |||
111 | let videoUUID = '' | 134 | let videoUUID = '' |
112 | let videoAudioUUID = '' | 135 | let videoAudioUUID = '' |
113 | 136 | ||
114 | function runTestSuite (hlsOnly: boolean) { | 137 | function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { |
115 | 138 | ||
116 | it('Should upload a video and transcode it to HLS', async function () { | 139 | it('Should upload a video and transcode it to HLS', async function () { |
117 | this.timeout(120000) | 140 | this.timeout(120000) |
@@ -121,7 +144,7 @@ describe('Test HLS videos', function () { | |||
121 | 144 | ||
122 | await waitJobs(servers) | 145 | await waitJobs(servers) |
123 | 146 | ||
124 | await checkHlsPlaylist(servers, videoUUID, hlsOnly) | 147 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) |
125 | }) | 148 | }) |
126 | 149 | ||
127 | it('Should upload an audio file and transcode it to HLS', async function () { | 150 | it('Should upload an audio file and transcode it to HLS', async function () { |
@@ -132,7 +155,13 @@ describe('Test HLS videos', function () { | |||
132 | 155 | ||
133 | await waitJobs(servers) | 156 | await waitJobs(servers) |
134 | 157 | ||
135 | await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ]) | 158 | await checkHlsPlaylist({ |
159 | servers, | ||
160 | videoUUID: videoAudioUUID, | ||
161 | hlsOnly, | ||
162 | resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], | ||
163 | objectStorageBaseUrl | ||
164 | }) | ||
136 | }) | 165 | }) |
137 | 166 | ||
138 | it('Should update the video', async function () { | 167 | it('Should update the video', async function () { |
@@ -142,7 +171,7 @@ describe('Test HLS videos', function () { | |||
142 | 171 | ||
143 | await waitJobs(servers) | 172 | await waitJobs(servers) |
144 | 173 | ||
145 | await checkHlsPlaylist(servers, videoUUID, hlsOnly) | 174 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) |
146 | }) | 175 | }) |
147 | 176 | ||
148 | it('Should delete videos', async function () { | 177 | it('Should delete videos', async function () { |
@@ -229,6 +258,22 @@ describe('Test HLS videos', function () { | |||
229 | runTestSuite(true) | 258 | runTestSuite(true) |
230 | }) | 259 | }) |
231 | 260 | ||
261 | describe('With object storage enabled', function () { | ||
262 | if (areObjectStorageTestsDisabled()) return | ||
263 | |||
264 | before(async function () { | ||
265 | this.timeout(120000) | ||
266 | |||
267 | const configOverride = ObjectStorageCommand.getDefaultConfig() | ||
268 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
269 | |||
270 | await servers[0].kill() | ||
271 | await servers[0].run(configOverride) | ||
272 | }) | ||
273 | |||
274 | runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
275 | }) | ||
276 | |||
232 | after(async function () { | 277 | after(async function () { |
233 | await cleanupTests(servers) | 278 | await cleanupTests(servers) |
234 | }) | 279 | }) |
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index bddcff5e7..9f1b57a2e 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts | |||
@@ -2,8 +2,19 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' | 5 | import { |
6 | import { VideoFile } from '@shared/models' | 6 | areObjectStorageTestsDisabled, |
7 | cleanupTests, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | waitJobs | ||
16 | } from '@shared/extra-utils' | ||
17 | import { HttpStatusCode, VideoDetails, VideoFile } from '@shared/models' | ||
7 | 18 | ||
8 | const expect = chai.expect | 19 | const expect = chai.expect |
9 | 20 | ||
@@ -17,22 +28,35 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s | |||
17 | if (size) expect(video.size).to.equal(size) | 28 | if (size) expect(video.size).to.equal(size) |
18 | } | 29 | } |
19 | 30 | ||
20 | describe('Test create import video jobs', function () { | 31 | async function checkFiles (video: VideoDetails, objectStorage: boolean) { |
21 | this.timeout(60000) | 32 | for (const file of video.files) { |
33 | if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
22 | 34 | ||
23 | let servers: PeerTubeServer[] = [] | 35 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
36 | } | ||
37 | } | ||
38 | |||
39 | function runTests (objectStorage: boolean) { | ||
24 | let video1UUID: string | 40 | let video1UUID: string |
25 | let video2UUID: string | 41 | let video2UUID: string |
26 | 42 | ||
43 | let servers: PeerTubeServer[] = [] | ||
44 | |||
27 | before(async function () { | 45 | before(async function () { |
28 | this.timeout(90000) | 46 | this.timeout(90000) |
29 | 47 | ||
48 | const config = objectStorage | ||
49 | ? ObjectStorageCommand.getDefaultConfig() | ||
50 | : {} | ||
51 | |||
30 | // Run server 2 to have transcoding enabled | 52 | // Run server 2 to have transcoding enabled |
31 | servers = await createMultipleServers(2) | 53 | servers = await createMultipleServers(2, config) |
32 | await setAccessTokensToServers(servers) | 54 | await setAccessTokensToServers(servers) |
33 | 55 | ||
34 | await doubleFollow(servers[0], servers[1]) | 56 | await doubleFollow(servers[0], servers[1]) |
35 | 57 | ||
58 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | ||
59 | |||
36 | // Upload two videos for our needs | 60 | // Upload two videos for our needs |
37 | { | 61 | { |
38 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) | 62 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) |
@@ -44,7 +68,6 @@ describe('Test create import video jobs', function () { | |||
44 | video2UUID = uuid | 68 | video2UUID = uuid |
45 | } | 69 | } |
46 | 70 | ||
47 | // Transcoding | ||
48 | await waitJobs(servers) | 71 | await waitJobs(servers) |
49 | }) | 72 | }) |
50 | 73 | ||
@@ -65,6 +88,8 @@ describe('Test create import video jobs', function () { | |||
65 | const [ originalVideo, transcodedVideo ] = videoDetails.files | 88 | const [ originalVideo, transcodedVideo ] = videoDetails.files |
66 | assertVideoProperties(originalVideo, 720, 'webm', 218910) | 89 | assertVideoProperties(originalVideo, 720, 'webm', 218910) |
67 | assertVideoProperties(transcodedVideo, 480, 'webm', 69217) | 90 | assertVideoProperties(transcodedVideo, 480, 'webm', 69217) |
91 | |||
92 | await checkFiles(videoDetails, objectStorage) | ||
68 | } | 93 | } |
69 | }) | 94 | }) |
70 | 95 | ||
@@ -87,6 +112,8 @@ describe('Test create import video jobs', function () { | |||
87 | assertVideoProperties(transcodedVideo420, 480, 'mp4') | 112 | assertVideoProperties(transcodedVideo420, 480, 'mp4') |
88 | assertVideoProperties(transcodedVideo320, 360, 'mp4') | 113 | assertVideoProperties(transcodedVideo320, 360, 'mp4') |
89 | assertVideoProperties(transcodedVideo240, 240, 'mp4') | 114 | assertVideoProperties(transcodedVideo240, 240, 'mp4') |
115 | |||
116 | await checkFiles(videoDetails, objectStorage) | ||
90 | } | 117 | } |
91 | }) | 118 | }) |
92 | 119 | ||
@@ -107,10 +134,25 @@ describe('Test create import video jobs', function () { | |||
107 | const [ video720, video480 ] = videoDetails.files | 134 | const [ video720, video480 ] = videoDetails.files |
108 | assertVideoProperties(video720, 720, 'webm', 942961) | 135 | assertVideoProperties(video720, 720, 'webm', 942961) |
109 | assertVideoProperties(video480, 480, 'webm', 69217) | 136 | assertVideoProperties(video480, 480, 'webm', 69217) |
137 | |||
138 | await checkFiles(videoDetails, objectStorage) | ||
110 | } | 139 | } |
111 | }) | 140 | }) |
112 | 141 | ||
113 | after(async function () { | 142 | after(async function () { |
114 | await cleanupTests(servers) | 143 | await cleanupTests(servers) |
115 | }) | 144 | }) |
145 | } | ||
146 | |||
147 | describe('Test create import video jobs', function () { | ||
148 | |||
149 | describe('On filesystem', function () { | ||
150 | runTests(false) | ||
151 | }) | ||
152 | |||
153 | describe('On object storage', function () { | ||
154 | if (areObjectStorageTestsDisabled()) return | ||
155 | |||
156 | runTests(true) | ||
157 | }) | ||
116 | }) | 158 | }) |
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts index df787ccdc..3313a492f 100644 --- a/server/tests/cli/create-transcoding-job.ts +++ b/server/tests/cli/create-transcoding-job.ts | |||
@@ -2,10 +2,15 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { HttpStatusCode, VideoFile } from '@shared/models' | ||
5 | import { | 6 | import { |
7 | areObjectStorageTestsDisabled, | ||
6 | cleanupTests, | 8 | cleanupTests, |
7 | createMultipleServers, | 9 | createMultipleServers, |
8 | doubleFollow, | 10 | doubleFollow, |
11 | expectStartWith, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
9 | PeerTubeServer, | 14 | PeerTubeServer, |
10 | setAccessTokensToServers, | 15 | setAccessTokensToServers, |
11 | waitJobs | 16 | waitJobs |
@@ -13,39 +18,39 @@ import { | |||
13 | 18 | ||
14 | const expect = chai.expect | 19 | const expect = chai.expect |
15 | 20 | ||
16 | describe('Test create transcoding jobs', function () { | 21 | async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { |
17 | let servers: PeerTubeServer[] = [] | 22 | for (const file of files) { |
18 | const videosUUID: string[] = [] | 23 | const shouldStartWith = type === 'webtorrent' |
24 | ? ObjectStorageCommand.getWebTorrentBaseUrl() | ||
25 | : ObjectStorageCommand.getPlaylistBaseUrl() | ||
19 | 26 | ||
20 | const config = { | 27 | expectStartWith(file.fileUrl, shouldStartWith) |
21 | transcoding: { | 28 | |
22 | enabled: false, | 29 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
23 | resolutions: { | ||
24 | '240p': true, | ||
25 | '360p': true, | ||
26 | '480p': true, | ||
27 | '720p': true, | ||
28 | '1080p': true, | ||
29 | '1440p': true, | ||
30 | '2160p': true | ||
31 | }, | ||
32 | hls: { | ||
33 | enabled: false | ||
34 | } | ||
35 | } | ||
36 | } | 30 | } |
31 | } | ||
32 | |||
33 | function runTests (objectStorage: boolean) { | ||
34 | let servers: PeerTubeServer[] = [] | ||
35 | const videosUUID: string[] = [] | ||
37 | 36 | ||
38 | before(async function () { | 37 | before(async function () { |
39 | this.timeout(60000) | 38 | this.timeout(60000) |
40 | 39 | ||
40 | const config = objectStorage | ||
41 | ? ObjectStorageCommand.getDefaultConfig() | ||
42 | : {} | ||
43 | |||
41 | // Run server 2 to have transcoding enabled | 44 | // Run server 2 to have transcoding enabled |
42 | servers = await createMultipleServers(2) | 45 | servers = await createMultipleServers(2, config) |
43 | await setAccessTokensToServers(servers) | 46 | await setAccessTokensToServers(servers) |
44 | 47 | ||
45 | await servers[0].config.updateCustomSubConfig({ newConfig: config }) | 48 | await servers[0].config.disableTranscoding() |
46 | 49 | ||
47 | await doubleFollow(servers[0], servers[1]) | 50 | await doubleFollow(servers[0], servers[1]) |
48 | 51 | ||
52 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | ||
53 | |||
49 | for (let i = 1; i <= 5; i++) { | 54 | for (let i = 1; i <= 5; i++) { |
50 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) | 55 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) |
51 | videosUUID.push(uuid) | 56 | videosUUID.push(uuid) |
@@ -81,27 +86,29 @@ describe('Test create transcoding jobs', function () { | |||
81 | let infoHashes: { [id: number]: string } | 86 | let infoHashes: { [id: number]: string } |
82 | 87 | ||
83 | for (const video of data) { | 88 | for (const video of data) { |
84 | const videoDetail = await server.videos.get({ id: video.uuid }) | 89 | const videoDetails = await server.videos.get({ id: video.uuid }) |
85 | 90 | ||
86 | if (video.uuid === videosUUID[1]) { | 91 | if (video.uuid === videosUUID[1]) { |
87 | expect(videoDetail.files).to.have.lengthOf(4) | 92 | expect(videoDetails.files).to.have.lengthOf(4) |
88 | expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) | 93 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
94 | |||
95 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
89 | 96 | ||
90 | if (!infoHashes) { | 97 | if (!infoHashes) { |
91 | infoHashes = {} | 98 | infoHashes = {} |
92 | 99 | ||
93 | for (const file of videoDetail.files) { | 100 | for (const file of videoDetails.files) { |
94 | infoHashes[file.resolution.id.toString()] = file.magnetUri | 101 | infoHashes[file.resolution.id.toString()] = file.magnetUri |
95 | } | 102 | } |
96 | } else { | 103 | } else { |
97 | for (const resolution of Object.keys(infoHashes)) { | 104 | for (const resolution of Object.keys(infoHashes)) { |
98 | const file = videoDetail.files.find(f => f.resolution.id.toString() === resolution) | 105 | const file = videoDetails.files.find(f => f.resolution.id.toString() === resolution) |
99 | expect(file.magnetUri).to.equal(infoHashes[resolution]) | 106 | expect(file.magnetUri).to.equal(infoHashes[resolution]) |
100 | } | 107 | } |
101 | } | 108 | } |
102 | } else { | 109 | } else { |
103 | expect(videoDetail.files).to.have.lengthOf(1) | 110 | expect(videoDetails.files).to.have.lengthOf(1) |
104 | expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) | 111 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
105 | } | 112 | } |
106 | } | 113 | } |
107 | } | 114 | } |
@@ -125,6 +132,8 @@ describe('Test create transcoding jobs', function () { | |||
125 | expect(videoDetails.files[1].resolution.id).to.equal(480) | 132 | expect(videoDetails.files[1].resolution.id).to.equal(480) |
126 | 133 | ||
127 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) | 134 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
135 | |||
136 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
128 | } | 137 | } |
129 | }) | 138 | }) |
130 | 139 | ||
@@ -139,11 +148,15 @@ describe('Test create transcoding jobs', function () { | |||
139 | const videoDetails = await server.videos.get({ id: videosUUID[2] }) | 148 | const videoDetails = await server.videos.get({ id: videosUUID[2] }) |
140 | 149 | ||
141 | expect(videoDetails.files).to.have.lengthOf(1) | 150 | expect(videoDetails.files).to.have.lengthOf(1) |
151 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
152 | |||
142 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | 153 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) |
143 | 154 | ||
144 | const files = videoDetails.streamingPlaylists[0].files | 155 | const files = videoDetails.streamingPlaylists[0].files |
145 | expect(files).to.have.lengthOf(1) | 156 | expect(files).to.have.lengthOf(1) |
146 | expect(files[0].resolution.id).to.equal(480) | 157 | expect(files[0].resolution.id).to.equal(480) |
158 | |||
159 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
147 | } | 160 | } |
148 | }) | 161 | }) |
149 | 162 | ||
@@ -160,6 +173,8 @@ describe('Test create transcoding jobs', function () { | |||
160 | const files = videoDetails.streamingPlaylists[0].files | 173 | const files = videoDetails.streamingPlaylists[0].files |
161 | expect(files).to.have.lengthOf(1) | 174 | expect(files).to.have.lengthOf(1) |
162 | expect(files[0].resolution.id).to.equal(480) | 175 | expect(files[0].resolution.id).to.equal(480) |
176 | |||
177 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
163 | } | 178 | } |
164 | }) | 179 | }) |
165 | 180 | ||
@@ -178,15 +193,15 @@ describe('Test create transcoding jobs', function () { | |||
178 | 193 | ||
179 | const files = videoDetails.streamingPlaylists[0].files | 194 | const files = videoDetails.streamingPlaylists[0].files |
180 | expect(files).to.have.lengthOf(4) | 195 | expect(files).to.have.lengthOf(4) |
196 | |||
197 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
181 | } | 198 | } |
182 | }) | 199 | }) |
183 | 200 | ||
184 | it('Should optimize the video file and generate HLS videos if enabled in config', async function () { | 201 | it('Should optimize the video file and generate HLS videos if enabled in config', async function () { |
185 | this.timeout(120000) | 202 | this.timeout(120000) |
186 | 203 | ||
187 | config.transcoding.hls.enabled = true | 204 | await servers[0].config.enableTranscoding() |
188 | await servers[0].config.updateCustomSubConfig({ newConfig: config }) | ||
189 | |||
190 | await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) | 205 | await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) |
191 | 206 | ||
192 | await waitJobs(servers) | 207 | await waitJobs(servers) |
@@ -197,10 +212,28 @@ describe('Test create transcoding jobs', function () { | |||
197 | expect(videoDetails.files).to.have.lengthOf(4) | 212 | expect(videoDetails.files).to.have.lengthOf(4) |
198 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | 213 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) |
199 | expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) | 214 | expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) |
215 | |||
216 | if (objectStorage) { | ||
217 | await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
218 | await checkFilesInObjectStorage(videoDetails.streamingPlaylists[0].files, 'playlist') | ||
219 | } | ||
200 | } | 220 | } |
201 | }) | 221 | }) |
202 | 222 | ||
203 | after(async function () { | 223 | after(async function () { |
204 | await cleanupTests(servers) | 224 | await cleanupTests(servers) |
205 | }) | 225 | }) |
226 | } | ||
227 | |||
228 | describe('Test create transcoding jobs', function () { | ||
229 | |||
230 | describe('On filesystem', function () { | ||
231 | runTests(false) | ||
232 | }) | ||
233 | |||
234 | describe('On object storage', function () { | ||
235 | if (areObjectStorageTestsDisabled()) return | ||
236 | |||
237 | runTests(true) | ||
238 | }) | ||
206 | }) | 239 | }) |
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts index 7f7873df3..c9a2eb831 100644 --- a/server/tests/helpers/request.ts +++ b/server/tests/helpers/request.ts | |||
@@ -13,7 +13,7 @@ describe('Request helpers', function () { | |||
13 | 13 | ||
14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { | 14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { |
15 | try { | 15 | try { |
16 | await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 3 }) | 16 | await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 }) |
17 | } catch { | 17 | } catch { |
18 | return | 18 | return |
19 | } | 19 | } |
@@ -23,7 +23,7 @@ describe('Request helpers', function () { | |||
23 | 23 | ||
24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { | 24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { |
25 | try { | 25 | try { |
26 | await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath1, { bodyKBLimit: 3 }) | 26 | await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 }) |
27 | } catch { | 27 | } catch { |
28 | 28 | ||
29 | await wait(500) | 29 | await wait(500) |
@@ -35,8 +35,8 @@ describe('Request helpers', function () { | |||
35 | }) | 35 | }) |
36 | 36 | ||
37 | it('Should succeed if the file is below the limit', async function () { | 37 | it('Should succeed if the file is below the limit', async function () { |
38 | await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 5 }) | 38 | await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 }) |
39 | await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath2, { bodyKBLimit: 5 }) | 39 | await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 }) |
40 | 40 | ||
41 | expect(await pathExists(destPath2)).to.be.true | 41 | expect(await pathExists(destPath2)).to.be.true |
42 | }) | 42 | }) |
diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index 7fc92f804..aa2c8e8fa 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts | |||
@@ -16,6 +16,10 @@ function dateIsValid (dateString: string, interval = 300000) { | |||
16 | return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval | 16 | return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval |
17 | } | 17 | } |
18 | 18 | ||
19 | function expectStartWith (str: string, start: string) { | ||
20 | expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true | ||
21 | } | ||
22 | |||
19 | async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { | 23 | async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { |
20 | const res = await makeGetRequest({ | 24 | const res = await makeGetRequest({ |
21 | url, | 25 | url, |
@@ -42,5 +46,6 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f | |||
42 | export { | 46 | export { |
43 | dateIsValid, | 47 | dateIsValid, |
44 | testImage, | 48 | testImage, |
45 | testFileExistsOrNot | 49 | testFileExistsOrNot, |
50 | expectStartWith | ||
46 | } | 51 | } |
diff --git a/shared/extra-utils/miscs/tests.ts b/shared/extra-utils/miscs/tests.ts index 3dfb2487e..dd86041fe 100644 --- a/shared/extra-utils/miscs/tests.ts +++ b/shared/extra-utils/miscs/tests.ts | |||
@@ -28,7 +28,9 @@ const FIXTURE_URLS = { | |||
28 | 28 | ||
29 | badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', | 29 | badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', |
30 | goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', | 30 | goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', |
31 | video4K: 'https://download.cpy.re/peertube/4k_file.txt' | 31 | goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4', |
32 | |||
33 | file4K: 'https://download.cpy.re/peertube/4k_file.txt' | ||
32 | } | 34 | } |
33 | 35 | ||
34 | function parallelTests () { | 36 | function parallelTests () { |
@@ -42,7 +44,15 @@ function isGithubCI () { | |||
42 | function areHttpImportTestsDisabled () { | 44 | function areHttpImportTestsDisabled () { |
43 | const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' | 45 | const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' |
44 | 46 | ||
45 | if (disabled) console.log('Import tests are disabled') | 47 | if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled') |
48 | |||
49 | return disabled | ||
50 | } | ||
51 | |||
52 | function areObjectStorageTestsDisabled () { | ||
53 | const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' | ||
54 | |||
55 | if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') | ||
46 | 56 | ||
47 | return disabled | 57 | return disabled |
48 | } | 58 | } |
@@ -89,6 +99,7 @@ export { | |||
89 | buildAbsoluteFixturePath, | 99 | buildAbsoluteFixturePath, |
90 | getFileSize, | 100 | getFileSize, |
91 | buildRequestStub, | 101 | buildRequestStub, |
102 | areObjectStorageTestsDisabled, | ||
92 | wait, | 103 | wait, |
93 | root | 104 | root |
94 | } | 105 | } |
diff --git a/shared/extra-utils/mock-servers/index.ts b/shared/extra-utils/mock-servers/index.ts index 0ec07f685..93c00c788 100644 --- a/shared/extra-utils/mock-servers/index.ts +++ b/shared/extra-utils/mock-servers/index.ts | |||
@@ -2,3 +2,4 @@ export * from './mock-email' | |||
2 | export * from './mock-instances-index' | 2 | export * from './mock-instances-index' |
3 | export * from './mock-joinpeertube-versions' | 3 | export * from './mock-joinpeertube-versions' |
4 | export * from './mock-plugin-blocklist' | 4 | export * from './mock-plugin-blocklist' |
5 | export * from './mock-object-storage' | ||
diff --git a/shared/extra-utils/mock-servers/mock-object-storage.ts b/shared/extra-utils/mock-servers/mock-object-storage.ts new file mode 100644 index 000000000..19ea7c87c --- /dev/null +++ b/shared/extra-utils/mock-servers/mock-object-storage.ts | |||
@@ -0,0 +1,42 @@ | |||
1 | import * as express from 'express' | ||
2 | import got, { RequestError } from 'got' | ||
3 | import { Server } from 'http' | ||
4 | import { pipeline } from 'stream' | ||
5 | import { randomInt } from '@shared/core-utils' | ||
6 | import { ObjectStorageCommand } from '../server' | ||
7 | |||
8 | export class MockObjectStorage { | ||
9 | private server: Server | ||
10 | |||
11 | initialize () { | ||
12 | return new Promise<number>(res => { | ||
13 | const app = express() | ||
14 | |||
15 | app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
16 | const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` | ||
17 | |||
18 | if (process.env.DEBUG) { | ||
19 | console.log('Receiving request on mocked server %s.', req.url) | ||
20 | console.log('Proxifying request to %s', url) | ||
21 | } | ||
22 | |||
23 | return pipeline( | ||
24 | got.stream(url, { throwHttpErrors: false }), | ||
25 | res, | ||
26 | (err: RequestError) => { | ||
27 | if (!err) return | ||
28 | |||
29 | console.error('Pipeline failed.', err) | ||
30 | } | ||
31 | ) | ||
32 | }) | ||
33 | |||
34 | const port = 42301 + randomInt(1, 100) | ||
35 | this.server = app.listen(port, () => res(port)) | ||
36 | }) | ||
37 | } | ||
38 | |||
39 | terminate () { | ||
40 | if (this.server) this.server.close() | ||
41 | } | ||
42 | } | ||
diff --git a/shared/extra-utils/requests/requests.ts b/shared/extra-utils/requests/requests.ts index 70f790222..e3ecd1af2 100644 --- a/shared/extra-utils/requests/requests.ts +++ b/shared/extra-utils/requests/requests.ts | |||
@@ -121,6 +121,20 @@ function unwrapText (test: request.Test): Promise<string> { | |||
121 | return test.then(res => res.text) | 121 | return test.then(res => res.text) |
122 | } | 122 | } |
123 | 123 | ||
124 | function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> { | ||
125 | return test.then(res => { | ||
126 | if (res.body instanceof Buffer) { | ||
127 | return JSON.parse(new TextDecoder().decode(res.body)) | ||
128 | } | ||
129 | |||
130 | return res.body | ||
131 | }) | ||
132 | } | ||
133 | |||
134 | function unwrapTextOrDecode (test: request.Test): Promise<string> { | ||
135 | return test.then(res => res.text || new TextDecoder().decode(res.body)) | ||
136 | } | ||
137 | |||
124 | // --------------------------------------------------------------------------- | 138 | // --------------------------------------------------------------------------- |
125 | 139 | ||
126 | export { | 140 | export { |
@@ -134,6 +148,8 @@ export { | |||
134 | makeRawRequest, | 148 | makeRawRequest, |
135 | makeActivityPubGetRequest, | 149 | makeActivityPubGetRequest, |
136 | unwrapBody, | 150 | unwrapBody, |
151 | unwrapTextOrDecode, | ||
152 | unwrapBodyOrDecodeToJSON, | ||
137 | unwrapText | 153 | unwrapText |
138 | } | 154 | } |
139 | 155 | ||
diff --git a/shared/extra-utils/server/config-command.ts b/shared/extra-utils/server/config-command.ts index 11148aa46..51d04fa63 100644 --- a/shared/extra-utils/server/config-command.ts +++ b/shared/extra-utils/server/config-command.ts | |||
@@ -18,6 +18,70 @@ export class ConfigCommand extends AbstractCommand { | |||
18 | } | 18 | } |
19 | } | 19 | } |
20 | 20 | ||
21 | enableImports () { | ||
22 | return this.updateExistingSubConfig({ | ||
23 | newConfig: { | ||
24 | import: { | ||
25 | videos: { | ||
26 | http: { | ||
27 | enabled: true | ||
28 | }, | ||
29 | |||
30 | torrent: { | ||
31 | enabled: true | ||
32 | } | ||
33 | } | ||
34 | } | ||
35 | } | ||
36 | }) | ||
37 | } | ||
38 | |||
39 | enableLive (options: { | ||
40 | allowReplay?: boolean | ||
41 | transcoding?: boolean | ||
42 | } = {}) { | ||
43 | return this.updateExistingSubConfig({ | ||
44 | newConfig: { | ||
45 | live: { | ||
46 | enabled: true, | ||
47 | allowReplay: options.allowReplay ?? true, | ||
48 | transcoding: { | ||
49 | enabled: options.transcoding ?? true, | ||
50 | resolutions: ConfigCommand.getCustomConfigResolutions(true) | ||
51 | } | ||
52 | } | ||
53 | } | ||
54 | }) | ||
55 | } | ||
56 | |||
57 | disableTranscoding () { | ||
58 | return this.updateExistingSubConfig({ | ||
59 | newConfig: { | ||
60 | transcoding: { | ||
61 | enabled: false | ||
62 | } | ||
63 | } | ||
64 | }) | ||
65 | } | ||
66 | |||
67 | enableTranscoding (webtorrent = true, hls = true) { | ||
68 | return this.updateExistingSubConfig({ | ||
69 | newConfig: { | ||
70 | transcoding: { | ||
71 | enabled: true, | ||
72 | resolutions: ConfigCommand.getCustomConfigResolutions(true), | ||
73 | |||
74 | webtorrent: { | ||
75 | enabled: webtorrent | ||
76 | }, | ||
77 | hls: { | ||
78 | enabled: hls | ||
79 | } | ||
80 | } | ||
81 | } | ||
82 | }) | ||
83 | } | ||
84 | |||
21 | getConfig (options: OverrideCommandOptions = {}) { | 85 | getConfig (options: OverrideCommandOptions = {}) { |
22 | const path = '/api/v1/config' | 86 | const path = '/api/v1/config' |
23 | 87 | ||
@@ -81,6 +145,14 @@ export class ConfigCommand extends AbstractCommand { | |||
81 | }) | 145 | }) |
82 | } | 146 | } |
83 | 147 | ||
148 | async updateExistingSubConfig (options: OverrideCommandOptions & { | ||
149 | newConfig: DeepPartial<CustomConfig> | ||
150 | }) { | ||
151 | const existing = await this.getCustomConfig(options) | ||
152 | |||
153 | return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) | ||
154 | } | ||
155 | |||
84 | updateCustomSubConfig (options: OverrideCommandOptions & { | 156 | updateCustomSubConfig (options: OverrideCommandOptions & { |
85 | newConfig: DeepPartial<CustomConfig> | 157 | newConfig: DeepPartial<CustomConfig> |
86 | }) { | 158 | }) { |
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts index 9055dfc57..92ff7a0f9 100644 --- a/shared/extra-utils/server/index.ts +++ b/shared/extra-utils/server/index.ts | |||
@@ -6,6 +6,7 @@ export * from './follows-command' | |||
6 | export * from './follows' | 6 | export * from './follows' |
7 | export * from './jobs' | 7 | export * from './jobs' |
8 | export * from './jobs-command' | 8 | export * from './jobs-command' |
9 | export * from './object-storage-command' | ||
9 | export * from './plugins-command' | 10 | export * from './plugins-command' |
10 | export * from './plugins' | 11 | export * from './plugins' |
11 | export * from './redundancy-command' | 12 | export * from './redundancy-command' |
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts index c4eb12dc2..91771c176 100644 --- a/shared/extra-utils/server/jobs-command.ts +++ b/shared/extra-utils/server/jobs-command.ts | |||
@@ -5,6 +5,16 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared' | |||
5 | 5 | ||
6 | export class JobsCommand extends AbstractCommand { | 6 | export class JobsCommand extends AbstractCommand { |
7 | 7 | ||
8 | async getLatest (options: OverrideCommandOptions & { | ||
9 | jobType: JobType | ||
10 | }) { | ||
11 | const { data } = await this.getJobsList({ ...options, start: 0, count: 1, sort: '-createdAt' }) | ||
12 | |||
13 | if (data.length === 0) return undefined | ||
14 | |||
15 | return data[0] | ||
16 | } | ||
17 | |||
8 | getJobsList (options: OverrideCommandOptions & { | 18 | getJobsList (options: OverrideCommandOptions & { |
9 | state?: JobState | 19 | state?: JobState |
10 | jobType?: JobType | 20 | jobType?: JobType |
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index 64a0353eb..27104bfdf 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts | |||
@@ -3,7 +3,7 @@ import { JobState } from '../../models' | |||
3 | import { wait } from '../miscs' | 3 | import { wait } from '../miscs' |
4 | import { PeerTubeServer } from './server' | 4 | import { PeerTubeServer } from './server' |
5 | 5 | ||
6 | async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { | 6 | async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer, skipDelayed = false) { |
7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT | 7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT |
8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) | 8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) |
9 | : 250 | 9 | : 250 |
@@ -13,7 +13,9 @@ async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { | |||
13 | if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] | 13 | if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] |
14 | else servers = serversArg as PeerTubeServer[] | 14 | else servers = serversArg as PeerTubeServer[] |
15 | 15 | ||
16 | const states: JobState[] = [ 'waiting', 'active', 'delayed' ] | 16 | const states: JobState[] = [ 'waiting', 'active' ] |
17 | if (!skipDelayed) states.push('delayed') | ||
18 | |||
17 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] | 19 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] |
18 | let pendingRequests: boolean | 20 | let pendingRequests: boolean |
19 | 21 | ||
diff --git a/shared/extra-utils/server/object-storage-command.ts b/shared/extra-utils/server/object-storage-command.ts new file mode 100644 index 000000000..b4de8f4cb --- /dev/null +++ b/shared/extra-utils/server/object-storage-command.ts | |||
@@ -0,0 +1,77 @@ | |||
1 | |||
2 | import { HttpStatusCode } from '@shared/models' | ||
3 | import { makePostBodyRequest } from '../requests' | ||
4 | import { AbstractCommand } from '../shared' | ||
5 | |||
6 | export class ObjectStorageCommand extends AbstractCommand { | ||
7 | static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists' | ||
8 | static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos' | ||
9 | |||
10 | static getDefaultConfig () { | ||
11 | return { | ||
12 | object_storage: { | ||
13 | enabled: true, | ||
14 | endpoint: 'http://' + this.getEndpointHost(), | ||
15 | region: this.getRegion(), | ||
16 | |||
17 | credentials: this.getCredentialsConfig(), | ||
18 | |||
19 | streaming_playlists: { | ||
20 | bucket_name: this.DEFAULT_PLAYLIST_BUCKET | ||
21 | }, | ||
22 | |||
23 | videos: { | ||
24 | bucket_name: this.DEFAULT_WEBTORRENT_BUCKET | ||
25 | } | ||
26 | } | ||
27 | } | ||
28 | } | ||
29 | |||
30 | static getCredentialsConfig () { | ||
31 | return { | ||
32 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
33 | secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
34 | } | ||
35 | } | ||
36 | |||
37 | static getEndpointHost () { | ||
38 | return 'localhost:9444' | ||
39 | } | ||
40 | |||
41 | static getRegion () { | ||
42 | return 'us-east-1' | ||
43 | } | ||
44 | |||
45 | static getWebTorrentBaseUrl () { | ||
46 | return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/` | ||
47 | } | ||
48 | |||
49 | static getPlaylistBaseUrl () { | ||
50 | return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/` | ||
51 | } | ||
52 | |||
53 | static async prepareDefaultBuckets () { | ||
54 | await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET) | ||
55 | await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET) | ||
56 | } | ||
57 | |||
58 | static async createBucket (name: string) { | ||
59 | await makePostBodyRequest({ | ||
60 | url: this.getEndpointHost(), | ||
61 | path: '/ui/' + name + '?delete', | ||
62 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
63 | }) | ||
64 | |||
65 | await makePostBodyRequest({ | ||
66 | url: this.getEndpointHost(), | ||
67 | path: '/ui/' + name + '?create', | ||
68 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
69 | }) | ||
70 | |||
71 | await makePostBodyRequest({ | ||
72 | url: this.getEndpointHost(), | ||
73 | path: '/ui/' + name + '?make-public', | ||
74 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
75 | }) | ||
76 | } | ||
77 | } | ||
diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 3c335b8e4..bc5e1cd5f 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts | |||
@@ -38,11 +38,13 @@ import { PluginsCommand } from './plugins-command' | |||
38 | import { RedundancyCommand } from './redundancy-command' | 38 | import { RedundancyCommand } from './redundancy-command' |
39 | import { ServersCommand } from './servers-command' | 39 | import { ServersCommand } from './servers-command' |
40 | import { StatsCommand } from './stats-command' | 40 | import { StatsCommand } from './stats-command' |
41 | import { ObjectStorageCommand } from './object-storage-command' | ||
41 | 42 | ||
42 | export type RunServerOptions = { | 43 | export type RunServerOptions = { |
43 | hideLogs?: boolean | 44 | hideLogs?: boolean |
44 | nodeArgs?: string[] | 45 | nodeArgs?: string[] |
45 | peertubeArgs?: string[] | 46 | peertubeArgs?: string[] |
47 | env?: { [ id: string ]: string } | ||
46 | } | 48 | } |
47 | 49 | ||
48 | export class PeerTubeServer { | 50 | export class PeerTubeServer { |
@@ -121,6 +123,7 @@ export class PeerTubeServer { | |||
121 | servers?: ServersCommand | 123 | servers?: ServersCommand |
122 | login?: LoginCommand | 124 | login?: LoginCommand |
123 | users?: UsersCommand | 125 | users?: UsersCommand |
126 | objectStorage?: ObjectStorageCommand | ||
124 | videos?: VideosCommand | 127 | videos?: VideosCommand |
125 | 128 | ||
126 | constructor (options: { serverNumber: number } | { url: string }) { | 129 | constructor (options: { serverNumber: number } | { url: string }) { |
@@ -202,6 +205,10 @@ export class PeerTubeServer { | |||
202 | env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() | 205 | env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() |
203 | env['NODE_CONFIG'] = JSON.stringify(configOverride) | 206 | env['NODE_CONFIG'] = JSON.stringify(configOverride) |
204 | 207 | ||
208 | if (options.env) { | ||
209 | Object.assign(env, options.env) | ||
210 | } | ||
211 | |||
205 | const forkOptions = { | 212 | const forkOptions = { |
206 | silent: true, | 213 | silent: true, |
207 | env, | 214 | env, |
@@ -209,10 +216,17 @@ export class PeerTubeServer { | |||
209 | execArgv: options.nodeArgs || [] | 216 | execArgv: options.nodeArgs || [] |
210 | } | 217 | } |
211 | 218 | ||
212 | return new Promise<void>(res => { | 219 | return new Promise<void>((res, rej) => { |
213 | const self = this | 220 | const self = this |
214 | 221 | ||
215 | this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) | 222 | this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) |
223 | |||
224 | const onExit = function () { | ||
225 | return rej(new Error('Process exited')) | ||
226 | } | ||
227 | |||
228 | this.app.on('exit', onExit) | ||
229 | |||
216 | this.app.stdout.on('data', function onStdout (data) { | 230 | this.app.stdout.on('data', function onStdout (data) { |
217 | let dontContinue = false | 231 | let dontContinue = false |
218 | 232 | ||
@@ -241,6 +255,7 @@ export class PeerTubeServer { | |||
241 | console.log(data.toString()) | 255 | console.log(data.toString()) |
242 | } else { | 256 | } else { |
243 | self.app.stdout.removeListener('data', onStdout) | 257 | self.app.stdout.removeListener('data', onStdout) |
258 | self.app.removeListener('exit', onExit) | ||
244 | } | 259 | } |
245 | 260 | ||
246 | process.on('exit', () => { | 261 | process.on('exit', () => { |
@@ -365,5 +380,6 @@ export class PeerTubeServer { | |||
365 | this.login = new LoginCommand(this) | 380 | this.login = new LoginCommand(this) |
366 | this.users = new UsersCommand(this) | 381 | this.users = new UsersCommand(this) |
367 | this.videos = new VideosCommand(this) | 382 | this.videos = new VideosCommand(this) |
383 | this.objectStorage = new ObjectStorageCommand(this) | ||
368 | } | 384 | } |
369 | } | 385 | } |
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index f0622feb0..21ab9405b 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts | |||
@@ -10,11 +10,11 @@ async function createSingleServer (serverNumber: number, configOverride?: Object | |||
10 | return server | 10 | return server |
11 | } | 11 | } |
12 | 12 | ||
13 | function createMultipleServers (totalServers: number, configOverride?: Object) { | 13 | function createMultipleServers (totalServers: number, configOverride?: Object, options: RunServerOptions = {}) { |
14 | const serverPromises: Promise<PeerTubeServer>[] = [] | 14 | const serverPromises: Promise<PeerTubeServer>[] = [] |
15 | 15 | ||
16 | for (let i = 1; i <= totalServers; i++) { | 16 | for (let i = 1; i <= totalServers; i++) { |
17 | serverPromises.push(createSingleServer(i, configOverride)) | 17 | serverPromises.push(createSingleServer(i, configOverride, options)) |
18 | } | 18 | } |
19 | 19 | ||
20 | return Promise.all(serverPromises) | 20 | return Promise.all(serverPromises) |
diff --git a/shared/extra-utils/videos/live-command.ts b/shared/extra-utils/videos/live-command.ts index 81ae458e0..74f5d3089 100644 --- a/shared/extra-utils/videos/live-command.ts +++ b/shared/extra-utils/videos/live-command.ts | |||
@@ -126,7 +126,7 @@ export class LiveCommand extends AbstractCommand { | |||
126 | video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) | 126 | video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) |
127 | 127 | ||
128 | await wait(500) | 128 | await wait(500) |
129 | } while (video.isLive === true && video.state.id !== VideoState.PUBLISHED) | 129 | } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED) |
130 | } | 130 | } |
131 | 131 | ||
132 | async countPlaylists (options: OverrideCommandOptions & { | 132 | async countPlaylists (options: OverrideCommandOptions & { |
diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts index 9a6df07a8..29f99ed6d 100644 --- a/shared/extra-utils/videos/live.ts +++ b/shared/extra-utils/videos/live.ts | |||
@@ -89,6 +89,12 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi | |||
89 | } | 89 | } |
90 | } | 90 | } |
91 | 91 | ||
92 | async function waitUntilLiveSavedOnAllServers (servers: PeerTubeServer[], videoId: string) { | ||
93 | for (const server of servers) { | ||
94 | await server.live.waitUntilSaved({ videoId }) | ||
95 | } | ||
96 | } | ||
97 | |||
92 | async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { | 98 | async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { |
93 | const basePath = server.servers.buildDirectory('streaming-playlists') | 99 | const basePath = server.servers.buildDirectory('streaming-playlists') |
94 | const hlsPath = join(basePath, 'hls', videoUUID) | 100 | const hlsPath = join(basePath, 'hls', videoUUID) |
@@ -126,5 +132,6 @@ export { | |||
126 | testFfmpegStreamError, | 132 | testFfmpegStreamError, |
127 | stopFfmpeg, | 133 | stopFfmpeg, |
128 | waitUntilLivePublishedOnAllServers, | 134 | waitUntilLivePublishedOnAllServers, |
135 | waitUntilLiveSavedOnAllServers, | ||
129 | checkLiveCleanupAfterSave | 136 | checkLiveCleanupAfterSave |
130 | } | 137 | } |
diff --git a/shared/extra-utils/videos/streaming-playlists-command.ts b/shared/extra-utils/videos/streaming-playlists-command.ts index 9662685da..5d40d35cb 100644 --- a/shared/extra-utils/videos/streaming-playlists-command.ts +++ b/shared/extra-utils/videos/streaming-playlists-command.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { HttpStatusCode } from '@shared/models' | 1 | import { HttpStatusCode } from '@shared/models' |
2 | import { unwrapBody, unwrapText } from '../requests' | 2 | import { unwrapBody, unwrapTextOrDecode, unwrapBodyOrDecodeToJSON } from '../requests' |
3 | import { AbstractCommand, OverrideCommandOptions } from '../shared' | 3 | import { AbstractCommand, OverrideCommandOptions } from '../shared' |
4 | 4 | ||
5 | export class StreamingPlaylistsCommand extends AbstractCommand { | 5 | export class StreamingPlaylistsCommand extends AbstractCommand { |
@@ -7,7 +7,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand { | |||
7 | get (options: OverrideCommandOptions & { | 7 | get (options: OverrideCommandOptions & { |
8 | url: string | 8 | url: string |
9 | }) { | 9 | }) { |
10 | return unwrapText(this.getRawRequest({ | 10 | return unwrapTextOrDecode(this.getRawRequest({ |
11 | ...options, | 11 | ...options, |
12 | 12 | ||
13 | url: options.url, | 13 | url: options.url, |
@@ -33,7 +33,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand { | |||
33 | getSegmentSha256 (options: OverrideCommandOptions & { | 33 | getSegmentSha256 (options: OverrideCommandOptions & { |
34 | url: string | 34 | url: string |
35 | }) { | 35 | }) { |
36 | return unwrapBody<{ [ id: string ]: string }>(this.getRawRequest({ | 36 | return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ |
37 | ...options, | 37 | ...options, |
38 | 38 | ||
39 | url: options.url, | 39 | url: options.url, |
diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index a224b8f5f..6671e3fa6 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts | |||
@@ -9,17 +9,16 @@ async function checkSegmentHash (options: { | |||
9 | server: PeerTubeServer | 9 | server: PeerTubeServer |
10 | baseUrlPlaylist: string | 10 | baseUrlPlaylist: string |
11 | baseUrlSegment: string | 11 | baseUrlSegment: string |
12 | videoUUID: string | ||
13 | resolution: number | 12 | resolution: number |
14 | hlsPlaylist: VideoStreamingPlaylist | 13 | hlsPlaylist: VideoStreamingPlaylist |
15 | }) { | 14 | }) { |
16 | const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options | 15 | const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist } = options |
17 | const command = server.streamingPlaylists | 16 | const command = server.streamingPlaylists |
18 | 17 | ||
19 | const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) | 18 | const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) |
20 | const videoName = basename(file.fileUrl) | 19 | const videoName = basename(file.fileUrl) |
21 | 20 | ||
22 | const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) | 21 | const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) |
23 | 22 | ||
24 | const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) | 23 | const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) |
25 | 24 | ||
@@ -28,7 +27,7 @@ async function checkSegmentHash (options: { | |||
28 | const range = `${offset}-${offset + length - 1}` | 27 | const range = `${offset}-${offset + length - 1}` |
29 | 28 | ||
30 | const segmentBody = await command.getSegment({ | 29 | const segmentBody = await command.getSegment({ |
31 | url: `${baseUrlSegment}/${videoUUID}/${videoName}`, | 30 | url: `${baseUrlSegment}/${videoName}`, |
32 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, | 31 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, |
33 | range: `bytes=${range}` | 32 | range: `bytes=${range}` |
34 | }) | 33 | }) |
diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 33725bfdc..d35339c8d 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts | |||
@@ -188,6 +188,17 @@ export class VideosCommand extends AbstractCommand { | |||
188 | return id | 188 | return id |
189 | } | 189 | } |
190 | 190 | ||
191 | async listFiles (options: OverrideCommandOptions & { | ||
192 | id: number | string | ||
193 | }) { | ||
194 | const video = await this.get(options) | ||
195 | |||
196 | const files = video.files || [] | ||
197 | const hlsFiles = video.streamingPlaylists[0]?.files || [] | ||
198 | |||
199 | return files.concat(hlsFiles) | ||
200 | } | ||
201 | |||
191 | // --------------------------------------------------------------------------- | 202 | // --------------------------------------------------------------------------- |
192 | 203 | ||
193 | listMyVideos (options: OverrideCommandOptions & { | 204 | listMyVideos (options: OverrideCommandOptions & { |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 4ab249e0b..ff96283a4 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -19,6 +19,7 @@ export type JobType = | |||
19 | | 'video-redundancy' | 19 | | 'video-redundancy' |
20 | | 'video-live-ending' | 20 | | 'video-live-ending' |
21 | | 'actor-keys' | 21 | | 'actor-keys' |
22 | | 'move-to-object-storage' | ||
22 | 23 | ||
23 | export interface Job { | 24 | export interface Job { |
24 | id: number | 25 | id: number |
@@ -136,3 +137,8 @@ export interface VideoLiveEndingPayload { | |||
136 | export interface ActorKeysPayload { | 137 | export interface ActorKeysPayload { |
137 | actorId: number | 138 | actorId: number |
138 | } | 139 | } |
140 | |||
141 | export interface MoveObjectStoragePayload { | ||
142 | videoUUID: string | ||
143 | isNewVideo: boolean | ||
144 | } | ||
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index faa9b9868..733c433a0 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts | |||
@@ -26,6 +26,7 @@ export * from './video-resolution.enum' | |||
26 | export * from './video-schedule-update.model' | 26 | export * from './video-schedule-update.model' |
27 | export * from './video-sort-field.type' | 27 | export * from './video-sort-field.type' |
28 | export * from './video-state.enum' | 28 | export * from './video-state.enum' |
29 | export * from './video-storage.enum' | ||
29 | 30 | ||
30 | export * from './video-streaming-playlist.model' | 31 | export * from './video-streaming-playlist.model' |
31 | export * from './video-streaming-playlist.type' | 32 | export * from './video-streaming-playlist.type' |
diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts index 49d997f24..c6af481e7 100644 --- a/shared/models/videos/video-state.enum.ts +++ b/shared/models/videos/video-state.enum.ts | |||
@@ -3,5 +3,6 @@ export const enum VideoState { | |||
3 | TO_TRANSCODE = 2, | 3 | TO_TRANSCODE = 2, |
4 | TO_IMPORT = 3, | 4 | TO_IMPORT = 3, |
5 | WAITING_FOR_LIVE = 4, | 5 | WAITING_FOR_LIVE = 4, |
6 | LIVE_ENDED = 5 | 6 | LIVE_ENDED = 5, |
7 | TO_MOVE_TO_EXTERNAL_STORAGE = 6 | ||
7 | } | 8 | } |
diff --git a/shared/models/videos/video-storage.enum.ts b/shared/models/videos/video-storage.enum.ts new file mode 100644 index 000000000..7c6690db2 --- /dev/null +++ b/shared/models/videos/video-storage.enum.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export const enum VideoStorage { | ||
2 | FILE_SYSTEM, | ||
3 | OBJECT_STORAGE, | ||
4 | } | ||
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml index ce0f89d7b..1b474582a 100644 --- a/support/docker/production/config/custom-environment-variables.yaml +++ b/support/docker/production/config/custom-environment-variables.yaml | |||
@@ -45,6 +45,29 @@ smtp: | |||
45 | __format: "json" | 45 | __format: "json" |
46 | from_address: "PEERTUBE_SMTP_FROM" | 46 | from_address: "PEERTUBE_SMTP_FROM" |
47 | 47 | ||
48 | object_storage: | ||
49 | enabled: | ||
50 | __name: "PEERTUBE_OBJECT_STORAGE_ENABLED" | ||
51 | __format: "json" | ||
52 | |||
53 | endpoint: "PEERTUBE_OBJECT_STORAGE_ENDPOINT" | ||
54 | |||
55 | region: "PEERTUBE_OBJECT_STORAGE_REGION" | ||
56 | |||
57 | max_upload_part: | ||
58 | __name: "PEERTUBE_OBJECT_STORAGE_MAX_UPLOAD_PART" | ||
59 | __format: "json" | ||
60 | |||
61 | streaming_playlists: | ||
62 | bucket_name: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BUCKET_NAME" | ||
63 | prefix: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_PREFIX" | ||
64 | base_url: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BASE_URL" | ||
65 | |||
66 | videos: | ||
67 | bucket_name: "PEERTUBE_OBJECT_STORAGE_VIDEOS_BUCKET_NAME" | ||
68 | prefix: "PEERTUBE_OBJECT_STORAGE_VIDEOS_PREFIX" | ||
69 | base_url: "PEERTUBE_OBJECT_STORAGE_VIDEOS_BASE_URL" | ||
70 | |||
48 | log: | 71 | log: |
49 | level: "PEERTUBE_LOG_LEVEL" | 72 | level: "PEERTUBE_LOG_LEVEL" |
50 | log_ping_requests: | 73 | log_ping_requests: |
@@ -49,6 +49,770 @@ | |||
49 | resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" | 49 | resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" |
50 | integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== | 50 | integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== |
51 | 51 | ||
52 | "@aws-crypto/crc32@^1.0.0": | ||
53 | version "1.1.0" | ||
54 | resolved "https://registry.yarnpkg.com/@aws-crypto/crc32/-/crc32-1.1.0.tgz#aff048e207798fad0b0e7765b12d474c273779b6" | ||
55 | integrity sha512-ifvfaaJVvT+JUTi3zSkX4wtuGGVJrAcjN7ftg+JiE/frNBP3zNwo4xipzWBsMLZfNuzMZuaesEYyqkZcs5tzCQ== | ||
56 | dependencies: | ||
57 | tslib "^1.11.1" | ||
58 | |||
59 | "@aws-crypto/ie11-detection@^1.0.0": | ||
60 | version "1.0.0" | ||
61 | resolved "https://registry.yarnpkg.com/@aws-crypto/ie11-detection/-/ie11-detection-1.0.0.tgz#d3a6af29ba7f15458f79c41d1cd8cac3925e726a" | ||
62 | integrity sha512-kCKVhCF1oDxFYgQrxXmIrS5oaWulkvRcPz+QBDMsUr2crbF4VGgGT6+uQhSwJFdUAQ2A//Vq+uT83eJrkzFgXA== | ||
63 | dependencies: | ||
64 | tslib "^1.11.1" | ||
65 | |||
66 | "@aws-crypto/sha256-browser@^1.0.0": | ||
67 | version "1.1.1" | ||
68 | resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-browser/-/sha256-browser-1.1.1.tgz#85dddf13e8f8d74c0d6592d993e4bf401da9f420" | ||
69 | integrity sha512-nS4vdan97It6HcweV58WXtjPbPSc0JXd3sAwlw3Ou5Mc3WllSycAS32Tv2LRn8butNQoU9AE3jEQAOgiMdNC1Q== | ||
70 | dependencies: | ||
71 | "@aws-crypto/ie11-detection" "^1.0.0" | ||
72 | "@aws-crypto/sha256-js" "^1.1.0" | ||
73 | "@aws-crypto/supports-web-crypto" "^1.0.0" | ||
74 | "@aws-sdk/types" "^3.1.0" | ||
75 | "@aws-sdk/util-locate-window" "^3.0.0" | ||
76 | "@aws-sdk/util-utf8-browser" "^3.0.0" | ||
77 | tslib "^1.11.1" | ||
78 | |||
79 | "@aws-crypto/sha256-js@^1.0.0", "@aws-crypto/sha256-js@^1.1.0": | ||
80 | version "1.1.0" | ||
81 | resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-js/-/sha256-js-1.1.0.tgz#a58386ad18186e392e0f1d98d18831261d27b071" | ||
82 | integrity sha512-VIhuqbPgXDVr8sZe2yhgQcDRRmzf4CI8fmC1A3bHiRfE6wlz1d8KpeemqbuoEHotz/Dch9yOxlshyQDNjNFeHA== | ||
83 | dependencies: | ||
84 | "@aws-sdk/types" "^3.1.0" | ||
85 | "@aws-sdk/util-utf8-browser" "^3.0.0" | ||
86 | tslib "^1.11.1" | ||
87 | |||
88 | "@aws-crypto/supports-web-crypto@^1.0.0": | ||
89 | version "1.0.0" | ||
90 | resolved "https://registry.yarnpkg.com/@aws-crypto/supports-web-crypto/-/supports-web-crypto-1.0.0.tgz#c40901bc17ac1e875e248df16a2b47ad8bfd9a93" | ||
91 | integrity sha512-IHLfv+WmVH89EW4n6a5eE8/hUlz6qkWGMn/v4r5ZgzcXdTC5nolii2z3k46y01hWRiC2PPhOdeSLzMUCUMco7g== | ||
92 | dependencies: | ||
93 | tslib "^1.11.1" | ||
94 | |||
95 | "@aws-sdk/abort-controller@3.25.0": | ||
96 | version "3.25.0" | ||
97 | resolved "https://registry.yarnpkg.com/@aws-sdk/abort-controller/-/abort-controller-3.25.0.tgz#a9ea250140de378d8beb6d2f427067fa30423e9e" | ||
98 | integrity sha512-uEVKqKkPVz6atbCxCNJY5O7V+ieSK8crUswXo8/WePyEbGEgxJ4t9x/WG4lV8kBjelmvQHDR4GqfJmb5Sh9xSg== | ||
99 | dependencies: | ||
100 | "@aws-sdk/types" "3.25.0" | ||
101 | tslib "^2.3.0" | ||
102 | |||
103 | "@aws-sdk/chunked-blob-reader-native@3.23.0": | ||
104 | version "3.23.0" | ||
105 | resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.23.0.tgz#72d711e3cc904bb380e99cdd60c59deacd1596ac" | ||
106 | integrity sha512-Ya5f8Ntv0EyZw+AHkpV6n6qqHzpCDNlkX50uj/dwFCMmPiHFWsWMvd0Qu04Y7miycJINEatRrJ5V8r/uVvZIDg== | ||
107 | dependencies: | ||
108 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
109 | tslib "^2.3.0" | ||
110 | |||
111 | "@aws-sdk/chunked-blob-reader@3.23.0": | ||
112 | version "3.23.0" | ||
113 | resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader/-/chunked-blob-reader-3.23.0.tgz#83eb6a437172b671e699850378bcb558e15374ec" | ||
114 | integrity sha512-gmJhCuXrKOOumppviE4K30NvsIQIqqxbGDNptrJrMYBO0qXCbK8/BypZ/hS/oT3loDzlSIxG2z5GDL/va9lbFw== | ||
115 | dependencies: | ||
116 | tslib "^2.3.0" | ||
117 | |||
118 | "@aws-sdk/client-s3@^3.23.0": | ||
119 | version "3.25.0" | ||
120 | resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.25.0.tgz#6b8146c18e76606378c5f736554cb41ad4ce229e" | ||
121 | integrity sha512-yVDPo6x7DCt9t833SjqWI+AQBx81/m54gLF/ePQZBeHL5mPMEyMXTF0o22yUP5t8f92U2VAyRCP2NvKtB9WgBg== | ||
122 | dependencies: | ||
123 | "@aws-crypto/sha256-browser" "^1.0.0" | ||
124 | "@aws-crypto/sha256-js" "^1.0.0" | ||
125 | "@aws-sdk/client-sts" "3.25.0" | ||
126 | "@aws-sdk/config-resolver" "3.25.0" | ||
127 | "@aws-sdk/credential-provider-node" "3.25.0" | ||
128 | "@aws-sdk/eventstream-serde-browser" "3.25.0" | ||
129 | "@aws-sdk/eventstream-serde-config-resolver" "3.25.0" | ||
130 | "@aws-sdk/eventstream-serde-node" "3.25.0" | ||
131 | "@aws-sdk/fetch-http-handler" "3.25.0" | ||
132 | "@aws-sdk/hash-blob-browser" "3.25.0" | ||
133 | "@aws-sdk/hash-node" "3.25.0" | ||
134 | "@aws-sdk/hash-stream-node" "3.25.0" | ||
135 | "@aws-sdk/invalid-dependency" "3.25.0" | ||
136 | "@aws-sdk/md5-js" "3.25.0" | ||
137 | "@aws-sdk/middleware-apply-body-checksum" "3.25.0" | ||
138 | "@aws-sdk/middleware-bucket-endpoint" "3.25.0" | ||
139 | "@aws-sdk/middleware-content-length" "3.25.0" | ||
140 | "@aws-sdk/middleware-expect-continue" "3.25.0" | ||
141 | "@aws-sdk/middleware-host-header" "3.25.0" | ||
142 | "@aws-sdk/middleware-location-constraint" "3.25.0" | ||
143 | "@aws-sdk/middleware-logger" "3.25.0" | ||
144 | "@aws-sdk/middleware-retry" "3.25.0" | ||
145 | "@aws-sdk/middleware-sdk-s3" "3.25.0" | ||
146 | "@aws-sdk/middleware-serde" "3.25.0" | ||
147 | "@aws-sdk/middleware-signing" "3.25.0" | ||
148 | "@aws-sdk/middleware-ssec" "3.25.0" | ||
149 | "@aws-sdk/middleware-stack" "3.25.0" | ||
150 | "@aws-sdk/middleware-user-agent" "3.25.0" | ||
151 | "@aws-sdk/node-config-provider" "3.25.0" | ||
152 | "@aws-sdk/node-http-handler" "3.25.0" | ||
153 | "@aws-sdk/protocol-http" "3.25.0" | ||
154 | "@aws-sdk/smithy-client" "3.25.0" | ||
155 | "@aws-sdk/types" "3.25.0" | ||
156 | "@aws-sdk/url-parser" "3.25.0" | ||
157 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
158 | "@aws-sdk/util-base64-node" "3.23.0" | ||
159 | "@aws-sdk/util-body-length-browser" "3.23.0" | ||
160 | "@aws-sdk/util-body-length-node" "3.23.0" | ||
161 | "@aws-sdk/util-user-agent-browser" "3.25.0" | ||
162 | "@aws-sdk/util-user-agent-node" "3.25.0" | ||
163 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
164 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
165 | "@aws-sdk/util-waiter" "3.25.0" | ||
166 | "@aws-sdk/xml-builder" "3.23.0" | ||
167 | entities "2.2.0" | ||
168 | fast-xml-parser "3.19.0" | ||
169 | tslib "^2.3.0" | ||
170 | |||
171 | "@aws-sdk/client-sso@3.25.0": | ||
172 | version "3.25.0" | ||
173 | resolved "https://registry.yarnpkg.com/@aws-sdk/client-sso/-/client-sso-3.25.0.tgz#9756178afb08e399b5aef5d12dfece3825bc2e26" | ||
174 | integrity sha512-b8v4tb7rncnqE5ktBlQEckFdNT+Pk2mBg4e1Uc9C1Z3XmZM+wOWtlbu+KRvgMgDWSx2FzLIjAKe3mLaM4o1Xhg== | ||
175 | dependencies: | ||
176 | "@aws-crypto/sha256-browser" "^1.0.0" | ||
177 | "@aws-crypto/sha256-js" "^1.0.0" | ||
178 | "@aws-sdk/config-resolver" "3.25.0" | ||
179 | "@aws-sdk/fetch-http-handler" "3.25.0" | ||
180 | "@aws-sdk/hash-node" "3.25.0" | ||
181 | "@aws-sdk/invalid-dependency" "3.25.0" | ||
182 | "@aws-sdk/middleware-content-length" "3.25.0" | ||
183 | "@aws-sdk/middleware-host-header" "3.25.0" | ||
184 | "@aws-sdk/middleware-logger" "3.25.0" | ||
185 | "@aws-sdk/middleware-retry" "3.25.0" | ||
186 | "@aws-sdk/middleware-serde" "3.25.0" | ||
187 | "@aws-sdk/middleware-stack" "3.25.0" | ||
188 | "@aws-sdk/middleware-user-agent" "3.25.0" | ||
189 | "@aws-sdk/node-config-provider" "3.25.0" | ||
190 | "@aws-sdk/node-http-handler" "3.25.0" | ||
191 | "@aws-sdk/protocol-http" "3.25.0" | ||
192 | "@aws-sdk/smithy-client" "3.25.0" | ||
193 | "@aws-sdk/types" "3.25.0" | ||
194 | "@aws-sdk/url-parser" "3.25.0" | ||
195 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
196 | "@aws-sdk/util-base64-node" "3.23.0" | ||
197 | "@aws-sdk/util-body-length-browser" "3.23.0" | ||
198 | "@aws-sdk/util-body-length-node" "3.23.0" | ||
199 | "@aws-sdk/util-user-agent-browser" "3.25.0" | ||
200 | "@aws-sdk/util-user-agent-node" "3.25.0" | ||
201 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
202 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
203 | tslib "^2.3.0" | ||
204 | |||
205 | "@aws-sdk/client-sts@3.25.0": | ||
206 | version "3.25.0" | ||
207 | resolved "https://registry.yarnpkg.com/@aws-sdk/client-sts/-/client-sts-3.25.0.tgz#e189c46d560daaa56b872330a5e7d125d00d5a1f" | ||
208 | integrity sha512-VQoG4GX+Pf5U/WtUgVgXLF2xC1jK6o4YmOxz09GhPfKT0y26x8hh42jY3zRCys7ldA3VKkfTLCeqMm3UKqXJZg== | ||
209 | dependencies: | ||
210 | "@aws-crypto/sha256-browser" "^1.0.0" | ||
211 | "@aws-crypto/sha256-js" "^1.0.0" | ||
212 | "@aws-sdk/config-resolver" "3.25.0" | ||
213 | "@aws-sdk/credential-provider-node" "3.25.0" | ||
214 | "@aws-sdk/fetch-http-handler" "3.25.0" | ||
215 | "@aws-sdk/hash-node" "3.25.0" | ||
216 | "@aws-sdk/invalid-dependency" "3.25.0" | ||
217 | "@aws-sdk/middleware-content-length" "3.25.0" | ||
218 | "@aws-sdk/middleware-host-header" "3.25.0" | ||
219 | "@aws-sdk/middleware-logger" "3.25.0" | ||
220 | "@aws-sdk/middleware-retry" "3.25.0" | ||
221 | "@aws-sdk/middleware-sdk-sts" "3.25.0" | ||
222 | "@aws-sdk/middleware-serde" "3.25.0" | ||
223 | "@aws-sdk/middleware-signing" "3.25.0" | ||
224 | "@aws-sdk/middleware-stack" "3.25.0" | ||
225 | "@aws-sdk/middleware-user-agent" "3.25.0" | ||
226 | "@aws-sdk/node-config-provider" "3.25.0" | ||
227 | "@aws-sdk/node-http-handler" "3.25.0" | ||
228 | "@aws-sdk/protocol-http" "3.25.0" | ||
229 | "@aws-sdk/smithy-client" "3.25.0" | ||
230 | "@aws-sdk/types" "3.25.0" | ||
231 | "@aws-sdk/url-parser" "3.25.0" | ||
232 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
233 | "@aws-sdk/util-base64-node" "3.23.0" | ||
234 | "@aws-sdk/util-body-length-browser" "3.23.0" | ||
235 | "@aws-sdk/util-body-length-node" "3.23.0" | ||
236 | "@aws-sdk/util-user-agent-browser" "3.25.0" | ||
237 | "@aws-sdk/util-user-agent-node" "3.25.0" | ||
238 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
239 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
240 | entities "2.2.0" | ||
241 | fast-xml-parser "3.19.0" | ||
242 | tslib "^2.3.0" | ||
243 | |||
244 | "@aws-sdk/config-resolver@3.25.0": | ||
245 | version "3.25.0" | ||
246 | resolved "https://registry.yarnpkg.com/@aws-sdk/config-resolver/-/config-resolver-3.25.0.tgz#d7caba201a00aeb9d60aeddb8901b7e58f7f5a2b" | ||
247 | integrity sha512-t5CE90jYkxQyGGxG22atf8040lHuL17wptGp1kN8nSxaG6PudKhxQuHPAGYt6FHgrqqeyFccp/P3jiDSjqUaVw== | ||
248 | dependencies: | ||
249 | "@aws-sdk/signature-v4" "3.25.0" | ||
250 | "@aws-sdk/types" "3.25.0" | ||
251 | tslib "^2.3.0" | ||
252 | |||
253 | "@aws-sdk/credential-provider-env@3.25.0": | ||
254 | version "3.25.0" | ||
255 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.25.0.tgz#9899ff627f40f09223126d6d2f1153b3ade2e804" | ||
256 | integrity sha512-I65/PNGQG+ktt1QSHCWwQ8v7QRK1eRdLkQl3zB5rwBuANbQ3Yu+vA+lAwU+IbpGCOEpHJO3lDN330It5B4Rtvg== | ||
257 | dependencies: | ||
258 | "@aws-sdk/property-provider" "3.25.0" | ||
259 | "@aws-sdk/types" "3.25.0" | ||
260 | tslib "^2.3.0" | ||
261 | |||
262 | "@aws-sdk/credential-provider-imds@3.25.0": | ||
263 | version "3.25.0" | ||
264 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.25.0.tgz#c40b76bb6a4561fb4c5fd94ce437aac938aaa23f" | ||
265 | integrity sha512-BhPM89tjeXsa0KXxz2UTLeAY798Qg1cddFXPZXaJyHQ6eWsrDSoKbSOaeP+rznp037NNLnLX6PB8MOtfu3MAzw== | ||
266 | dependencies: | ||
267 | "@aws-sdk/property-provider" "3.25.0" | ||
268 | "@aws-sdk/types" "3.25.0" | ||
269 | tslib "^2.3.0" | ||
270 | |||
271 | "@aws-sdk/credential-provider-ini@3.25.0": | ||
272 | version "3.25.0" | ||
273 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.25.0.tgz#32652f30247f84dd49e4c96ecec91577f972f2e3" | ||
274 | integrity sha512-p6yvqcZMN+eNZbJXnrFQgLpA06pVA2XagGJdkdDb3q9J4HYoWQduocWUfr3dy0HJdjDZ01BVT/ldBanUyhznQQ== | ||
275 | dependencies: | ||
276 | "@aws-sdk/credential-provider-env" "3.25.0" | ||
277 | "@aws-sdk/credential-provider-imds" "3.25.0" | ||
278 | "@aws-sdk/credential-provider-sso" "3.25.0" | ||
279 | "@aws-sdk/credential-provider-web-identity" "3.25.0" | ||
280 | "@aws-sdk/property-provider" "3.25.0" | ||
281 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
282 | "@aws-sdk/types" "3.25.0" | ||
283 | "@aws-sdk/util-credentials" "3.23.0" | ||
284 | tslib "^2.3.0" | ||
285 | |||
286 | "@aws-sdk/credential-provider-node@3.25.0": | ||
287 | version "3.25.0" | ||
288 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.25.0.tgz#f8f4c9b8ae51a89f44c11fbbf999e1363424f39e" | ||
289 | integrity sha512-GZedy79oSpnDr2I54su3EE1fwpTRFBw/Sn4RBE4VWCM8AWq7ZNk7IKAmbnBrmt+gpFpr9k2PifUIJ7fAcbNvJQ== | ||
290 | dependencies: | ||
291 | "@aws-sdk/credential-provider-env" "3.25.0" | ||
292 | "@aws-sdk/credential-provider-imds" "3.25.0" | ||
293 | "@aws-sdk/credential-provider-ini" "3.25.0" | ||
294 | "@aws-sdk/credential-provider-process" "3.25.0" | ||
295 | "@aws-sdk/credential-provider-sso" "3.25.0" | ||
296 | "@aws-sdk/credential-provider-web-identity" "3.25.0" | ||
297 | "@aws-sdk/property-provider" "3.25.0" | ||
298 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
299 | "@aws-sdk/types" "3.25.0" | ||
300 | "@aws-sdk/util-credentials" "3.23.0" | ||
301 | tslib "^2.3.0" | ||
302 | |||
303 | "@aws-sdk/credential-provider-process@3.25.0": | ||
304 | version "3.25.0" | ||
305 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.25.0.tgz#472938d6582152252fb69247531125ed24017d4e" | ||
306 | integrity sha512-qMldWWDvvy6Q+HMcTAVWUJP7MLjLXqf0P08Vb5oGYOlyh4TCJDorccRVVsQvutjQggpBaIMTQdzjdamqtZ1y+w== | ||
307 | dependencies: | ||
308 | "@aws-sdk/property-provider" "3.25.0" | ||
309 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
310 | "@aws-sdk/types" "3.25.0" | ||
311 | "@aws-sdk/util-credentials" "3.23.0" | ||
312 | tslib "^2.3.0" | ||
313 | |||
314 | "@aws-sdk/credential-provider-sso@3.25.0": | ||
315 | version "3.25.0" | ||
316 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.25.0.tgz#e2065ee6aec63a647acc816732ffcd270eb3c669" | ||
317 | integrity sha512-cGP1Zcw2fZHn4CYGgq4soody4x5TrsWk0Pf9F8yCjRMSSZqs3rj0+PrXy4xqkiLCvTSrse6p4e4wMMpaFAm7Tg== | ||
318 | dependencies: | ||
319 | "@aws-sdk/client-sso" "3.25.0" | ||
320 | "@aws-sdk/property-provider" "3.25.0" | ||
321 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
322 | "@aws-sdk/types" "3.25.0" | ||
323 | "@aws-sdk/util-credentials" "3.23.0" | ||
324 | tslib "^2.3.0" | ||
325 | |||
326 | "@aws-sdk/credential-provider-web-identity@3.25.0": | ||
327 | version "3.25.0" | ||
328 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.25.0.tgz#9c330322eea3a5f1f0166556c1f18ecc0992b0bf" | ||
329 | integrity sha512-6NvOaynsXGuNYbrGzT5h+kkGMaKtAI6zKgPqS/20NKlO5PJc9Eo56Hdbq0gBohXSBzRJE5Jx/1OOrTdvRlwniw== | ||
330 | dependencies: | ||
331 | "@aws-sdk/property-provider" "3.25.0" | ||
332 | "@aws-sdk/types" "3.25.0" | ||
333 | tslib "^2.3.0" | ||
334 | |||
335 | "@aws-sdk/eventstream-marshaller@3.25.0": | ||
336 | version "3.25.0" | ||
337 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-marshaller/-/eventstream-marshaller-3.25.0.tgz#8db1f633a638f50d8e37441f01d739238d374549" | ||
338 | integrity sha512-gUZIIxupgCIGyspiIV6bEplSRWnhAR9MkyrCJbHhbs4GjWIYlFqp7W0+Y7HY1tIeeXCUf0O8KE3paUMszKPXtg== | ||
339 | dependencies: | ||
340 | "@aws-crypto/crc32" "^1.0.0" | ||
341 | "@aws-sdk/types" "3.25.0" | ||
342 | "@aws-sdk/util-hex-encoding" "3.23.0" | ||
343 | tslib "^2.3.0" | ||
344 | |||
345 | "@aws-sdk/eventstream-serde-browser@3.25.0": | ||
346 | version "3.25.0" | ||
347 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-browser/-/eventstream-serde-browser-3.25.0.tgz#55481e23acb454d876948fd3b6e609a79977fa7d" | ||
348 | integrity sha512-QJF08OIZiufoBPPoVcRwBPvZIpKMSZpISZfpCHcY1GaTpMIzz35N7Nkd10JGpfzpUO9oFcgcmm2q3XHo1XJyyw== | ||
349 | dependencies: | ||
350 | "@aws-sdk/eventstream-marshaller" "3.25.0" | ||
351 | "@aws-sdk/eventstream-serde-universal" "3.25.0" | ||
352 | "@aws-sdk/types" "3.25.0" | ||
353 | tslib "^2.3.0" | ||
354 | |||
355 | "@aws-sdk/eventstream-serde-config-resolver@3.25.0": | ||
356 | version "3.25.0" | ||
357 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.25.0.tgz#5b8f4ef24fb1bf6c9f0353fb219a68206bad5eb4" | ||
358 | integrity sha512-Fb4VS3waKNzc6pK6tQBmWM+JmCNQJYNG/QBfb8y4AoJOZ+I7yX0Qgo90drh8IiUcIKDeprUFjSi/cGIa/KHIsg== | ||
359 | dependencies: | ||
360 | "@aws-sdk/types" "3.25.0" | ||
361 | tslib "^2.3.0" | ||
362 | |||
363 | "@aws-sdk/eventstream-serde-node@3.25.0": | ||
364 | version "3.25.0" | ||
365 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-node/-/eventstream-serde-node-3.25.0.tgz#7ae7fcb8db1e554638f8f1c0fea514cfb07e2524" | ||
366 | integrity sha512-gPs+6w0zXf+p0PuOxxmpAlCvP/7E7+8oAar8Ys27exnLXNgqJJK1k5hMBSrfR9GLVti3EhJ1M9x5Seg1SN0/SA== | ||
367 | dependencies: | ||
368 | "@aws-sdk/eventstream-marshaller" "3.25.0" | ||
369 | "@aws-sdk/eventstream-serde-universal" "3.25.0" | ||
370 | "@aws-sdk/types" "3.25.0" | ||
371 | tslib "^2.3.0" | ||
372 | |||
373 | "@aws-sdk/eventstream-serde-universal@3.25.0": | ||
374 | version "3.25.0" | ||
375 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-universal/-/eventstream-serde-universal-3.25.0.tgz#bf84056fcad894c14f7239272248ea5b3ff39d47" | ||
376 | integrity sha512-NgsQk5dXg7NlRDEKGRUdiAx7WESQGD1jEhXitklL3/PHRZ7Y9BJugEFlBvKpU7tiHZBcomTbl/gE2o6i2op/jA== | ||
377 | dependencies: | ||
378 | "@aws-sdk/eventstream-marshaller" "3.25.0" | ||
379 | "@aws-sdk/types" "3.25.0" | ||
380 | tslib "^2.3.0" | ||
381 | |||
382 | "@aws-sdk/fetch-http-handler@3.25.0": | ||
383 | version "3.25.0" | ||
384 | resolved "https://registry.yarnpkg.com/@aws-sdk/fetch-http-handler/-/fetch-http-handler-3.25.0.tgz#0ba013ced267b8ead120be1fcba5bdbbc379b82f" | ||
385 | integrity sha512-792kkbfSRBdiFb7Q2cDJts9MKxzAwuQSwUIwRKAOMazU8HkKbKnXXAFSsK3T7VasOFOh7O7YEGN0q9UgEw1q+g== | ||
386 | dependencies: | ||
387 | "@aws-sdk/protocol-http" "3.25.0" | ||
388 | "@aws-sdk/querystring-builder" "3.25.0" | ||
389 | "@aws-sdk/types" "3.25.0" | ||
390 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
391 | tslib "^2.3.0" | ||
392 | |||
393 | "@aws-sdk/hash-blob-browser@3.25.0": | ||
394 | version "3.25.0" | ||
395 | resolved "https://registry.yarnpkg.com/@aws-sdk/hash-blob-browser/-/hash-blob-browser-3.25.0.tgz#2708daf0f2b53c6670a94276c1048a9a34706108" | ||
396 | integrity sha512-dsvV/nkW8v9wIotd3xJn3TQ8AxVLl56H82WkGkHcfw61csRxj3eSUNv0apUBopCcQPK8OK4l2nHAg08r0+LWXg== | ||
397 | dependencies: | ||
398 | "@aws-sdk/chunked-blob-reader" "3.23.0" | ||
399 | "@aws-sdk/chunked-blob-reader-native" "3.23.0" | ||
400 | "@aws-sdk/types" "3.25.0" | ||
401 | tslib "^2.3.0" | ||
402 | |||
403 | "@aws-sdk/hash-node@3.25.0": | ||
404 | version "3.25.0" | ||
405 | resolved "https://registry.yarnpkg.com/@aws-sdk/hash-node/-/hash-node-3.25.0.tgz#b149ddf170f4038c7cc3afe8f12e21b0f63e0771" | ||
406 | integrity sha512-qRn6iqG9VLt8D29SBABcbauDLn92ssMjtpyVApiOhDYyFm2VA2avomOHD6y2PRBMwM5FMQAygZbpA2HIN2F96w== | ||
407 | dependencies: | ||
408 | "@aws-sdk/types" "3.25.0" | ||
409 | "@aws-sdk/util-buffer-from" "3.23.0" | ||
410 | tslib "^2.3.0" | ||
411 | |||
412 | "@aws-sdk/hash-stream-node@3.25.0": | ||
413 | version "3.25.0" | ||
414 | resolved "https://registry.yarnpkg.com/@aws-sdk/hash-stream-node/-/hash-stream-node-3.25.0.tgz#6fa38cc349a9037367f20ce2601ff0510035dfa2" | ||
415 | integrity sha512-pzScUO9pPEEHQ5YQk1sl1bPlU2tt0OCblxUwboZJ9mRgNnWwkMWxe7Mec5IfyMWVUcbIznUHn7qRYEvJQ9JXmw== | ||
416 | dependencies: | ||
417 | "@aws-sdk/types" "3.25.0" | ||
418 | tslib "^2.3.0" | ||
419 | |||
420 | "@aws-sdk/invalid-dependency@3.25.0": | ||
421 | version "3.25.0" | ||
422 | resolved "https://registry.yarnpkg.com/@aws-sdk/invalid-dependency/-/invalid-dependency-3.25.0.tgz#a75dfb7e86a0e1eb6083b61397dc49a1db041434" | ||
423 | integrity sha512-ZBXjBAF2JSiO/wGBa1oaXsd1q5YG3diS8TfIUMXeQoe9O66R5LGoGOQeAbB/JjlwFot6DZfAcfocvl6CtWwqkw== | ||
424 | dependencies: | ||
425 | "@aws-sdk/types" "3.25.0" | ||
426 | tslib "^2.3.0" | ||
427 | |||
428 | "@aws-sdk/is-array-buffer@3.23.0": | ||
429 | version "3.23.0" | ||
430 | resolved "https://registry.yarnpkg.com/@aws-sdk/is-array-buffer/-/is-array-buffer-3.23.0.tgz#3a5d601b0102ea3a4d832bde647509c8405b2ec9" | ||
431 | integrity sha512-XN20/scFthok0lCbjtinW77CoIBoar8cbOzmu+HkYTnBBpJrF6Ai5g9sgglO8r+X+OLn4PrDrTP+BxdpNuIh9g== | ||
432 | dependencies: | ||
433 | tslib "^2.3.0" | ||
434 | |||
435 | "@aws-sdk/md5-js@3.25.0": | ||
436 | version "3.25.0" | ||
437 | resolved "https://registry.yarnpkg.com/@aws-sdk/md5-js/-/md5-js-3.25.0.tgz#32cefc43a8c0ee1d85586b95eba0be4912cde534" | ||
438 | integrity sha512-97MtL1VF3JCkyJJnwi8LcXpqItnH1VtgoqtVqmaASYp5GXnlsnA1WDnB0754ufPHlssS1aBj/gkLzMZ0Htw/Rg== | ||
439 | dependencies: | ||
440 | "@aws-sdk/types" "3.25.0" | ||
441 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
442 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
443 | tslib "^2.3.0" | ||
444 | |||
445 | "@aws-sdk/middleware-apply-body-checksum@3.25.0": | ||
446 | version "3.25.0" | ||
447 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-apply-body-checksum/-/middleware-apply-body-checksum-3.25.0.tgz#4263ea8c8e1808e5a4a278fb704ebe7aa891f698" | ||
448 | integrity sha512-162qFG7eap4vDKuKrpXWQYE4tbIETNrpTQX6jrPgqostOy1O0Nc5Bn1COIoOMgeMVnkOAZV7qV1J/XAYGz32Yw== | ||
449 | dependencies: | ||
450 | "@aws-sdk/is-array-buffer" "3.23.0" | ||
451 | "@aws-sdk/protocol-http" "3.25.0" | ||
452 | "@aws-sdk/types" "3.25.0" | ||
453 | tslib "^2.3.0" | ||
454 | |||
455 | "@aws-sdk/middleware-bucket-endpoint@3.25.0": | ||
456 | version "3.25.0" | ||
457 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.25.0.tgz#d698230ae552533a1b8ded2c3e6885b4a8374795" | ||
458 | integrity sha512-r/6ECFiw/TNjzhAuZzUx3M/1mAtezHTp3e8twB4dDbRRQqABrEZ/dynXi1VxrT2kKW0ZgZNXqEer/NfPOtWB8g== | ||
459 | dependencies: | ||
460 | "@aws-sdk/protocol-http" "3.25.0" | ||
461 | "@aws-sdk/types" "3.25.0" | ||
462 | "@aws-sdk/util-arn-parser" "3.23.0" | ||
463 | tslib "^2.3.0" | ||
464 | |||
465 | "@aws-sdk/middleware-content-length@3.25.0": | ||
466 | version "3.25.0" | ||
467 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-content-length/-/middleware-content-length-3.25.0.tgz#71031d326e52f788396e0ed8216410840059ac53" | ||
468 | integrity sha512-uOXus0MmZi/mucRIr5yfwM1vDhYG66CujNfnhyEaq5f4kcDA1Q5qPWSn9dkQPV9JWTZK3WTuYiOPSgtmlAYTAg== | ||
469 | dependencies: | ||
470 | "@aws-sdk/protocol-http" "3.25.0" | ||
471 | "@aws-sdk/types" "3.25.0" | ||
472 | tslib "^2.3.0" | ||
473 | |||
474 | "@aws-sdk/middleware-expect-continue@3.25.0": | ||
475 | version "3.25.0" | ||
476 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.25.0.tgz#bb41ea9d716c6ce04c4d8fb2cc2dd5fd37f6ccd9" | ||
477 | integrity sha512-o3euv8NIO0zlHML81krtfs4TrF5gZwoxBYtY+6tRHXlgutsHe1yfg1wrhWnJNbJg1QhPwXxbMNfYX7MM83D8Ng== | ||
478 | dependencies: | ||
479 | "@aws-sdk/middleware-header-default" "3.25.0" | ||
480 | "@aws-sdk/protocol-http" "3.25.0" | ||
481 | "@aws-sdk/types" "3.25.0" | ||
482 | tslib "^2.3.0" | ||
483 | |||
484 | "@aws-sdk/middleware-header-default@3.25.0": | ||
485 | version "3.25.0" | ||
486 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-header-default/-/middleware-header-default-3.25.0.tgz#17fec9b1941e81059a1374eba58b52230da35a2b" | ||
487 | integrity sha512-xkFfZcctPL0VTxmEKITf6/MSDv/8rY+8uA9OMt/YZqfbg0RfeqR2+R1xlDNDxeHeK/v+g5gTNIYTQLM8L2unNA== | ||
488 | dependencies: | ||
489 | "@aws-sdk/protocol-http" "3.25.0" | ||
490 | "@aws-sdk/types" "3.25.0" | ||
491 | tslib "^2.3.0" | ||
492 | |||
493 | "@aws-sdk/middleware-host-header@3.25.0": | ||
494 | version "3.25.0" | ||
495 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.25.0.tgz#f08dd8c45362cf5cb152c478027092e3d1f4aa58" | ||
496 | integrity sha512-xKD/CfsUS3ul2VaQ3IgIUXgA7jU2/Guo/DUhYKrLZTOxm0nuvsIFw0RqSCtRBCLptE5Qi+unkc1LcFDbfqrRbg== | ||
497 | dependencies: | ||
498 | "@aws-sdk/protocol-http" "3.25.0" | ||
499 | "@aws-sdk/types" "3.25.0" | ||
500 | tslib "^2.3.0" | ||
501 | |||
502 | "@aws-sdk/middleware-location-constraint@3.25.0": | ||
503 | version "3.25.0" | ||
504 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.25.0.tgz#7ba5798aa46cd08c90823f649fcdae0ce5227095" | ||
505 | integrity sha512-diwmJ+MRQrq3H9VH+8CNAT4dImf2j3CLewlMrUEY+HsJN9xl2mtU6GQaluQg60iw6FjurLUKKGTTZCul4PGkIQ== | ||
506 | dependencies: | ||
507 | "@aws-sdk/types" "3.25.0" | ||
508 | tslib "^2.3.0" | ||
509 | |||
510 | "@aws-sdk/middleware-logger@3.25.0": | ||
511 | version "3.25.0" | ||
512 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.25.0.tgz#03294611be7a2f4aba06e9d80e04318c0991d769" | ||
513 | integrity sha512-M1F7BlAsDKoEM8hBaU2pHlLSM40rzzgtZ6jFNhfmTwGcjxe1N7JXCH5QPa7aI8wnJq2RoIRHVfVsUH4GwvOZnA== | ||
514 | dependencies: | ||
515 | "@aws-sdk/types" "3.25.0" | ||
516 | tslib "^2.3.0" | ||
517 | |||
518 | "@aws-sdk/middleware-retry@3.25.0": | ||
519 | version "3.25.0" | ||
520 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-retry/-/middleware-retry-3.25.0.tgz#e9f1b011494142aa27ece3ef881e8a3d4866797c" | ||
521 | integrity sha512-SzdWPo4ESUR6AXvIf4eC8s5sko2G9Hou6cUIr+BWI4h7whA32j/aWUmvcMHxWT/eaSuPeruXrnvKyLvuM0RjJg== | ||
522 | dependencies: | ||
523 | "@aws-sdk/protocol-http" "3.25.0" | ||
524 | "@aws-sdk/service-error-classification" "3.25.0" | ||
525 | "@aws-sdk/types" "3.25.0" | ||
526 | tslib "^2.3.0" | ||
527 | uuid "^8.3.2" | ||
528 | |||
529 | "@aws-sdk/middleware-sdk-s3@3.25.0": | ||
530 | version "3.25.0" | ||
531 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.25.0.tgz#64278bbc97c3a2c26411f155642cc35e8de38887" | ||
532 | integrity sha512-Y1P6JnpAdj7p5Q43aSLSuYBCc3hKpZ/mrqFSGN8VFXl7Tzo7tYfjpd9SVRxNGJK7O7tDAUsPNmuGqBrdA2tj8w== | ||
533 | dependencies: | ||
534 | "@aws-sdk/protocol-http" "3.25.0" | ||
535 | "@aws-sdk/types" "3.25.0" | ||
536 | "@aws-sdk/util-arn-parser" "3.23.0" | ||
537 | tslib "^2.3.0" | ||
538 | |||
539 | "@aws-sdk/middleware-sdk-sts@3.25.0": | ||
540 | version "3.25.0" | ||
541 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.25.0.tgz#15d4836958f70187cbb6819a0c0742b751fb44ed" | ||
542 | integrity sha512-1SoZZTVejo+32eH0WqXaFvt/NIkVEYWquh3OJpkghMi2oOnMfeIRI0uSoqshL6949f4iSfUvvtuzDpyA7XNCQA== | ||
543 | dependencies: | ||
544 | "@aws-sdk/middleware-signing" "3.25.0" | ||
545 | "@aws-sdk/property-provider" "3.25.0" | ||
546 | "@aws-sdk/protocol-http" "3.25.0" | ||
547 | "@aws-sdk/signature-v4" "3.25.0" | ||
548 | "@aws-sdk/types" "3.25.0" | ||
549 | tslib "^2.3.0" | ||
550 | |||
551 | "@aws-sdk/middleware-serde@3.25.0": | ||
552 | version "3.25.0" | ||
553 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-serde/-/middleware-serde-3.25.0.tgz#e1284ed4af64b4444cfeb7b5275f489418fa2f58" | ||
554 | integrity sha512-065Kugo8yXzBkcVAxctxFCHKlHcINnaQRsJ8ifvgc+UOEgvTG9+LfGWDwfdgarW9CkF7RkCoZOyaqFsO+HJWsg== | ||
555 | dependencies: | ||
556 | "@aws-sdk/types" "3.25.0" | ||
557 | tslib "^2.3.0" | ||
558 | |||
559 | "@aws-sdk/middleware-signing@3.25.0": | ||
560 | version "3.25.0" | ||
561 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-signing/-/middleware-signing-3.25.0.tgz#de19f5b27c34161081553a87285f1b5690e2cb9a" | ||
562 | integrity sha512-FkhxGMV3UY5HIAwUcarfxdq/CF/tYukdg+bkbTNluMpkcJczqn6shpEIQAGa5FFQP3Lya+STL1NuNXfOP7bG9w== | ||
563 | dependencies: | ||
564 | "@aws-sdk/property-provider" "3.25.0" | ||
565 | "@aws-sdk/protocol-http" "3.25.0" | ||
566 | "@aws-sdk/signature-v4" "3.25.0" | ||
567 | "@aws-sdk/types" "3.25.0" | ||
568 | tslib "^2.3.0" | ||
569 | |||
570 | "@aws-sdk/middleware-ssec@3.25.0": | ||
571 | version "3.25.0" | ||
572 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.25.0.tgz#f8cf5bb6fe48d842b1df77f35ccb0f77f1a07b71" | ||
573 | integrity sha512-bnrHb8oddW+vDexbNzZtpfshshKru+skcmq3dyXlL8LB/NlJsMiQJE8xoGbq5odTLiflIgaDBt527m5q58i+fg== | ||
574 | dependencies: | ||
575 | "@aws-sdk/types" "3.25.0" | ||
576 | tslib "^2.3.0" | ||
577 | |||
578 | "@aws-sdk/middleware-stack@3.25.0": | ||
579 | version "3.25.0" | ||
580 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-stack/-/middleware-stack-3.25.0.tgz#8fc022c90b030c80308bf2930c4a7040052234b4" | ||
581 | integrity sha512-s2VgdsasOVKHY3/SIGsw9AeZMMsdcIbBGWim9n5IO3j8C8y54EdRLVCEja8ePvMDZKIzuummwatYPHaUrnqPtQ== | ||
582 | dependencies: | ||
583 | tslib "^2.3.0" | ||
584 | |||
585 | "@aws-sdk/middleware-user-agent@3.25.0": | ||
586 | version "3.25.0" | ||
587 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.25.0.tgz#2033da6cdcfbf4641b991e3ee3c60ba9809898e7" | ||
588 | integrity sha512-HXd/Qknq8Cp7fzJYU7jDDpN7ReJ3arUrnt+dAPNaDDrhmrBbCZp+24UXN6X6DAj0JICRoRuF/l7KxjwdF5FShw== | ||
589 | dependencies: | ||
590 | "@aws-sdk/protocol-http" "3.25.0" | ||
591 | "@aws-sdk/types" "3.25.0" | ||
592 | tslib "^2.3.0" | ||
593 | |||
594 | "@aws-sdk/node-config-provider@3.25.0": | ||
595 | version "3.25.0" | ||
596 | resolved "https://registry.yarnpkg.com/@aws-sdk/node-config-provider/-/node-config-provider-3.25.0.tgz#6ec3e9031b7ff0c51d6e0b33aeff3547ea5619b3" | ||
597 | integrity sha512-95FiUDuh1YGo0Giti0Xz9l2TV0Wzw75M1xx0TduFcm1dpLKl+znxTgYh+4G+MOSMHNGy+6K91yxurv4PGYgCWw== | ||
598 | dependencies: | ||
599 | "@aws-sdk/property-provider" "3.25.0" | ||
600 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
601 | "@aws-sdk/types" "3.25.0" | ||
602 | tslib "^2.3.0" | ||
603 | |||
604 | "@aws-sdk/node-http-handler@3.25.0": | ||
605 | version "3.25.0" | ||
606 | resolved "https://registry.yarnpkg.com/@aws-sdk/node-http-handler/-/node-http-handler-3.25.0.tgz#b636ea2c39b4a47cf9bffd4cdb6a41c603b99bff" | ||
607 | integrity sha512-zVeAM/bXewZiuMtcUZI/xGDID6knkzOv73ueVkzUbP0Ki8bfao7diR3hMbIt5Fy/r8cAVjJce9v6zFqo4sr1WA== | ||
608 | dependencies: | ||
609 | "@aws-sdk/abort-controller" "3.25.0" | ||
610 | "@aws-sdk/protocol-http" "3.25.0" | ||
611 | "@aws-sdk/querystring-builder" "3.25.0" | ||
612 | "@aws-sdk/types" "3.25.0" | ||
613 | tslib "^2.3.0" | ||
614 | |||
615 | "@aws-sdk/property-provider@3.25.0": | ||
616 | version "3.25.0" | ||
617 | resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.25.0.tgz#2fd7246917b9b6ff448a599163a479bc417a1421" | ||
618 | integrity sha512-jUnPDguLWsyGLPfdxGdeaXe3j/CjS3kxBmctvI+soZg57rA2hntP9rm7SUZ2+5rj4mmJaI3bzchiaY3kE3JmpA== | ||
619 | dependencies: | ||
620 | "@aws-sdk/types" "3.25.0" | ||
621 | tslib "^2.3.0" | ||
622 | |||
623 | "@aws-sdk/protocol-http@3.25.0": | ||
624 | version "3.25.0" | ||
625 | resolved "https://registry.yarnpkg.com/@aws-sdk/protocol-http/-/protocol-http-3.25.0.tgz#4b638cb90672fc2d6cb6d15bebc8bb1fb297da2e" | ||
626 | integrity sha512-4Jebt5G8uIFa+HZO7KOgOtA66E/CXysQekiV5dfAsU8ca+rX5PB6qhpWZ2unX/l6He+oDQ0zMoW70JkNiP4/4w== | ||
627 | dependencies: | ||
628 | "@aws-sdk/types" "3.25.0" | ||
629 | tslib "^2.3.0" | ||
630 | |||
631 | "@aws-sdk/querystring-builder@3.25.0": | ||
632 | version "3.25.0" | ||
633 | resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.25.0.tgz#9e6f5eaa5d6805fbf45ae4a47ccbaf823584a4a2" | ||
634 | integrity sha512-o/R3/viOxjWckI+kepkxJSL7fIdg1hHYOW/rOpo9HbXS0CJrHVnB8vlBb+Xwl1IFyY2gg+5YZTjiufcgpgRBkw== | ||
635 | dependencies: | ||
636 | "@aws-sdk/types" "3.25.0" | ||
637 | "@aws-sdk/util-uri-escape" "3.23.0" | ||
638 | tslib "^2.3.0" | ||
639 | |||
640 | "@aws-sdk/querystring-parser@3.25.0": | ||
641 | version "3.25.0" | ||
642 | resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.25.0.tgz#7fe0a3ddf95a4e5475f53be056fce435fb24b774" | ||
643 | integrity sha512-FCNyaOLFLVS5j43MhVA7/VJUDX0t/9RyNTNulHgzFjj6ffsgqcY0uwUq1RO3QCL4asl56zOrLVJgK+Z7wMbvFg== | ||
644 | dependencies: | ||
645 | "@aws-sdk/types" "3.25.0" | ||
646 | tslib "^2.3.0" | ||
647 | |||
648 | "@aws-sdk/service-error-classification@3.25.0": | ||
649 | version "3.25.0" | ||
650 | resolved "https://registry.yarnpkg.com/@aws-sdk/service-error-classification/-/service-error-classification-3.25.0.tgz#1f24fe74f0a89f00d4f6f2ad1d7bb6b0e2f871e7" | ||
651 | integrity sha512-66FfIab87LnnHtOLrGrVOht9Pw6lE8appyOpBdtoeoU5DP7ARSWuDdsYmKdGdRCWvn/RaVFbSYua9k0M1WsGqg== | ||
652 | |||
653 | "@aws-sdk/shared-ini-file-loader@3.23.0": | ||
654 | version "3.23.0" | ||
655 | resolved "https://registry.yarnpkg.com/@aws-sdk/shared-ini-file-loader/-/shared-ini-file-loader-3.23.0.tgz#574901a31e65e425632a9cae6a64f6382a2b76e8" | ||
656 | integrity sha512-YUp46l6E3dLKHp1cKMkZI4slTjsVc/Lm7nPCTVc3oQvZ1MvC99N/jMCmZ7X5YYofuAUSdc9eJ8sYiF2BnUww9g== | ||
657 | dependencies: | ||
658 | tslib "^2.3.0" | ||
659 | |||
660 | "@aws-sdk/signature-v4@3.25.0": | ||
661 | version "3.25.0" | ||
662 | resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4/-/signature-v4-3.25.0.tgz#c7fb8184a09593ef6dc62029ca45e252b51247b2" | ||
663 | integrity sha512-6KDRRz9XVrj9RxrBLC6dzfnb2TDl3CjIzcNpLdRuKFgzEEdwV+5D+EZuAQU3MuHG5pWTIwG72k/dmCbJ2MDPUQ== | ||
664 | dependencies: | ||
665 | "@aws-sdk/is-array-buffer" "3.23.0" | ||
666 | "@aws-sdk/types" "3.25.0" | ||
667 | "@aws-sdk/util-hex-encoding" "3.23.0" | ||
668 | "@aws-sdk/util-uri-escape" "3.23.0" | ||
669 | tslib "^2.3.0" | ||
670 | |||
671 | "@aws-sdk/smithy-client@3.25.0": | ||
672 | version "3.25.0" | ||
673 | resolved "https://registry.yarnpkg.com/@aws-sdk/smithy-client/-/smithy-client-3.25.0.tgz#bfdf77f1fa82b26bb7893f16056e8e60e49a140a" | ||
674 | integrity sha512-+/iMCNziL5/muaY/gl3xkRsSZyeoVCUSjSbbZjDIXbqDbB9SOz4o3UAIgWHoCgYNfsF25GQR6rThLi61FrSyoQ== | ||
675 | dependencies: | ||
676 | "@aws-sdk/middleware-stack" "3.25.0" | ||
677 | "@aws-sdk/types" "3.25.0" | ||
678 | tslib "^2.3.0" | ||
679 | |||
680 | "@aws-sdk/types@3.25.0", "@aws-sdk/types@^3.1.0": | ||
681 | version "3.25.0" | ||
682 | resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.25.0.tgz#981210272dae2d259130f6dca8429522d9a564bb" | ||
683 | integrity sha512-vS0+cTKwj6CujlR07HmeEBxzWPWSrdmZMYnxn/QC9KW9dFu0lsyCGSCqWsFluI6GI0flsnYYWNkP5y4bfD9tqg== | ||
684 | |||
685 | "@aws-sdk/url-parser@3.25.0": | ||
686 | version "3.25.0" | ||
687 | resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.25.0.tgz#668c7d9d4bc21854c10bfb8bdf762a9206776fae" | ||
688 | integrity sha512-qZ3Vq0NjHsE7Qq6R5NVRswIAsiyYjCDnAV+/Vt4jU/K0V3mGumiasiJyRyblW4Da8R6kfcJk0mHSMFRJfoHh8Q== | ||
689 | dependencies: | ||
690 | "@aws-sdk/querystring-parser" "3.25.0" | ||
691 | "@aws-sdk/types" "3.25.0" | ||
692 | tslib "^2.3.0" | ||
693 | |||
694 | "@aws-sdk/util-arn-parser@3.23.0": | ||
695 | version "3.23.0" | ||
696 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.23.0.tgz#7372460ba98a6826f97d9622759764bcf09add79" | ||
697 | integrity sha512-J3+/wnC21kbb3UAHo7x31aCZxzIa7GBijt6Q7nad/j2aF38EZtE3SI0aZpD8250Vi+9zsZ4672QDUeSZ5BR5kg== | ||
698 | dependencies: | ||
699 | tslib "^2.3.0" | ||
700 | |||
701 | "@aws-sdk/util-base64-browser@3.23.0": | ||
702 | version "3.23.0" | ||
703 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-base64-browser/-/util-base64-browser-3.23.0.tgz#61594ac9529756361c81ece287548ab5b8c5a768" | ||
704 | integrity sha512-xlI/qw+uhLJWa3k0mRtRHQ42v5QzsMFEUXScredQMfJ/34qzXyocsG6OHPOTV1I8WSANrxnHR5m1Ae3iU6JuVw== | ||
705 | dependencies: | ||
706 | tslib "^2.3.0" | ||
707 | |||
708 | "@aws-sdk/util-base64-node@3.23.0": | ||
709 | version "3.23.0" | ||
710 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-base64-node/-/util-base64-node-3.23.0.tgz#d0da9ed6b8aaa7513ba4b36a20b4794c72c074ce" | ||
711 | integrity sha512-Kf8JIAUtjrPcD5CJzrig2B5CtegWswUNpW4zBarww/UJhHlp8WzKlCxxA+yNS1ghT0ZMjrRvxPabKDGpkyUfmQ== | ||
712 | dependencies: | ||
713 | "@aws-sdk/util-buffer-from" "3.23.0" | ||
714 | tslib "^2.3.0" | ||
715 | |||
716 | "@aws-sdk/util-body-length-browser@3.23.0": | ||
717 | version "3.23.0" | ||
718 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-body-length-browser/-/util-body-length-browser-3.23.0.tgz#1a5c5e7ea5e15d93bd178021c54d2ea41faeb1cd" | ||
719 | integrity sha512-Bi6u/5omQbOBSB5BxqVvaPgVplLRjhhSuqK3XAukbeBPh7lcibIBdy7YvbhQyl4i8Hb2QjFnqqfzA0lNBe5eiw== | ||
720 | dependencies: | ||
721 | tslib "^2.3.0" | ||
722 | |||
723 | "@aws-sdk/util-body-length-node@3.23.0": | ||
724 | version "3.23.0" | ||
725 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-body-length-node/-/util-body-length-node-3.23.0.tgz#2a7890b4fa6de78a042db9537a67f90ccb2a3034" | ||
726 | integrity sha512-8kSczloA78mikPaJ742SU9Wpwfcz3HOruoXiP/pOy69UZEsMe4P7zTZI1bo8BAp7j6IFUPCXth9E3UAtkbz+CQ== | ||
727 | dependencies: | ||
728 | tslib "^2.3.0" | ||
729 | |||
730 | "@aws-sdk/util-buffer-from@3.23.0": | ||
731 | version "3.23.0" | ||
732 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-buffer-from/-/util-buffer-from-3.23.0.tgz#3bc02f50c6e8a5c2b9db61faeb3bebc9de701c3b" | ||
733 | integrity sha512-axXy1FvEOM1uECgMPmyHF1S3Hd7JI+BerhhcAlGig0bbqUsZVQUNL9yhOsWreA+nf1v08Ucj8P2SHPCT9Hvpgg== | ||
734 | dependencies: | ||
735 | "@aws-sdk/is-array-buffer" "3.23.0" | ||
736 | tslib "^2.3.0" | ||
737 | |||
738 | "@aws-sdk/util-credentials@3.23.0": | ||
739 | version "3.23.0" | ||
740 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-credentials/-/util-credentials-3.23.0.tgz#6b3138c3853c72adc93c3f57e8fb28f58ffdc364" | ||
741 | integrity sha512-6TDGZnFa0kZr+vSsWXXMfWt347jbMGKtzGnBxbrmiQgZMijz9s/wLYxsjglZ+CyqI/QrSMOTtqy6mEgJxdnGWQ== | ||
742 | dependencies: | ||
743 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
744 | tslib "^2.3.0" | ||
745 | |||
746 | "@aws-sdk/util-hex-encoding@3.23.0": | ||
747 | version "3.23.0" | ||
748 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.23.0.tgz#a8de34faf9e51dd4be379be0e9d3bdc093ae6bf4" | ||
749 | integrity sha512-RFDCwNrJMmmPSMVRadxRNePqTXGwtL9s4844x44D0bbGg1TdC42rrg0PRKYkxFL7wd1FbibVQOzciZAvzF+Z+w== | ||
750 | dependencies: | ||
751 | tslib "^2.3.0" | ||
752 | |||
753 | "@aws-sdk/util-locate-window@^3.0.0": | ||
754 | version "3.23.0" | ||
755 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-locate-window/-/util-locate-window-3.23.0.tgz#e9bf2a023dce2ea1d13ec2e8c7c92abb333a1442" | ||
756 | integrity sha512-mM8kWW7SWIxCshkNllpYqCQi5SzwJ+sv5nURhtquOB5/H3qGqZm0V5lUE3qpE1AYmqKwk6qbGUy1woFn1T5nrw== | ||
757 | dependencies: | ||
758 | tslib "^2.3.0" | ||
759 | |||
760 | "@aws-sdk/util-uri-escape@3.23.0": | ||
761 | version "3.23.0" | ||
762 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-uri-escape/-/util-uri-escape-3.23.0.tgz#52539674966eb456d65408d9028ed114e94dfd49" | ||
763 | integrity sha512-SvQx2E/FDlI5vLT67wwn/k1j2R/G58tYj4Te6GNgEwPGL43X2+7c0+d/WTgndMaRvxSBHZMUTxBYh1HOeU7loA== | ||
764 | dependencies: | ||
765 | tslib "^2.3.0" | ||
766 | |||
767 | "@aws-sdk/util-user-agent-browser@3.25.0": | ||
768 | version "3.25.0" | ||
769 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.25.0.tgz#a0f480f1a5b10350370643445b09413102187935" | ||
770 | integrity sha512-qGqiWfs49NRmQVXPsBXgMRVkjDZocicU0V2wak98e0t7TOI+KmP8hnwsTkE6c4KwhsFOOUhAzjn5zk3kOwi6tQ== | ||
771 | dependencies: | ||
772 | "@aws-sdk/types" "3.25.0" | ||
773 | bowser "^2.11.0" | ||
774 | tslib "^2.3.0" | ||
775 | |||
776 | "@aws-sdk/util-user-agent-node@3.25.0": | ||
777 | version "3.25.0" | ||
778 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.25.0.tgz#db22cb64893c4635adf17086c5cb4a5070c4ac16" | ||
779 | integrity sha512-4AWyCNP3n/qxv36OS+WH3l4ooRvwyfdbYWFXNXeGcxMcLANDG0upJQRT1g7H8+/afMaJ6v/BQM/H6tdocJSKjQ== | ||
780 | dependencies: | ||
781 | "@aws-sdk/node-config-provider" "3.25.0" | ||
782 | "@aws-sdk/types" "3.25.0" | ||
783 | tslib "^2.3.0" | ||
784 | |||
785 | "@aws-sdk/util-utf8-browser@3.23.0", "@aws-sdk/util-utf8-browser@^3.0.0": | ||
786 | version "3.23.0" | ||
787 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.23.0.tgz#dff7e891c67936de677b7d7a6c796e5c2e1b1510" | ||
788 | integrity sha512-fSB95AKnvCnAbCd7o0xLbErfAgD9wnLCaEu23AgfGAiaG3nFF8Z2+wtjebU/9Z4RI9d/x83Ho/yguRnJdkMsPA== | ||
789 | dependencies: | ||
790 | tslib "^2.3.0" | ||
791 | |||
792 | "@aws-sdk/util-utf8-node@3.23.0": | ||
793 | version "3.23.0" | ||
794 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-node/-/util-utf8-node-3.23.0.tgz#9f9fe76745c79c8a148f15d78e9a5c03d2bf0441" | ||
795 | integrity sha512-yao8+8okyfCxRvxZe3GBdO7lJnQEBf3P6rDgleOQD/0DZmMjOQGXCvDd42oagE2TegXhkUnJfVOZU2GqdoR0hg== | ||
796 | dependencies: | ||
797 | "@aws-sdk/util-buffer-from" "3.23.0" | ||
798 | tslib "^2.3.0" | ||
799 | |||
800 | "@aws-sdk/util-waiter@3.25.0": | ||
801 | version "3.25.0" | ||
802 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-waiter/-/util-waiter-3.25.0.tgz#cd2252c99f335e461134f55c3b7eb89ef6893dca" | ||
803 | integrity sha512-rhJ7Q2fcPD8y4H0qNEpaspkSUya0OaNcVrca9wCZKs7jWnropPzrQ+e2MH7fWJ/8jgcBV890+Txr4fWkD4J01g== | ||
804 | dependencies: | ||
805 | "@aws-sdk/abort-controller" "3.25.0" | ||
806 | "@aws-sdk/types" "3.25.0" | ||
807 | tslib "^2.3.0" | ||
808 | |||
809 | "@aws-sdk/xml-builder@3.23.0": | ||
810 | version "3.23.0" | ||
811 | resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.23.0.tgz#e318f539b68fa9c0a36da49e85a96cdca13a8113" | ||
812 | integrity sha512-5LEGdhQIJtGTwg4dIYyNtpz5QvPcQoxsqJygmj+VB8KLd+mWorH1IOpiL74z0infeK9N+ZFUUPKIzPJa9xLPqw== | ||
813 | dependencies: | ||
814 | tslib "^2.3.0" | ||
815 | |||
52 | "@babel/code-frame@7.12.11": | 816 | "@babel/code-frame@7.12.11": |
53 | version "7.12.11" | 817 | version "7.12.11" |
54 | resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" | 818 | resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" |
@@ -1652,6 +2416,11 @@ boolean@3.0.4: | |||
1652 | resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.4.tgz#aa1df8749af41d7211b66b4eee584722ff428c27" | 2416 | resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.4.tgz#aa1df8749af41d7211b66b4eee584722ff428c27" |
1653 | integrity sha512-5pyOr+w2LNN72F2mAq6J0ckHUfJYSgRKma7e/wlcMMhgOLV9OI0ERhERYXxUqo+dPyVxcbXKy9n+wg13+LpNnA== | 2417 | integrity sha512-5pyOr+w2LNN72F2mAq6J0ckHUfJYSgRKma7e/wlcMMhgOLV9OI0ERhERYXxUqo+dPyVxcbXKy9n+wg13+LpNnA== |
1654 | 2418 | ||
2419 | bowser@^2.11.0: | ||
2420 | version "2.11.0" | ||
2421 | resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f" | ||
2422 | integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA== | ||
2423 | |||
1655 | boxen@^4.2.0: | 2424 | boxen@^4.2.0: |
1656 | version "4.2.0" | 2425 | version "4.2.0" |
1657 | resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" | 2426 | resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" |
@@ -2909,7 +3678,7 @@ enquirer@^2.3.5: | |||
2909 | dependencies: | 3678 | dependencies: |
2910 | ansi-colors "^4.1.1" | 3679 | ansi-colors "^4.1.1" |
2911 | 3680 | ||
2912 | entities@^2.0.0: | 3681 | entities@2.2.0, entities@^2.0.0: |
2913 | version "2.2.0" | 3682 | version "2.2.0" |
2914 | resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" | 3683 | resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" |
2915 | integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== | 3684 | integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== |
@@ -3403,7 +4172,7 @@ fast-safe-stringify@^2.0.4, fast-safe-stringify@^2.0.7: | |||
3403 | resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f" | 4172 | resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f" |
3404 | integrity sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag== | 4173 | integrity sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag== |
3405 | 4174 | ||
3406 | fast-xml-parser@^3.19.0: | 4175 | fast-xml-parser@3.19.0, fast-xml-parser@^3.19.0: |
3407 | version "3.19.0" | 4176 | version "3.19.0" |
3408 | resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" | 4177 | resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" |
3409 | integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== | 4178 | integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== |
@@ -7947,7 +8716,7 @@ tsconfig-paths@^3.9.0: | |||
7947 | minimist "^1.2.0" | 8716 | minimist "^1.2.0" |
7948 | strip-bom "^3.0.0" | 8717 | strip-bom "^3.0.0" |
7949 | 8718 | ||
7950 | tslib@^1.8.1, tslib@^1.9.0: | 8719 | tslib@^1.11.1, tslib@^1.8.1, tslib@^1.9.0: |
7951 | version "1.14.1" | 8720 | version "1.14.1" |
7952 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" | 8721 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" |
7953 | integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== | 8722 | integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== |
@@ -7957,6 +8726,11 @@ tslib@^2.0.0, tslib@^2.2.0: | |||
7957 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" | 8726 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" |
7958 | integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== | 8727 | integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== |
7959 | 8728 | ||
8729 | tslib@^2.3.0: | ||
8730 | version "2.3.1" | ||
8731 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" | ||
8732 | integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== | ||
8733 | |||
7960 | tsutils@^3.21.0: | 8734 | tsutils@^3.21.0: |
7961 | version "3.21.0" | 8735 | version "3.21.0" |
7962 | resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" | 8736 | resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" |