diff options
86 files changed, 3585 insertions, 573 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5bbd9e2c..35c91bf85 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml | |||
@@ -31,6 +31,11 @@ jobs: | |||
31 | ports: | 31 | ports: |
32 | - 10389:10389 | 32 | - 10389:10389 |
33 | 33 | ||
34 | s3ninja: | ||
35 | image: scireum/s3-ninja | ||
36 | ports: | ||
37 | - 9444:9000 | ||
38 | |||
34 | strategy: | 39 | strategy: |
35 | fail-fast: false | 40 | fail-fast: false |
36 | matrix: | 41 | matrix: |
@@ -40,6 +45,7 @@ jobs: | |||
40 | PGUSER: peertube | 45 | PGUSER: peertube |
41 | PGHOST: localhost | 46 | PGHOST: localhost |
42 | NODE_PENDING_JOB_WAIT: 250 | 47 | NODE_PENDING_JOB_WAIT: 250 |
48 | ENABLE_OBJECT_STORAGE_TESTS: true | ||
43 | 49 | ||
44 | steps: | 50 | steps: |
45 | - uses: actions/checkout@v2 | 51 | - uses: actions/checkout@v2 |
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 29ba95c5c..4b02e1bc1 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts | |||
@@ -36,7 +36,8 @@ export class JobsComponent extends RestTable implements OnInit { | |||
36 | 'video-live-ending', | 36 | 'video-live-ending', |
37 | 'video-redundancy', | 37 | 'video-redundancy', |
38 | 'video-transcoding', | 38 | 'video-transcoding', |
39 | 'videos-views' | 39 | 'videos-views', |
40 | 'move-to-object-storage' | ||
40 | ] | 41 | ] |
41 | 42 | ||
42 | jobs: Job[] = [] | 43 | jobs: Job[] = [] |
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html index 3480d3656..e2dd44bf7 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html | |||
@@ -6,6 +6,10 @@ | |||
6 | The video is being transcoded, it may not work properly. | 6 | The video is being transcoded, it may not work properly. |
7 | </div> | 7 | </div> |
8 | 8 | ||
9 | <div i18n class="alert alert-warning" *ngIf="isVideoToMoveToExternalStorage()"> | ||
10 | The video is being moved to an external server, it may not work properly. | ||
11 | </div> | ||
12 | |||
9 | <div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()"> | 13 | <div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()"> |
10 | This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}. | 14 | This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}. |
11 | </div> | 15 | </div> |
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts index 8a46ba0d5..0072492ac 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts | |||
@@ -18,6 +18,10 @@ export class VideoAlertComponent { | |||
18 | return this.video && this.video.state.id === VideoState.TO_IMPORT | 18 | return this.video && this.video.state.id === VideoState.TO_IMPORT |
19 | } | 19 | } |
20 | 20 | ||
21 | isVideoToMoveToExternalStorage () { | ||
22 | return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | ||
23 | } | ||
24 | |||
21 | hasVideoScheduledPublication () { | 25 | hasVideoScheduledPublication () { |
22 | return this.video && this.video.scheduledUpdate !== undefined | 26 | return this.video && this.video.scheduledUpdate !== undefined |
23 | } | 27 | } |
diff --git a/config/default.yaml b/config/default.yaml index 3d0ae6e87..3865ab5cf 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -95,6 +95,39 @@ storage: | |||
95 | # If not, peertube will fallback to the default fil | 95 | # If not, peertube will fallback to the default fil |
96 | client_overrides: 'storage/client-overrides/' | 96 | client_overrides: 'storage/client-overrides/' |
97 | 97 | ||
98 | object_storage: | ||
99 | enabled: false | ||
100 | |||
101 | # Without protocol, will default to HTTPS | ||
102 | endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example | ||
103 | |||
104 | region: 'us-east-1' | ||
105 | |||
106 | credentials: | ||
107 | # You can also use AWS_ACCESS_KEY_ID env variable | ||
108 | access_key_id: '' | ||
109 | # You can also use AWS_SECRET_ACCESS_KEY env variable | ||
110 | secret_access_key: '' | ||
111 | |||
112 | # Maximum amount to upload in one request to object storage | ||
113 | max_upload_part: 2GB | ||
114 | |||
115 | streaming_playlists: | ||
116 | bucket_name: 'streaming-playlists' | ||
117 | |||
118 | # Allows setting all buckets to the same value but with a different prefix | ||
119 | prefix: '' # Example: 'streaming-playlists:' | ||
120 | |||
121 | # Base url for object URL generation, scheme and host will be replaced by this URL | ||
122 | # Useful when you want to use a CDN/external proxy | ||
123 | base_url: '' # Example: 'https://mirror.example.com' | ||
124 | |||
125 | # Same settings but for webtorrent videos | ||
126 | videos: | ||
127 | bucket_name: 'videos' | ||
128 | prefix: '' | ||
129 | base_url: '' | ||
130 | |||
98 | log: | 131 | log: |
99 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' | 132 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' |
100 | rotation: | 133 | rotation: |
diff --git a/config/production.yaml.example b/config/production.yaml.example index 514ab99a4..94238fad0 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example | |||
@@ -93,6 +93,39 @@ storage: | |||
93 | # If not, peertube will fallback to the default file | 93 | # If not, peertube will fallback to the default file |
94 | client_overrides: '/var/www/peertube/storage/client-overrides/' | 94 | client_overrides: '/var/www/peertube/storage/client-overrides/' |
95 | 95 | ||
96 | object_storage: | ||
97 | enabled: false | ||
98 | |||
99 | # Without protocol, will default to HTTPS | ||
100 | endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example | ||
101 | |||
102 | region: 'us-east-1' | ||
103 | |||
104 | credentials: | ||
105 | # You can also use AWS_ACCESS_KEY_ID env variable | ||
106 | access_key_id: '' | ||
107 | # You can also use AWS_SECRET_ACCESS_KEY env variable | ||
108 | secret_access_key: '' | ||
109 | |||
110 | # Maximum amount to upload in one request to object storage | ||
111 | max_upload_part: 2GB | ||
112 | |||
113 | streaming_playlists: | ||
114 | bucket_name: 'streaming-playlists' | ||
115 | |||
116 | # Allows setting all buckets to the same value but with a different prefix | ||
117 | prefix: '' # Example: 'streaming-playlists:' | ||
118 | |||
119 | # Base url for object URL generation, scheme and host will be replaced by this URL | ||
120 | # Useful when you want to use a CDN/external proxy | ||
121 | base_url: '' # Example: 'https://mirror.example.com' | ||
122 | |||
123 | # Same settings but for webtorrent videos | ||
124 | videos: | ||
125 | bucket_name: 'videos' | ||
126 | prefix: '' | ||
127 | base_url: '' | ||
128 | |||
96 | log: | 129 | log: |
97 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' | 130 | level: 'info' # 'debug' | 'info' | 'warn' | 'error' |
98 | rotation: | 131 | rotation: |
diff --git a/package.json b/package.json index 807a90742..d2b1e1245 100644 --- a/package.json +++ b/package.json | |||
@@ -73,6 +73,7 @@ | |||
73 | "swagger-cli": "swagger-cli" | 73 | "swagger-cli": "swagger-cli" |
74 | }, | 74 | }, |
75 | "dependencies": { | 75 | "dependencies": { |
76 | "@aws-sdk/client-s3": "^3.23.0", | ||
76 | "@uploadx/core": "^4.4.0", | 77 | "@uploadx/core": "^4.4.0", |
77 | "async": "^3.0.1", | 78 | "async": "^3.0.1", |
78 | "async-lru": "^1.1.1", | 79 | "async-lru": "^1.1.1", |
diff --git a/scripts/ci.sh b/scripts/ci.sh index 71b1be53b..f49dbe6ad 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh | |||
@@ -89,9 +89,10 @@ elif [ "$1" = "api-4" ]; then | |||
89 | 89 | ||
90 | moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation) | 90 | moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation) |
91 | redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy) | 91 | redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy) |
92 | objectStorageFiles=$(findTestFiles ./dist/server/tests/api/object-storage) | ||
92 | activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub) | 93 | activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub) |
93 | 94 | ||
94 | MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles | 95 | MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles |
95 | elif [ "$1" = "external-plugins" ]; then | 96 | elif [ "$1" = "external-plugins" ]; then |
96 | npm run build:server | 97 | npm run build:server |
97 | 98 | ||
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index 3a552c19a..0bb9bfeab 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts | |||
@@ -6,9 +6,10 @@ import { VideoModel } from '../server/models/video/video' | |||
6 | import { initDatabaseModels } from '../server/initializers/database' | 6 | import { initDatabaseModels } from '../server/initializers/database' |
7 | import { JobQueue } from '../server/lib/job-queue' | 7 | import { JobQueue } from '../server/lib/job-queue' |
8 | import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils' | 8 | import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils' |
9 | import { VideoTranscodingPayload } from '@shared/models' | 9 | import { VideoState, VideoTranscodingPayload } from '@shared/models' |
10 | import { CONFIG } from '@server/initializers/config' | 10 | import { CONFIG } from '@server/initializers/config' |
11 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' | 11 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' |
12 | import { addTranscodingJob } from '@server/lib/video' | ||
12 | 13 | ||
13 | program | 14 | program |
14 | .option('-v, --video [videoUUID]', 'Video UUID') | 15 | .option('-v, --video [videoUUID]', 'Video UUID') |
@@ -47,7 +48,7 @@ async function run () { | |||
47 | if (!video) throw new Error('Video not found.') | 48 | if (!video) throw new Error('Video not found.') |
48 | 49 | ||
49 | const dataInput: VideoTranscodingPayload[] = [] | 50 | const dataInput: VideoTranscodingPayload[] = [] |
50 | const { resolution } = await video.getMaxQualityResolution() | 51 | const resolution = video.getMaxQualityFile().resolution |
51 | 52 | ||
52 | // Generate HLS files | 53 | // Generate HLS files |
53 | if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { | 54 | if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { |
@@ -62,6 +63,7 @@ async function run () { | |||
62 | resolution, | 63 | resolution, |
63 | isPortraitMode: false, | 64 | isPortraitMode: false, |
64 | copyCodecs: false, | 65 | copyCodecs: false, |
66 | isNewVideo: false, | ||
65 | isMaxQuality: false | 67 | isMaxQuality: false |
66 | }) | 68 | }) |
67 | } | 69 | } |
@@ -87,10 +89,13 @@ async function run () { | |||
87 | } | 89 | } |
88 | } | 90 | } |
89 | 91 | ||
90 | await JobQueue.Instance.init() | 92 | JobQueue.Instance.init() |
93 | |||
94 | video.state = VideoState.TO_TRANSCODE | ||
95 | await video.save() | ||
91 | 96 | ||
92 | for (const d of dataInput) { | 97 | for (const d of dataInput) { |
93 | await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: d }) | 98 | await addTranscodingJob(d, {}) |
94 | console.log('Transcoding job for video %s created.', video.uuid) | 99 | console.log('Transcoding job for video %s created.', video.uuid) |
95 | } | 100 | } |
96 | } | 101 | } |
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index 9e66105dd..245e4cf28 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts | |||
@@ -1,15 +1,18 @@ | |||
1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' | 1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' |
2 | registerTSPaths() | 2 | registerTSPaths() |
3 | 3 | ||
4 | import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils' | ||
5 | import { VideoModel } from '../server/models/video/video' | ||
6 | import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding' | ||
7 | import { initDatabaseModels } from '../server/initializers/database' | ||
8 | import { basename, dirname } from 'path' | ||
9 | import { copy, move, remove } from 'fs-extra' | 4 | import { copy, move, remove } from 'fs-extra' |
5 | import { basename, dirname } from 'path' | ||
10 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
11 | import { getVideoFilePath } from '@server/lib/video-paths' | 7 | import { CONFIG } from '@server/initializers/config' |
8 | import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage' | ||
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
12 | import { getMaxBitrate } from '@shared/core-utils' | 10 | import { getMaxBitrate } from '@shared/core-utils' |
11 | import { MoveObjectStoragePayload } from '@shared/models' | ||
12 | import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils' | ||
13 | import { initDatabaseModels } from '../server/initializers/database' | ||
14 | import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding' | ||
15 | import { VideoModel } from '../server/models/video/video' | ||
13 | 16 | ||
14 | run() | 17 | run() |
15 | .then(() => process.exit(0)) | 18 | .then(() => process.exit(0)) |
@@ -39,43 +42,49 @@ async function run () { | |||
39 | currentVideoId = video.id | 42 | currentVideoId = video.id |
40 | 43 | ||
41 | for (const file of video.VideoFiles) { | 44 | for (const file of video.VideoFiles) { |
42 | currentFilePath = getVideoFilePath(video, file) | 45 | await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => { |
43 | 46 | currentFilePath = path | |
44 | const [ videoBitrate, fps, dataResolution ] = await Promise.all([ | 47 | |
45 | getVideoFileBitrate(currentFilePath), | 48 | const [ videoBitrate, fps, dataResolution ] = await Promise.all([ |
46 | getVideoFileFPS(currentFilePath), | 49 | getVideoFileBitrate(currentFilePath), |
47 | getVideoFileResolution(currentFilePath) | 50 | getVideoFileFPS(currentFilePath), |
48 | ]) | 51 | getVideoFileResolution(currentFilePath) |
49 | 52 | ]) | |
50 | const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) | 53 | |
51 | const isMaxBitrateExceeded = videoBitrate > maxBitrate | 54 | const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) |
52 | if (isMaxBitrateExceeded) { | 55 | const isMaxBitrateExceeded = videoBitrate > maxBitrate |
53 | console.log( | 56 | if (isMaxBitrateExceeded) { |
54 | 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', | 57 | console.log( |
55 | basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 | 58 | 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', |
56 | ) | 59 | basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 |
57 | 60 | ) | |
58 | const backupFile = `${currentFilePath}_backup` | 61 | |
59 | await copy(currentFilePath, backupFile) | 62 | const backupFile = `${currentFilePath}_backup` |
60 | 63 | await copy(currentFilePath, backupFile) | |
61 | await optimizeOriginalVideofile(video, file) | 64 | |
62 | // Update file path, the video filename changed | 65 | await optimizeOriginalVideofile(video, file) |
63 | currentFilePath = getVideoFilePath(video, file) | 66 | // Update file path, the video filename changed |
64 | 67 | currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) | |
65 | const originalDuration = await getDurationFromVideoFile(backupFile) | 68 | |
66 | const newDuration = await getDurationFromVideoFile(currentFilePath) | 69 | const originalDuration = await getDurationFromVideoFile(backupFile) |
67 | 70 | const newDuration = await getDurationFromVideoFile(currentFilePath) | |
68 | if (originalDuration === newDuration) { | 71 | |
69 | console.log('Finished optimizing %s', basename(currentFilePath)) | 72 | if (originalDuration === newDuration) { |
70 | await remove(backupFile) | 73 | console.log('Finished optimizing %s', basename(currentFilePath)) |
71 | continue | 74 | await remove(backupFile) |
75 | return | ||
76 | } | ||
77 | |||
78 | console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) | ||
79 | await move(backupFile, currentFilePath, { overwrite: true }) | ||
80 | await createTorrentAndSetInfoHash(video, file) | ||
81 | await file.save() | ||
72 | } | 82 | } |
83 | }) | ||
84 | } | ||
73 | 85 | ||
74 | console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) | 86 | if (CONFIG.OBJECT_STORAGE.ENABLED === true) { |
75 | await move(backupFile, currentFilePath, { overwrite: true }) | 87 | await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any) |
76 | await createTorrentAndSetInfoHash(video, file) | ||
77 | await file.save() | ||
78 | } | ||
79 | } | 88 | } |
80 | } | 89 | } |
81 | 90 | ||
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 89f50714d..5c740c041 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts | |||
@@ -1,12 +1,21 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { move } from 'fs-extra' | 2 | import { move } from 'fs-extra' |
3 | import { basename } from 'path' | ||
3 | import { getLowercaseExtension } from '@server/helpers/core-utils' | 4 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' | 5 | import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' |
5 | import { uuidToShort } from '@server/helpers/uuid' | 6 | import { uuidToShort } from '@server/helpers/uuid' |
6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
7 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 8 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
8 | import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' | 9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' |
9 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 10 | import { |
11 | addMoveToObjectStorageJob, | ||
12 | addOptimizeOrMergeAudioJob, | ||
13 | buildLocalVideoFromReq, | ||
14 | buildVideoThumbnailsFromReq, | ||
15 | setVideoTags | ||
16 | } from '@server/lib/video' | ||
17 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
18 | import { buildNextVideoState } from '@server/lib/video-state' | ||
10 | import { openapiOperationDoc } from '@server/middlewares/doc' | 19 | import { openapiOperationDoc } from '@server/middlewares/doc' |
11 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 20 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
12 | import { uploadx } from '@uploadx/core' | 21 | import { uploadx } from '@uploadx/core' |
@@ -139,23 +148,20 @@ async function addVideo (options: { | |||
139 | 148 | ||
140 | const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) | 149 | const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) |
141 | 150 | ||
142 | videoData.state = CONFIG.TRANSCODING.ENABLED | 151 | videoData.state = buildNextVideoState() |
143 | ? VideoState.TO_TRANSCODE | ||
144 | : VideoState.PUBLISHED | ||
145 | |||
146 | videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware | 152 | videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware |
147 | 153 | ||
148 | const video = new VideoModel(videoData) as MVideoFullLight | 154 | const video = new VideoModel(videoData) as MVideoFullLight |
149 | video.VideoChannel = videoChannel | 155 | video.VideoChannel = videoChannel |
150 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object | 156 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object |
151 | 157 | ||
152 | const videoFile = await buildNewFile(video, videoPhysicalFile) | 158 | const videoFile = await buildNewFile(videoPhysicalFile) |
153 | 159 | ||
154 | // Move physical file | 160 | // Move physical file |
155 | const destination = getVideoFilePath(video, videoFile) | 161 | const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) |
156 | await move(videoPhysicalFile.path, destination) | 162 | await move(videoPhysicalFile.path, destination) |
157 | // This is important in case if there is another attempt in the retry process | 163 | // This is important in case if there is another attempt in the retry process |
158 | videoPhysicalFile.filename = getVideoFilePath(video, videoFile) | 164 | videoPhysicalFile.filename = basename(destination) |
159 | videoPhysicalFile.path = destination | 165 | videoPhysicalFile.path = destination |
160 | 166 | ||
161 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ | 167 | const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ |
@@ -210,9 +216,13 @@ async function addVideo (options: { | |||
210 | 216 | ||
211 | createTorrentFederate(video, videoFile) | 217 | createTorrentFederate(video, videoFile) |
212 | .then(() => { | 218 | .then(() => { |
213 | if (video.state !== VideoState.TO_TRANSCODE) return | 219 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { |
220 | return addMoveToObjectStorageJob(video) | ||
221 | } | ||
214 | 222 | ||
215 | return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) | 223 | if (video.state === VideoState.TO_TRANSCODE) { |
224 | return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) | ||
225 | } | ||
216 | }) | 226 | }) |
217 | .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) | 227 | .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) |
218 | 228 | ||
@@ -227,7 +237,7 @@ async function addVideo (options: { | |||
227 | }) | 237 | }) |
228 | } | 238 | } |
229 | 239 | ||
230 | async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { | 240 | async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) { |
231 | const videoFile = new VideoFileModel({ | 241 | const videoFile = new VideoFileModel({ |
232 | extname: getLowercaseExtension(videoPhysicalFile.filename), | 242 | extname: getLowercaseExtension(videoPhysicalFile.filename), |
233 | size: videoPhysicalFile.size, | 243 | size: videoPhysicalFile.size, |
diff --git a/server/controllers/download.ts b/server/controllers/download.ts index ddacc1b68..ffe40d57e 100644 --- a/server/controllers/download.ts +++ b/server/controllers/download.ts | |||
@@ -3,9 +3,9 @@ import * as express from 'express' | |||
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' | 4 | import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' |
5 | import { Hooks } from '@server/lib/plugins/hooks' | 5 | import { Hooks } from '@server/lib/plugins/hooks' |
6 | import { getVideoFilePath } from '@server/lib/video-paths' | 6 | import { VideoPathManager } from '@server/lib/video-path-manager' |
7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 7 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
8 | import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' | 8 | import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' |
9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' | 9 | import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' |
10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' | 10 | import { asyncMiddleware, videosDownloadValidator } from '../middlewares' |
11 | 11 | ||
@@ -81,7 +81,15 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { | |||
81 | 81 | ||
82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 82 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
83 | 83 | ||
84 | return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) | 84 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
85 | return res.redirect(videoFile.getObjectStorageUrl()) | ||
86 | } | ||
87 | |||
88 | await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => { | ||
89 | const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}` | ||
90 | |||
91 | return res.download(path, filename) | ||
92 | }) | ||
85 | } | 93 | } |
86 | 94 | ||
87 | async function downloadHLSVideoFile (req: express.Request, res: express.Response) { | 95 | async function downloadHLSVideoFile (req: express.Request, res: express.Response) { |
@@ -107,8 +115,15 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response | |||
107 | 115 | ||
108 | if (!checkAllowResult(res, allowParameters, allowedResult)) return | 116 | if (!checkAllowResult(res, allowParameters, allowedResult)) return |
109 | 117 | ||
110 | const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` | 118 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { |
111 | return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename) | 119 | return res.redirect(videoFile.getObjectStorageUrl()) |
120 | } | ||
121 | |||
122 | await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => { | ||
123 | const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` | ||
124 | |||
125 | return res.download(path, filename) | ||
126 | }) | ||
112 | } | 127 | } |
113 | 128 | ||
114 | function getVideoFile (req: express.Request, files: MVideoFile[]) { | 129 | function getVideoFile (req: express.Request, files: MVideoFile[]) { |
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index ecf63e93e..c84376304 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -6,7 +6,8 @@ import { dirname, join } from 'path' | |||
6 | import * as WebTorrent from 'webtorrent' | 6 | import * as WebTorrent from 'webtorrent' |
7 | import { isArray } from '@server/helpers/custom-validators/misc' | 7 | import { isArray } from '@server/helpers/custom-validators/misc' |
8 | import { WEBSERVER } from '@server/initializers/constants' | 8 | import { WEBSERVER } from '@server/initializers/constants' |
9 | import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' | 9 | import { generateTorrentFileName } from '@server/lib/paths' |
10 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
10 | import { MVideo } from '@server/types/models/video/video' | 11 | import { MVideo } from '@server/types/models/video/video' |
11 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' | 12 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' |
12 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' | 13 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' |
@@ -78,7 +79,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName | |||
78 | }) | 79 | }) |
79 | } | 80 | } |
80 | 81 | ||
81 | async function createTorrentAndSetInfoHash ( | 82 | function createTorrentAndSetInfoHash ( |
82 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | 83 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, |
83 | videoFile: MVideoFile | 84 | videoFile: MVideoFile |
84 | ) { | 85 | ) { |
@@ -95,22 +96,24 @@ async function createTorrentAndSetInfoHash ( | |||
95 | urlList: [ videoFile.getFileUrl(video) ] | 96 | urlList: [ videoFile.getFileUrl(video) ] |
96 | } | 97 | } |
97 | 98 | ||
98 | const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) | 99 | return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { |
100 | const torrent = await createTorrentPromise(videoPath, options) | ||
99 | 101 | ||
100 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) | 102 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) |
101 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) | 103 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) |
102 | logger.info('Creating torrent %s.', torrentPath) | 104 | logger.info('Creating torrent %s.', torrentPath) |
103 | 105 | ||
104 | await writeFile(torrentPath, torrent) | 106 | await writeFile(torrentPath, torrent) |
105 | 107 | ||
106 | // Remove old torrent file if it existed | 108 | // Remove old torrent file if it existed |
107 | if (videoFile.hasTorrent()) { | 109 | if (videoFile.hasTorrent()) { |
108 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) | 110 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) |
109 | } | 111 | } |
110 | 112 | ||
111 | const parsedTorrent = parseTorrent(torrent) | 113 | const parsedTorrent = parseTorrent(torrent) |
112 | videoFile.infoHash = parsedTorrent.infoHash | 114 | videoFile.infoHash = parsedTorrent.infoHash |
113 | videoFile.torrentFilename = torrentFilename | 115 | videoFile.torrentFilename = torrentFilename |
116 | }) | ||
114 | } | 117 | } |
115 | 118 | ||
116 | function generateMagnetUri ( | 119 | function generateMagnetUri ( |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index 911734fa0..09f587274 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -153,6 +153,29 @@ function checkConfig () { | |||
153 | } | 153 | } |
154 | } | 154 | } |
155 | 155 | ||
156 | // Object storage | ||
157 | if (CONFIG.OBJECT_STORAGE.ENABLED === true) { | ||
158 | |||
159 | if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) { | ||
160 | return 'videos_bucket should be set when object storage support is enabled.' | ||
161 | } | ||
162 | |||
163 | if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { | ||
164 | return 'streaming_playlists_bucket should be set when object storage support is enabled.' | ||
165 | } | ||
166 | |||
167 | if ( | ||
168 | CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && | ||
169 | CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX | ||
170 | ) { | ||
171 | if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') { | ||
172 | return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.' | ||
173 | } else { | ||
174 | return 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' | ||
175 | } | ||
176 | } | ||
177 | } | ||
178 | |||
156 | return null | 179 | return null |
157 | } | 180 | } |
158 | 181 | ||
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 30a9823b9..0e684eef8 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -73,6 +73,26 @@ const CONFIG = { | |||
73 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), | 73 | PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), |
74 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) | 74 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) |
75 | }, | 75 | }, |
76 | OBJECT_STORAGE: { | ||
77 | ENABLED: config.get<boolean>('object_storage.enabled'), | ||
78 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), | ||
79 | ENDPOINT: config.get<string>('object_storage.endpoint'), | ||
80 | REGION: config.get<string>('object_storage.region'), | ||
81 | CREDENTIALS: { | ||
82 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), | ||
83 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') | ||
84 | }, | ||
85 | VIDEOS: { | ||
86 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), | ||
87 | PREFIX: config.get<string>('object_storage.videos.prefix'), | ||
88 | BASE_URL: config.get<string>('object_storage.videos.base_url') | ||
89 | }, | ||
90 | STREAMING_PLAYLISTS: { | ||
91 | BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'), | ||
92 | PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'), | ||
93 | BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url') | ||
94 | } | ||
95 | }, | ||
76 | WEBSERVER: { | 96 | WEBSERVER: { |
77 | SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', | 97 | SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', |
78 | WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', | 98 | WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5f121d9a4..8a1526ae8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
24 | 24 | ||
25 | // --------------------------------------------------------------------------- | 25 | // --------------------------------------------------------------------------- |
26 | 26 | ||
27 | const LAST_MIGRATION_VERSION = 655 | 27 | const LAST_MIGRATION_VERSION = 660 |
28 | 28 | ||
29 | // --------------------------------------------------------------------------- | 29 | // --------------------------------------------------------------------------- |
30 | 30 | ||
@@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
147 | 'videos-views': 1, | 147 | 'videos-views': 1, |
148 | 'activitypub-refresher': 1, | 148 | 'activitypub-refresher': 1, |
149 | 'video-redundancy': 1, | 149 | 'video-redundancy': 1, |
150 | 'video-live-ending': 1 | 150 | 'video-live-ending': 1, |
151 | 'move-to-object-storage': 3 | ||
151 | } | 152 | } |
152 | // Excluded keys are jobs that can be configured by admins | 153 | // Excluded keys are jobs that can be configured by admins |
153 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { | 154 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { |
@@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im | |||
162 | 'videos-views': 1, | 163 | 'videos-views': 1, |
163 | 'activitypub-refresher': 1, | 164 | 'activitypub-refresher': 1, |
164 | 'video-redundancy': 1, | 165 | 'video-redundancy': 1, |
165 | 'video-live-ending': 10 | 166 | 'video-live-ending': 10, |
167 | 'move-to-object-storage': 1 | ||
166 | } | 168 | } |
167 | const JOB_TTL: { [id in JobType]: number } = { | 169 | const JOB_TTL: { [id in JobType]: number } = { |
168 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | 170 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes |
@@ -178,7 +180,8 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
178 | 'videos-views': undefined, // Unlimited | 180 | 'videos-views': undefined, // Unlimited |
179 | 'activitypub-refresher': 60000 * 10, // 10 minutes | 181 | 'activitypub-refresher': 60000 * 10, // 10 minutes |
180 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours | 182 | 'video-redundancy': 1000 * 3600 * 3, // 3 hours |
181 | 'video-live-ending': 1000 * 60 * 10 // 10 minutes | 183 | 'video-live-ending': 1000 * 60 * 10, // 10 minutes |
184 | 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours | ||
182 | } | 185 | } |
183 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { | 186 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { |
184 | 'videos-views': { | 187 | 'videos-views': { |
@@ -412,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = { | |||
412 | [VideoState.TO_TRANSCODE]: 'To transcode', | 415 | [VideoState.TO_TRANSCODE]: 'To transcode', |
413 | [VideoState.TO_IMPORT]: 'To import', | 416 | [VideoState.TO_IMPORT]: 'To import', |
414 | [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', | 417 | [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', |
415 | [VideoState.LIVE_ENDED]: 'Livestream ended' | 418 | [VideoState.LIVE_ENDED]: 'Livestream ended', |
419 | [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage' | ||
416 | } | 420 | } |
417 | 421 | ||
418 | const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { | 422 | const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 38e7a76d0..0e690f6ae 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -45,6 +45,7 @@ import { VideoTagModel } from '../models/video/video-tag' | |||
45 | import { VideoViewModel } from '../models/video/video-view' | 45 | import { VideoViewModel } from '../models/video/video-view' |
46 | import { CONFIG } from './config' | 46 | import { CONFIG } from './config' |
47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | 47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' |
48 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
48 | 49 | ||
49 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 50 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
50 | 51 | ||
@@ -143,7 +144,8 @@ async function initDatabaseModels (silent: boolean) { | |||
143 | TrackerModel, | 144 | TrackerModel, |
144 | VideoTrackerModel, | 145 | VideoTrackerModel, |
145 | PluginModel, | 146 | PluginModel, |
146 | ActorCustomPageModel | 147 | ActorCustomPageModel, |
148 | VideoJobInfoModel | ||
147 | ]) | 149 | ]) |
148 | 150 | ||
149 | // Check extensions exist in the database | 151 | // Check extensions exist in the database |
diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts index 1aeb27f2d..ac952a98c 100644 --- a/server/initializers/migrations/0065-video-file-size.ts +++ b/server/initializers/migrations/0065-video-file-size.ts | |||
@@ -1,7 +1,4 @@ | |||
1 | import * as Sequelize from 'sequelize' | 1 | import * as Sequelize from 'sequelize' |
2 | import { stat } from 'fs-extra' | ||
3 | import { VideoModel } from '../../models/video/video' | ||
4 | import { getVideoFilePath } from '@server/lib/video-paths' | ||
5 | 2 | ||
6 | function up (utils: { | 3 | function up (utils: { |
7 | transaction: Sequelize.Transaction | 4 | transaction: Sequelize.Transaction |
@@ -9,30 +6,7 @@ function up (utils: { | |||
9 | sequelize: Sequelize.Sequelize | 6 | sequelize: Sequelize.Sequelize |
10 | db: any | 7 | db: any |
11 | }): Promise<void> { | 8 | }): Promise<void> { |
12 | return utils.db.Video.listOwnedAndPopulateAuthorAndTags() | 9 | throw new Error('Removed, please upgrade from a previous version first.') |
13 | .then((videos: VideoModel[]) => { | ||
14 | const tasks: Promise<any>[] = [] | ||
15 | |||
16 | videos.forEach(video => { | ||
17 | video.VideoFiles.forEach(videoFile => { | ||
18 | const p = new Promise((res, rej) => { | ||
19 | stat(getVideoFilePath(video, videoFile), (err, stats) => { | ||
20 | if (err) return rej(err) | ||
21 | |||
22 | videoFile.size = stats.size | ||
23 | videoFile.save().then(res).catch(rej) | ||
24 | }) | ||
25 | }) | ||
26 | |||
27 | tasks.push(p) | ||
28 | }) | ||
29 | }) | ||
30 | |||
31 | return tasks | ||
32 | }) | ||
33 | .then((tasks: Promise<any>[]) => { | ||
34 | return Promise.all(tasks) | ||
35 | }) | ||
36 | } | 10 | } |
37 | 11 | ||
38 | function down (options) { | 12 | function down (options) { |
diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts new file mode 100644 index 000000000..c815c71c6 --- /dev/null +++ b/server/initializers/migrations/0660-object-storage.ts | |||
@@ -0,0 +1,58 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { VideoStorage } from '@shared/models' | ||
3 | |||
4 | async function up (utils: { | ||
5 | transaction: Sequelize.Transaction | ||
6 | queryInterface: Sequelize.QueryInterface | ||
7 | sequelize: Sequelize.Sequelize | ||
8 | db: any | ||
9 | }): Promise<void> { | ||
10 | { | ||
11 | const query = ` | ||
12 | CREATE TABLE IF NOT EXISTS "videoJobInfo" ( | ||
13 | "id" serial, | ||
14 | "pendingMove" INTEGER NOT NULL, | ||
15 | "pendingTranscode" INTEGER NOT NULL, | ||
16 | "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
17 | "createdAt" timestamp WITH time zone NOT NULL, | ||
18 | "updatedAt" timestamp WITH time zone NOT NULL, | ||
19 | PRIMARY KEY ("id") | ||
20 | ); | ||
21 | ` | ||
22 | |||
23 | await utils.sequelize.query(query) | ||
24 | } | ||
25 | |||
26 | { | ||
27 | await utils.queryInterface.addColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: true }) | ||
28 | } | ||
29 | { | ||
30 | await utils.sequelize.query( | ||
31 | `UPDATE "videoFile" SET "storage" = ${VideoStorage.FILE_SYSTEM}` | ||
32 | ) | ||
33 | } | ||
34 | { | ||
35 | await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false }) | ||
36 | } | ||
37 | |||
38 | { | ||
39 | await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: true }) | ||
40 | } | ||
41 | { | ||
42 | await utils.sequelize.query( | ||
43 | `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.FILE_SYSTEM}` | ||
44 | ) | ||
45 | } | ||
46 | { | ||
47 | await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: false }) | ||
48 | } | ||
49 | } | ||
50 | |||
51 | function down (options) { | ||
52 | throw new Error('Not implemented.') | ||
53 | } | ||
54 | |||
55 | export { | ||
56 | up, | ||
57 | down | ||
58 | } | ||
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index 1fa16295d..bd9ed45a9 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -6,7 +6,7 @@ import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/vide | |||
6 | import { logger } from '@server/helpers/logger' | 6 | import { logger } from '@server/helpers/logger' |
7 | import { getExtFromMimetype } from '@server/helpers/video' | 7 | import { getExtFromMimetype } from '@server/helpers/video' |
8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' | 8 | import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' |
9 | import { generateTorrentFileName } from '@server/lib/video-paths' | 9 | import { generateTorrentFileName } from '@server/lib/paths' |
10 | import { VideoCaptionModel } from '@server/models/video/video-caption' | 10 | import { VideoCaptionModel } from '@server/models/video/video-caption' |
11 | import { VideoFileModel } from '@server/models/video/video-file' | 11 | import { VideoFileModel } from '@server/models/video/video-file' |
12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 12 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 0e77ab9fa..0828a2d0f 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
2 | import { flatten, uniq } from 'lodash' | 2 | import { flatten, uniq } from 'lodash' |
3 | import { basename, dirname, join } from 'path' | 3 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' | 4 | import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' |
@@ -8,11 +8,12 @@ import { logger } from '../helpers/logger' | |||
8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | 8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
9 | import { generateRandomString } from '../helpers/utils' | 9 | import { generateRandomString } from '../helpers/utils' |
10 | import { CONFIG } from '../initializers/config' | 10 | import { CONFIG } from '../initializers/config' |
11 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' | 11 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' |
12 | import { sequelizeTypescript } from '../initializers/database' | 12 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { VideoFileModel } from '../models/video/video-file' | 13 | import { VideoFileModel } from '../models/video/video-file' |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
15 | import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' | 15 | import { getHlsResolutionPlaylistFilename } from './paths' |
16 | import { VideoPathManager } from './video-path-manager' | ||
16 | 17 | ||
17 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | 18 | async function updateStreamingPlaylistsInfohashesIfNeeded () { |
18 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | 19 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() |
@@ -31,75 +32,66 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { | |||
31 | } | 32 | } |
32 | 33 | ||
33 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 34 | async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
34 | const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
35 | |||
36 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | 35 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
37 | 36 | ||
38 | const masterPlaylistPath = join(directory, playlist.playlistFilename) | ||
39 | |||
40 | for (const file of playlist.VideoFiles) { | 37 | for (const file of playlist.VideoFiles) { |
41 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | 38 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) |
42 | 39 | ||
43 | // If we did not generated a playlist for this resolution, skip | 40 | await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => { |
44 | const filePlaylistPath = join(directory, playlistFilename) | 41 | const size = await getVideoStreamSize(videoFilePath) |
45 | if (await pathExists(filePlaylistPath) === false) continue | ||
46 | |||
47 | const videoFilePath = getVideoFilePath(playlist, file) | ||
48 | 42 | ||
49 | const size = await getVideoStreamSize(videoFilePath) | 43 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) |
44 | const resolution = `RESOLUTION=${size.width}x${size.height}` | ||
50 | 45 | ||
51 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | 46 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` |
52 | const resolution = `RESOLUTION=${size.width}x${size.height}` | 47 | if (file.fps) line += ',FRAME-RATE=' + file.fps |
53 | 48 | ||
54 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | 49 | const codecs = await Promise.all([ |
55 | if (file.fps) line += ',FRAME-RATE=' + file.fps | 50 | getVideoStreamCodec(videoFilePath), |
51 | getAudioStreamCodec(videoFilePath) | ||
52 | ]) | ||
56 | 53 | ||
57 | const codecs = await Promise.all([ | 54 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
58 | getVideoStreamCodec(videoFilePath), | ||
59 | getAudioStreamCodec(videoFilePath) | ||
60 | ]) | ||
61 | 55 | ||
62 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` | 56 | masterPlaylists.push(line) |
63 | 57 | masterPlaylists.push(playlistFilename) | |
64 | masterPlaylists.push(line) | 58 | }) |
65 | masterPlaylists.push(playlistFilename) | ||
66 | } | 59 | } |
67 | 60 | ||
68 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 61 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { |
62 | return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | ||
63 | }) | ||
69 | } | 64 | } |
70 | 65 | ||
71 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { | 66 | async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { |
72 | const json: { [filename: string]: { [range: string]: string } } = {} | 67 | const json: { [filename: string]: { [range: string]: string } } = {} |
73 | 68 | ||
74 | const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | ||
75 | |||
76 | // For all the resolutions available for this video | 69 | // For all the resolutions available for this video |
77 | for (const file of playlist.VideoFiles) { | 70 | for (const file of playlist.VideoFiles) { |
78 | const rangeHashes: { [range: string]: string } = {} | 71 | const rangeHashes: { [range: string]: string } = {} |
79 | 72 | ||
80 | const videoPath = getVideoFilePath(playlist, file) | 73 | await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => { |
81 | const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename)) | ||
82 | |||
83 | // Maybe the playlist is not generated for this resolution yet | ||
84 | if (!await pathExists(resolutionPlaylistPath)) continue | ||
85 | 74 | ||
86 | const playlistContent = await readFile(resolutionPlaylistPath) | 75 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => { |
87 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | 76 | const playlistContent = await readFile(resolutionPlaylistPath) |
77 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
88 | 78 | ||
89 | const fd = await open(videoPath, 'r') | 79 | const fd = await open(videoPath, 'r') |
90 | for (const range of ranges) { | 80 | for (const range of ranges) { |
91 | const buf = Buffer.alloc(range.length) | 81 | const buf = Buffer.alloc(range.length) |
92 | await read(fd, buf, 0, range.length, range.offset) | 82 | await read(fd, buf, 0, range.length, range.offset) |
93 | 83 | ||
94 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | 84 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) |
95 | } | 85 | } |
96 | await close(fd) | 86 | await close(fd) |
97 | 87 | ||
98 | const videoFilename = file.filename | 88 | const videoFilename = file.filename |
99 | json[videoFilename] = rangeHashes | 89 | json[videoFilename] = rangeHashes |
90 | }) | ||
91 | }) | ||
100 | } | 92 | } |
101 | 93 | ||
102 | const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) | 94 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
103 | await outputJSON(outputPath, json) | 95 | await outputJSON(outputPath, json) |
104 | } | 96 | } |
105 | 97 | ||
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts new file mode 100644 index 000000000..a0c58d211 --- /dev/null +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -0,0 +1,114 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { remove } from 'fs-extra' | ||
3 | import { join } from 'path' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
6 | import { CONFIG } from '@server/initializers/config' | ||
7 | import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage' | ||
8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
9 | import { moveToNextState } from '@server/lib/video-state' | ||
10 | import { VideoModel } from '@server/models/video/video' | ||
11 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
12 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' | ||
13 | import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared' | ||
14 | |||
15 | export async function processMoveToObjectStorage (job: Bull.Job) { | ||
16 | const payload = job.data as MoveObjectStoragePayload | ||
17 | logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) | ||
18 | |||
19 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | ||
20 | // No video, maybe deleted? | ||
21 | if (!video) { | ||
22 | logger.info('Can\'t process job %d, video does not exist.', job.id) | ||
23 | return undefined | ||
24 | } | ||
25 | |||
26 | if (video.VideoFiles) { | ||
27 | await moveWebTorrentFiles(video) | ||
28 | } | ||
29 | |||
30 | if (video.VideoStreamingPlaylists) { | ||
31 | await moveHLSFiles(video) | ||
32 | } | ||
33 | |||
34 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') | ||
35 | if (pendingMove === 0) { | ||
36 | logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) | ||
37 | await doAfterLastJob(video, payload.isNewVideo) | ||
38 | } | ||
39 | |||
40 | return payload.videoUUID | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | ||
46 | for (const file of video.VideoFiles) { | ||
47 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
48 | |||
49 | const fileUrl = await storeWebTorrentFile(file.filename) | ||
50 | |||
51 | const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename) | ||
52 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | ||
53 | } | ||
54 | } | ||
55 | |||
56 | async function moveHLSFiles (video: MVideoWithAllFiles) { | ||
57 | for (const playlist of video.VideoStreamingPlaylists) { | ||
58 | |||
59 | for (const file of playlist.VideoFiles) { | ||
60 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | ||
61 | |||
62 | // Resolution playlist | ||
63 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | ||
64 | await storeHLSFile(playlist, video, playlistFilename) | ||
65 | |||
66 | // Resolution fragmented file | ||
67 | const fileUrl = await storeHLSFile(playlist, video, file.filename) | ||
68 | |||
69 | const oldPath = join(getHLSDirectory(video), file.filename) | ||
70 | |||
71 | await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) | ||
72 | } | ||
73 | } | ||
74 | } | ||
75 | |||
76 | async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) { | ||
77 | for (const playlist of video.VideoStreamingPlaylists) { | ||
78 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue | ||
79 | |||
80 | // Master playlist | ||
81 | playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename) | ||
82 | // Sha256 segments file | ||
83 | playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename) | ||
84 | |||
85 | playlist.storage = VideoStorage.OBJECT_STORAGE | ||
86 | |||
87 | await playlist.save() | ||
88 | } | ||
89 | |||
90 | // Remove empty hls video directory | ||
91 | if (video.VideoStreamingPlaylists) { | ||
92 | await remove(getHLSDirectory(video)) | ||
93 | } | ||
94 | |||
95 | await moveToNextState(video, isNewVideo) | ||
96 | } | ||
97 | |||
98 | async function onFileMoved (options: { | ||
99 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo | ||
100 | file: MVideoFile | ||
101 | fileUrl: string | ||
102 | oldPath: string | ||
103 | }) { | ||
104 | const { videoOrPlaylist, file, fileUrl, oldPath } = options | ||
105 | |||
106 | file.fileUrl = fileUrl | ||
107 | file.storage = VideoStorage.OBJECT_STORAGE | ||
108 | |||
109 | await createTorrentAndSetInfoHash(videoOrPlaylist, file) | ||
110 | await file.save() | ||
111 | |||
112 | logger.debug('Removing %s because it\'s now on object storage', oldPath) | ||
113 | await remove(oldPath) | ||
114 | } | ||
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 2f4abf730..e8ee1f759 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -2,15 +2,19 @@ import * as Bull from 'bull' | |||
2 | import { copy, stat } from 'fs-extra' | 2 | import { copy, stat } from 'fs-extra' |
3 | import { getLowercaseExtension } from '@server/helpers/core-utils' | 3 | import { getLowercaseExtension } from '@server/helpers/core-utils' |
4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 4 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
5 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 5 | import { CONFIG } from '@server/initializers/config' |
6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
8 | import { addMoveToObjectStorageJob } from '@server/lib/video' | ||
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
6 | import { UserModel } from '@server/models/user/user' | 10 | import { UserModel } from '@server/models/user/user' |
7 | import { MVideoFullLight } from '@server/types/models' | 11 | import { MVideoFullLight } from '@server/types/models' |
8 | import { VideoFileImportPayload } from '@shared/models' | 12 | import { VideoFileImportPayload, VideoStorage } from '@shared/models' |
9 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' | 13 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' |
10 | import { logger } from '../../../helpers/logger' | 14 | import { logger } from '../../../helpers/logger' |
11 | import { VideoModel } from '../../../models/video/video' | 15 | import { VideoModel } from '../../../models/video/video' |
12 | import { VideoFileModel } from '../../../models/video/video-file' | 16 | import { VideoFileModel } from '../../../models/video/video-file' |
13 | import { onNewWebTorrentFileResolution } from './video-transcoding' | 17 | import { createHlsJobIfEnabled } from './video-transcoding' |
14 | 18 | ||
15 | async function processVideoFileImport (job: Bull.Job) { | 19 | async function processVideoFileImport (job: Bull.Job) { |
16 | const payload = job.data as VideoFileImportPayload | 20 | const payload = job.data as VideoFileImportPayload |
@@ -29,15 +33,19 @@ async function processVideoFileImport (job: Bull.Job) { | |||
29 | 33 | ||
30 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) | 34 | const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) |
31 | 35 | ||
32 | const newResolutionPayload = { | 36 | await createHlsJobIfEnabled(user, { |
33 | type: 'new-resolution-to-webtorrent' as 'new-resolution-to-webtorrent', | ||
34 | videoUUID: video.uuid, | 37 | videoUUID: video.uuid, |
35 | resolution: data.resolution, | 38 | resolution: data.resolution, |
36 | isPortraitMode: data.isPortraitMode, | 39 | isPortraitMode: data.isPortraitMode, |
37 | copyCodecs: false, | 40 | copyCodecs: true, |
38 | isNewVideo: false | 41 | isMaxQuality: false |
42 | }) | ||
43 | |||
44 | if (CONFIG.OBJECT_STORAGE.ENABLED) { | ||
45 | await addMoveToObjectStorageJob(video) | ||
46 | } else { | ||
47 | await federateVideoIfNeeded(video, false) | ||
39 | } | 48 | } |
40 | await onNewWebTorrentFileResolution(video, user, newResolutionPayload) | ||
41 | 49 | ||
42 | return video | 50 | return video |
43 | } | 51 | } |
@@ -72,12 +80,13 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
72 | resolution, | 80 | resolution, |
73 | extname: fileExt, | 81 | extname: fileExt, |
74 | filename: generateWebTorrentVideoFilename(resolution, fileExt), | 82 | filename: generateWebTorrentVideoFilename(resolution, fileExt), |
83 | storage: VideoStorage.FILE_SYSTEM, | ||
75 | size, | 84 | size, |
76 | fps, | 85 | fps, |
77 | videoId: video.id | 86 | videoId: video.id |
78 | }) | 87 | }) |
79 | 88 | ||
80 | const outputPath = getVideoFilePath(video, newVideoFile) | 89 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
81 | await copy(inputFilePath, outputPath) | 90 | await copy(inputFilePath, outputPath) |
82 | 91 | ||
83 | video.VideoFiles.push(newVideoFile) | 92 | video.VideoFiles.push(newVideoFile) |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index fec553f2b..a5fa204f5 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -4,11 +4,13 @@ import { getLowercaseExtension } from '@server/helpers/core-utils' | |||
4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | 4 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
5 | import { YoutubeDL } from '@server/helpers/youtube-dl' | 5 | import { YoutubeDL } from '@server/helpers/youtube-dl' |
6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | 6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' |
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
7 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
8 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
9 | import { isAbleToUploadVideo } from '@server/lib/user' | 10 | import { isAbleToUploadVideo } from '@server/lib/user' |
10 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' | 11 | import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video' |
11 | import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 12 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { buildNextVideoState } from '@server/lib/video-state' | ||
12 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 14 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
13 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | 15 | import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' |
14 | import { | 16 | import { |
@@ -25,7 +27,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro | |||
25 | import { logger } from '../../../helpers/logger' | 27 | import { logger } from '../../../helpers/logger' |
26 | import { getSecureTorrentName } from '../../../helpers/utils' | 28 | import { getSecureTorrentName } from '../../../helpers/utils' |
27 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 29 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
28 | import { CONFIG } from '../../../initializers/config' | ||
29 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' | 30 | import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' |
30 | import { sequelizeTypescript } from '../../../initializers/database' | 31 | import { sequelizeTypescript } from '../../../initializers/database' |
31 | import { VideoModel } from '../../../models/video/video' | 32 | import { VideoModel } from '../../../models/video/video' |
@@ -100,7 +101,6 @@ type ProcessFileOptions = { | |||
100 | } | 101 | } |
101 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { | 102 | async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { |
102 | let tempVideoPath: string | 103 | let tempVideoPath: string |
103 | let videoDestFile: string | ||
104 | let videoFile: VideoFileModel | 104 | let videoFile: VideoFileModel |
105 | 105 | ||
106 | try { | 106 | try { |
@@ -159,7 +159,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
159 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) | 159 | const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) |
160 | 160 | ||
161 | // Move file | 161 | // Move file |
162 | videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) | 162 | const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) |
163 | await move(tempVideoPath, videoDestFile) | 163 | await move(tempVideoPath, videoDestFile) |
164 | tempVideoPath = null // This path is not used anymore | 164 | tempVideoPath = null // This path is not used anymore |
165 | 165 | ||
@@ -204,7 +204,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
204 | 204 | ||
205 | // Update video DB object | 205 | // Update video DB object |
206 | video.duration = duration | 206 | video.duration = duration |
207 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED | 207 | video.state = buildNextVideoState(video.state) |
208 | await video.save({ transaction: t }) | 208 | await video.save({ transaction: t }) |
209 | 209 | ||
210 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) | 210 | if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) |
@@ -245,6 +245,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
245 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 245 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) |
246 | } | 246 | } |
247 | 247 | ||
248 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
249 | return addMoveToObjectStorageJob(videoImportUpdated.Video) | ||
250 | } | ||
251 | |||
248 | // Create transcoding jobs? | 252 | // Create transcoding jobs? |
249 | if (video.state === VideoState.TO_TRANSCODE) { | 253 | if (video.state === VideoState.TO_TRANSCODE) { |
250 | await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) | 254 | await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) |
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index aa5bd573a..9ccf724c2 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -4,10 +4,11 @@ import { join } from 'path' | |||
4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' | 4 | import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' |
5 | import { VIDEO_LIVE } from '@server/initializers/constants' | 5 | import { VIDEO_LIVE } from '@server/initializers/constants' |
6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' | 6 | import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' |
7 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory } from '@server/lib/paths' | ||
7 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 8 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
8 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' | 9 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' |
9 | import { publishAndFederateIfNeeded } from '@server/lib/video' | 10 | import { VideoPathManager } from '@server/lib/video-path-manager' |
10 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' | 11 | import { moveToNextState } from '@server/lib/video-state' |
11 | import { VideoModel } from '@server/models/video/video' | 12 | import { VideoModel } from '@server/models/video/video' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 13 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { VideoLiveModel } from '@server/models/video/video-live' | 14 | import { VideoLiveModel } from '@server/models/video/video-live' |
@@ -55,16 +56,15 @@ export { | |||
55 | // --------------------------------------------------------------------------- | 56 | // --------------------------------------------------------------------------- |
56 | 57 | ||
57 | async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { | 58 | async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { |
58 | const hlsDirectory = getHLSDirectory(video, false) | 59 | const replayDirectory = VideoPathManager.Instance.getFSHLSOutputPath(video, VIDEO_LIVE.REPLAY_DIRECTORY) |
59 | const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY) | ||
60 | 60 | ||
61 | const rootFiles = await readdir(hlsDirectory) | 61 | const rootFiles = await readdir(getLiveDirectory(video)) |
62 | 62 | ||
63 | const playlistFiles = rootFiles.filter(file => { | 63 | const playlistFiles = rootFiles.filter(file => { |
64 | return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename | 64 | return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename |
65 | }) | 65 | }) |
66 | 66 | ||
67 | await cleanupLiveFiles(hlsDirectory) | 67 | await cleanupTMPLiveFiles(getLiveDirectory(video)) |
68 | 68 | ||
69 | await live.destroy() | 69 | await live.destroy() |
70 | 70 | ||
@@ -98,7 +98,7 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt | |||
98 | 98 | ||
99 | const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) | 99 | const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) |
100 | 100 | ||
101 | const outputPath = await generateHlsPlaylistResolutionFromTS({ | 101 | const { resolutionPlaylistPath: outputPath } = await generateHlsPlaylistResolutionFromTS({ |
102 | video: videoWithFiles, | 102 | video: videoWithFiles, |
103 | concatenatedTsFilePath, | 103 | concatenatedTsFilePath, |
104 | resolution, | 104 | resolution, |
@@ -133,10 +133,10 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt | |||
133 | }) | 133 | }) |
134 | } | 134 | } |
135 | 135 | ||
136 | await publishAndFederateIfNeeded(videoWithFiles, true) | 136 | await moveToNextState(videoWithFiles, false) |
137 | } | 137 | } |
138 | 138 | ||
139 | async function cleanupLiveFiles (hlsDirectory: string) { | 139 | async function cleanupTMPLiveFiles (hlsDirectory: string) { |
140 | if (!await pathExists(hlsDirectory)) return | 140 | if (!await pathExists(hlsDirectory)) return |
141 | 141 | ||
142 | const files = await readdir(hlsDirectory) | 142 | const files = await readdir(hlsDirectory) |
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 876d1460c..b3149dde8 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -1,9 +1,11 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' | 2 | import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' |
3 | import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' | 3 | import { addTranscodingJob, getTranscodingJobPriority } from '@server/lib/video' |
4 | import { getVideoFilePath } from '@server/lib/video-paths' | 4 | import { VideoPathManager } from '@server/lib/video-path-manager' |
5 | import { moveToNextState } from '@server/lib/video-state' | ||
5 | import { UserModel } from '@server/models/user/user' | 6 | import { UserModel } from '@server/models/user/user' |
6 | import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' | 7 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
8 | import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models' | ||
7 | import { | 9 | import { |
8 | HLSTranscodingPayload, | 10 | HLSTranscodingPayload, |
9 | MergeAudioTranscodingPayload, | 11 | MergeAudioTranscodingPayload, |
@@ -16,17 +18,14 @@ import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' | |||
16 | import { logger } from '../../../helpers/logger' | 18 | import { logger } from '../../../helpers/logger' |
17 | import { CONFIG } from '../../../initializers/config' | 19 | import { CONFIG } from '../../../initializers/config' |
18 | import { VideoModel } from '../../../models/video/video' | 20 | import { VideoModel } from '../../../models/video/video' |
19 | import { federateVideoIfNeeded } from '../../activitypub/videos' | ||
20 | import { Notifier } from '../../notifier' | ||
21 | import { | 21 | import { |
22 | generateHlsPlaylistResolution, | 22 | generateHlsPlaylistResolution, |
23 | mergeAudioVideofile, | 23 | mergeAudioVideofile, |
24 | optimizeOriginalVideofile, | 24 | optimizeOriginalVideofile, |
25 | transcodeNewWebTorrentResolution | 25 | transcodeNewWebTorrentResolution |
26 | } from '../../transcoding/video-transcoding' | 26 | } from '../../transcoding/video-transcoding' |
27 | import { JobQueue } from '../job-queue' | ||
28 | 27 | ||
29 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> | 28 | type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void> |
30 | 29 | ||
31 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { | 30 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { |
32 | 'new-resolution-to-hls': handleHLSJob, | 31 | 'new-resolution-to-hls': handleHLSJob, |
@@ -69,15 +68,16 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide | |||
69 | : video.getMaxQualityFile() | 68 | : video.getMaxQualityFile() |
70 | 69 | ||
71 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | 70 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() |
72 | const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput) | ||
73 | 71 | ||
74 | await generateHlsPlaylistResolution({ | 72 | await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => { |
75 | video, | 73 | return generateHlsPlaylistResolution({ |
76 | videoInputPath, | 74 | video, |
77 | resolution: payload.resolution, | 75 | videoInputPath, |
78 | copyCodecs: payload.copyCodecs, | 76 | resolution: payload.resolution, |
79 | isPortraitMode: payload.isPortraitMode || false, | 77 | copyCodecs: payload.copyCodecs, |
80 | job | 78 | isPortraitMode: payload.isPortraitMode || false, |
79 | job | ||
80 | }) | ||
81 | }) | 81 | }) |
82 | 82 | ||
83 | await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) | 83 | await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) |
@@ -101,7 +101,7 @@ async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudio | |||
101 | } | 101 | } |
102 | 102 | ||
103 | async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 103 | async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
104 | const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) | 104 | const { transcodeType } = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) |
105 | 105 | ||
106 | await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) | 106 | await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) |
107 | } | 107 | } |
@@ -121,10 +121,18 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay | |||
121 | video.VideoFiles = [] | 121 | video.VideoFiles = [] |
122 | 122 | ||
123 | // Create HLS new resolution jobs | 123 | // Create HLS new resolution jobs |
124 | await createLowerResolutionsJobs(video, user, payload.resolution, payload.isPortraitMode, 'hls') | 124 | await createLowerResolutionsJobs({ |
125 | video, | ||
126 | user, | ||
127 | videoFileResolution: payload.resolution, | ||
128 | isPortraitMode: payload.isPortraitMode, | ||
129 | isNewVideo: payload.isNewVideo ?? true, | ||
130 | type: 'hls' | ||
131 | }) | ||
125 | } | 132 | } |
126 | 133 | ||
127 | return publishAndFederateIfNeeded(video) | 134 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') |
135 | await moveToNextState(video, payload.isNewVideo) | ||
128 | } | 136 | } |
129 | 137 | ||
130 | async function onVideoFileOptimizer ( | 138 | async function onVideoFileOptimizer ( |
@@ -143,58 +151,54 @@ async function onVideoFileOptimizer ( | |||
143 | // Video does not exist anymore | 151 | // Video does not exist anymore |
144 | if (!videoDatabase) return undefined | 152 | if (!videoDatabase) return undefined |
145 | 153 | ||
146 | let videoPublished = false | ||
147 | |||
148 | // Generate HLS version of the original file | 154 | // Generate HLS version of the original file |
149 | const originalFileHLSPayload = Object.assign({}, payload, { | 155 | const originalFileHLSPayload = { |
156 | ...payload, | ||
157 | |||
150 | isPortraitMode, | 158 | isPortraitMode, |
151 | resolution: videoDatabase.getMaxQualityFile().resolution, | 159 | resolution: videoDatabase.getMaxQualityFile().resolution, |
152 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | 160 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues |
153 | copyCodecs: transcodeType !== 'quick-transcode', | 161 | copyCodecs: transcodeType !== 'quick-transcode', |
154 | isMaxQuality: true | 162 | isMaxQuality: true |
155 | }) | 163 | } |
156 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | 164 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) |
165 | const hasNewResolutions = await createLowerResolutionsJobs({ | ||
166 | video: videoDatabase, | ||
167 | user, | ||
168 | videoFileResolution: resolution, | ||
169 | isPortraitMode, | ||
170 | type: 'webtorrent', | ||
171 | isNewVideo: payload.isNewVideo ?? true | ||
172 | }) | ||
157 | 173 | ||
158 | const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, resolution, isPortraitMode, 'webtorrent') | 174 | await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') |
159 | 175 | ||
176 | // Move to next state if there are no other resolutions to generate | ||
160 | if (!hasHls && !hasNewResolutions) { | 177 | if (!hasHls && !hasNewResolutions) { |
161 | // No transcoding to do, it's now published | 178 | await moveToNextState(videoDatabase, payload.isNewVideo) |
162 | videoPublished = await videoDatabase.publishIfNeededAndSave(undefined) | ||
163 | } | 179 | } |
164 | |||
165 | await federateVideoIfNeeded(videoDatabase, payload.isNewVideo) | ||
166 | |||
167 | if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) | ||
168 | if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) | ||
169 | } | 180 | } |
170 | 181 | ||
171 | async function onNewWebTorrentFileResolution ( | 182 | async function onNewWebTorrentFileResolution ( |
172 | video: MVideoUUID, | 183 | video: MVideo, |
173 | user: MUserId, | 184 | user: MUserId, |
174 | payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload | 185 | payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload |
175 | ) { | 186 | ) { |
176 | await publishAndFederateIfNeeded(video) | 187 | await createHlsJobIfEnabled(user, { ...payload, copyCodecs: true, isMaxQuality: false }) |
188 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
177 | 189 | ||
178 | await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true, isMaxQuality: false })) | 190 | await moveToNextState(video, payload.isNewVideo) |
179 | } | 191 | } |
180 | 192 | ||
181 | // --------------------------------------------------------------------------- | ||
182 | |||
183 | export { | ||
184 | processVideoTranscoding, | ||
185 | onNewWebTorrentFileResolution | ||
186 | } | ||
187 | |||
188 | // --------------------------------------------------------------------------- | ||
189 | |||
190 | async function createHlsJobIfEnabled (user: MUserId, payload: { | 193 | async function createHlsJobIfEnabled (user: MUserId, payload: { |
191 | videoUUID: string | 194 | videoUUID: string |
192 | resolution: number | 195 | resolution: number |
193 | isPortraitMode?: boolean | 196 | isPortraitMode?: boolean |
194 | copyCodecs: boolean | 197 | copyCodecs: boolean |
195 | isMaxQuality: boolean | 198 | isMaxQuality: boolean |
199 | isNewVideo?: boolean | ||
196 | }) { | 200 | }) { |
197 | if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false | 201 | if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false |
198 | 202 | ||
199 | const jobOptions = { | 203 | const jobOptions = { |
200 | priority: await getTranscodingJobPriority(user) | 204 | priority: await getTranscodingJobPriority(user) |
@@ -206,21 +210,35 @@ async function createHlsJobIfEnabled (user: MUserId, payload: { | |||
206 | resolution: payload.resolution, | 210 | resolution: payload.resolution, |
207 | isPortraitMode: payload.isPortraitMode, | 211 | isPortraitMode: payload.isPortraitMode, |
208 | copyCodecs: payload.copyCodecs, | 212 | copyCodecs: payload.copyCodecs, |
209 | isMaxQuality: payload.isMaxQuality | 213 | isMaxQuality: payload.isMaxQuality, |
214 | isNewVideo: payload.isNewVideo | ||
210 | } | 215 | } |
211 | 216 | ||
212 | JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions) | 217 | await addTranscodingJob(hlsTranscodingPayload, jobOptions) |
213 | 218 | ||
214 | return true | 219 | return true |
215 | } | 220 | } |
216 | 221 | ||
217 | async function createLowerResolutionsJobs ( | 222 | // --------------------------------------------------------------------------- |
218 | video: MVideoFullLight, | 223 | |
219 | user: MUserId, | 224 | export { |
220 | videoFileResolution: number, | 225 | processVideoTranscoding, |
221 | isPortraitMode: boolean, | 226 | createHlsJobIfEnabled, |
227 | onNewWebTorrentFileResolution | ||
228 | } | ||
229 | |||
230 | // --------------------------------------------------------------------------- | ||
231 | |||
232 | async function createLowerResolutionsJobs (options: { | ||
233 | video: MVideoFullLight | ||
234 | user: MUserId | ||
235 | videoFileResolution: number | ||
236 | isPortraitMode: boolean | ||
237 | isNewVideo: boolean | ||
222 | type: 'hls' | 'webtorrent' | 238 | type: 'hls' | 'webtorrent' |
223 | ) { | 239 | }) { |
240 | const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options | ||
241 | |||
224 | // Create transcoding jobs if there are enabled resolutions | 242 | // Create transcoding jobs if there are enabled resolutions |
225 | const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') | 243 | const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') |
226 | const resolutionCreated: number[] = [] | 244 | const resolutionCreated: number[] = [] |
@@ -234,7 +252,8 @@ async function createLowerResolutionsJobs ( | |||
234 | type: 'new-resolution-to-webtorrent', | 252 | type: 'new-resolution-to-webtorrent', |
235 | videoUUID: video.uuid, | 253 | videoUUID: video.uuid, |
236 | resolution, | 254 | resolution, |
237 | isPortraitMode | 255 | isPortraitMode, |
256 | isNewVideo | ||
238 | } | 257 | } |
239 | } | 258 | } |
240 | 259 | ||
@@ -245,7 +264,8 @@ async function createLowerResolutionsJobs ( | |||
245 | resolution, | 264 | resolution, |
246 | isPortraitMode, | 265 | isPortraitMode, |
247 | copyCodecs: false, | 266 | copyCodecs: false, |
248 | isMaxQuality: false | 267 | isMaxQuality: false, |
268 | isNewVideo | ||
249 | } | 269 | } |
250 | } | 270 | } |
251 | 271 | ||
@@ -257,7 +277,7 @@ async function createLowerResolutionsJobs ( | |||
257 | priority: await getTranscodingJobPriority(user) | 277 | priority: await getTranscodingJobPriority(user) |
258 | } | 278 | } |
259 | 279 | ||
260 | JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions) | 280 | await addTranscodingJob(dataInput, jobOptions) |
261 | } | 281 | } |
262 | 282 | ||
263 | if (resolutionCreated.length === 0) { | 283 | if (resolutionCreated.length === 0) { |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 42e8347b1..7a3a1bf82 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -11,6 +11,7 @@ import { | |||
11 | EmailPayload, | 11 | EmailPayload, |
12 | JobState, | 12 | JobState, |
13 | JobType, | 13 | JobType, |
14 | MoveObjectStoragePayload, | ||
14 | RefreshPayload, | 15 | RefreshPayload, |
15 | VideoFileImportPayload, | 16 | VideoFileImportPayload, |
16 | VideoImportPayload, | 17 | VideoImportPayload, |
@@ -34,6 +35,7 @@ import { processVideoImport } from './handlers/video-import' | |||
34 | import { processVideoLiveEnding } from './handlers/video-live-ending' | 35 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
35 | import { processVideoTranscoding } from './handlers/video-transcoding' | 36 | import { processVideoTranscoding } from './handlers/video-transcoding' |
36 | import { processVideosViews } from './handlers/video-views' | 37 | import { processVideosViews } from './handlers/video-views' |
38 | import { processMoveToObjectStorage } from './handlers/move-to-object-storage' | ||
37 | 39 | ||
38 | type CreateJobArgument = | 40 | type CreateJobArgument = |
39 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 41 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -49,9 +51,10 @@ type CreateJobArgument = | |||
49 | { type: 'videos-views', payload: {} } | | 51 | { type: 'videos-views', payload: {} } | |
50 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | | 52 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | |
51 | { type: 'actor-keys', payload: ActorKeysPayload } | | 53 | { type: 'actor-keys', payload: ActorKeysPayload } | |
52 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | 54 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | |
55 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | ||
53 | 56 | ||
54 | type CreateJobOptions = { | 57 | export type CreateJobOptions = { |
55 | delay?: number | 58 | delay?: number |
56 | priority?: number | 59 | priority?: number |
57 | } | 60 | } |
@@ -70,7 +73,8 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = { | |||
70 | 'activitypub-refresher': refreshAPObject, | 73 | 'activitypub-refresher': refreshAPObject, |
71 | 'video-live-ending': processVideoLiveEnding, | 74 | 'video-live-ending': processVideoLiveEnding, |
72 | 'actor-keys': processActorKeys, | 75 | 'actor-keys': processActorKeys, |
73 | 'video-redundancy': processVideoRedundancy | 76 | 'video-redundancy': processVideoRedundancy, |
77 | 'move-to-object-storage': processMoveToObjectStorage | ||
74 | } | 78 | } |
75 | 79 | ||
76 | const jobTypes: JobType[] = [ | 80 | const jobTypes: JobType[] = [ |
@@ -87,7 +91,8 @@ const jobTypes: JobType[] = [ | |||
87 | 'activitypub-refresher', | 91 | 'activitypub-refresher', |
88 | 'video-redundancy', | 92 | 'video-redundancy', |
89 | 'actor-keys', | 93 | 'actor-keys', |
90 | 'video-live-ending' | 94 | 'video-live-ending', |
95 | 'move-to-object-storage' | ||
91 | ] | 96 | ] |
92 | 97 | ||
93 | class JobQueue { | 98 | class JobQueue { |
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 2a429fb33..d7dc841d9 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts | |||
@@ -20,7 +20,7 @@ import { VideoState, VideoStreamingPlaylistType } from '@shared/models' | |||
20 | import { federateVideoIfNeeded } from '../activitypub/videos' | 20 | import { federateVideoIfNeeded } from '../activitypub/videos' |
21 | import { JobQueue } from '../job-queue' | 21 | import { JobQueue } from '../job-queue' |
22 | import { PeerTubeSocket } from '../peertube-socket' | 22 | import { PeerTubeSocket } from '../peertube-socket' |
23 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' | 23 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../paths' |
24 | import { LiveQuotaStore } from './live-quota-store' | 24 | import { LiveQuotaStore } from './live-quota-store' |
25 | import { LiveSegmentShaStore } from './live-segment-sha-store' | 25 | import { LiveSegmentShaStore } from './live-segment-sha-store' |
26 | import { cleanupLive } from './live-utils' | 26 | import { cleanupLive } from './live-utils' |
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index e4526c7a5..3bf723b98 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
2 | import { basename } from 'path' | 2 | import { basename } from 'path' |
3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 3 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
4 | import { getHLSDirectory } from '../video-paths' | 4 | import { getLiveDirectory } from '../paths' |
5 | 5 | ||
6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { | 6 | function buildConcatenatedName (segmentOrPlaylistPath: string) { |
7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) | 7 | const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) |
@@ -10,7 +10,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) { | |||
10 | } | 10 | } |
11 | 11 | ||
12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { | 12 | async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { |
13 | const hlsDirectory = getHLSDirectory(video) | 13 | const hlsDirectory = getLiveDirectory(video) |
14 | 14 | ||
15 | await remove(hlsDirectory) | 15 | await remove(hlsDirectory) |
16 | 16 | ||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index a80abc843..9b5b6c4fc 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -11,9 +11,9 @@ import { CONFIG } from '@server/initializers/config' | |||
11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' | 11 | import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' |
12 | import { VideoFileModel } from '@server/models/video/video-file' | 12 | import { VideoFileModel } from '@server/models/video/video-file' |
13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' | 13 | import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' |
14 | import { getLiveDirectory } from '../../paths' | ||
14 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' | 15 | import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' |
15 | import { isAbleToUploadVideo } from '../../user' | 16 | import { isAbleToUploadVideo } from '../../user' |
16 | import { getHLSDirectory } from '../../video-paths' | ||
17 | import { LiveQuotaStore } from '../live-quota-store' | 17 | import { LiveQuotaStore } from '../live-quota-store' |
18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | 18 | import { LiveSegmentShaStore } from '../live-segment-sha-store' |
19 | import { buildConcatenatedName } from '../live-utils' | 19 | import { buildConcatenatedName } from '../live-utils' |
@@ -282,7 +282,7 @@ class MuxingSession extends EventEmitter { | |||
282 | } | 282 | } |
283 | 283 | ||
284 | private async prepareDirectories () { | 284 | private async prepareDirectories () { |
285 | const outPath = getHLSDirectory(this.videoLive.Video) | 285 | const outPath = getLiveDirectory(this.videoLive.Video) |
286 | await ensureDir(outPath) | 286 | await ensureDir(outPath) |
287 | 287 | ||
288 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) | 288 | const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) |
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts new file mode 100644 index 000000000..8b413a40e --- /dev/null +++ b/server/lib/object-storage/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './keys' | ||
2 | export * from './urls' | ||
3 | export * from './videos' | ||
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts new file mode 100644 index 000000000..519474775 --- /dev/null +++ b/server/lib/object-storage/keys.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import { join } from 'path' | ||
2 | import { MStreamingPlaylist, MVideoUUID } from '@server/types/models' | ||
3 | |||
4 | function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { | ||
5 | return join(generateHLSObjectBaseStorageKey(playlist, video), filename) | ||
6 | } | ||
7 | |||
8 | function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { | ||
9 | return playlist.getStringType() + '_' + video.uuid | ||
10 | } | ||
11 | |||
12 | function generateWebTorrentObjectStorageKey (filename: string) { | ||
13 | return filename | ||
14 | } | ||
15 | |||
16 | export { | ||
17 | generateHLSObjectStorageKey, | ||
18 | generateHLSObjectBaseStorageKey, | ||
19 | generateWebTorrentObjectStorageKey | ||
20 | } | ||
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts new file mode 100644 index 000000000..c9a614593 --- /dev/null +++ b/server/lib/object-storage/shared/client.ts | |||
@@ -0,0 +1,56 @@ | |||
1 | import { S3Client } from '@aws-sdk/client-s3' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { lTags } from './logger' | ||
5 | |||
6 | let endpointParsed: URL | ||
7 | function getEndpointParsed () { | ||
8 | if (endpointParsed) return endpointParsed | ||
9 | |||
10 | endpointParsed = new URL(getEndpoint()) | ||
11 | |||
12 | return endpointParsed | ||
13 | } | ||
14 | |||
15 | let s3Client: S3Client | ||
16 | function getClient () { | ||
17 | if (s3Client) return s3Client | ||
18 | |||
19 | const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE | ||
20 | |||
21 | s3Client = new S3Client({ | ||
22 | endpoint: getEndpoint(), | ||
23 | region: OBJECT_STORAGE.REGION, | ||
24 | credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID | ||
25 | ? { | ||
26 | accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, | ||
27 | secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY | ||
28 | } | ||
29 | : undefined | ||
30 | }) | ||
31 | |||
32 | logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) | ||
33 | |||
34 | return s3Client | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | export { | ||
40 | getEndpointParsed, | ||
41 | getClient | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | let endpoint: string | ||
47 | function getEndpoint () { | ||
48 | if (endpoint) return endpoint | ||
49 | |||
50 | const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT | ||
51 | endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') | ||
52 | ? CONFIG.OBJECT_STORAGE.ENDPOINT | ||
53 | : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT | ||
54 | |||
55 | return endpoint | ||
56 | } | ||
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts new file mode 100644 index 000000000..11e10aa9f --- /dev/null +++ b/server/lib/object-storage/shared/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './client' | ||
2 | export * from './logger' | ||
3 | export * from './object-storage-helpers' | ||
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts new file mode 100644 index 000000000..8ab7cbd71 --- /dev/null +++ b/server/lib/object-storage/shared/logger.ts | |||
@@ -0,0 +1,7 @@ | |||
1 | import { loggerTagsFactory } from '@server/helpers/logger' | ||
2 | |||
3 | const lTags = loggerTagsFactory('object-storage') | ||
4 | |||
5 | export { | ||
6 | lTags | ||
7 | } | ||
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts new file mode 100644 index 000000000..e23216907 --- /dev/null +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -0,0 +1,229 @@ | |||
1 | import { close, createReadStream, createWriteStream, ensureDir, open, ReadStream, stat } from 'fs-extra' | ||
2 | import { min } from 'lodash' | ||
3 | import { dirname } from 'path' | ||
4 | import { Readable } from 'stream' | ||
5 | import { | ||
6 | CompletedPart, | ||
7 | CompleteMultipartUploadCommand, | ||
8 | CreateMultipartUploadCommand, | ||
9 | DeleteObjectCommand, | ||
10 | GetObjectCommand, | ||
11 | ListObjectsV2Command, | ||
12 | PutObjectCommand, | ||
13 | UploadPartCommand | ||
14 | } from '@aws-sdk/client-s3' | ||
15 | import { pipelinePromise } from '@server/helpers/core-utils' | ||
16 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
17 | import { logger } from '@server/helpers/logger' | ||
18 | import { CONFIG } from '@server/initializers/config' | ||
19 | import { getPrivateUrl } from '../urls' | ||
20 | import { getClient } from './client' | ||
21 | import { lTags } from './logger' | ||
22 | |||
23 | type BucketInfo = { | ||
24 | BUCKET_NAME: string | ||
25 | PREFIX?: string | ||
26 | } | ||
27 | |||
28 | async function storeObject (options: { | ||
29 | inputPath: string | ||
30 | objectStorageKey: string | ||
31 | bucketInfo: BucketInfo | ||
32 | }): Promise<string> { | ||
33 | const { inputPath, objectStorageKey, bucketInfo } = options | ||
34 | |||
35 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | ||
36 | |||
37 | const stats = await stat(inputPath) | ||
38 | |||
39 | // If bigger than max allowed size we do a multipart upload | ||
40 | if (stats.size > CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART) { | ||
41 | return multiPartUpload({ inputPath, objectStorageKey, bucketInfo }) | ||
42 | } | ||
43 | |||
44 | const fileStream = createReadStream(inputPath) | ||
45 | return objectStoragePut({ objectStorageKey, content: fileStream, bucketInfo }) | ||
46 | } | ||
47 | |||
48 | async function removeObject (filename: string, bucketInfo: BucketInfo) { | ||
49 | const command = new DeleteObjectCommand({ | ||
50 | Bucket: bucketInfo.BUCKET_NAME, | ||
51 | Key: buildKey(filename, bucketInfo) | ||
52 | }) | ||
53 | |||
54 | return getClient().send(command) | ||
55 | } | ||
56 | |||
57 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | ||
58 | const s3Client = getClient() | ||
59 | |||
60 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
61 | const listCommand = new ListObjectsV2Command({ | ||
62 | Bucket: bucketInfo.BUCKET_NAME, | ||
63 | Prefix: commandPrefix | ||
64 | }) | ||
65 | |||
66 | const listedObjects = await s3Client.send(listCommand) | ||
67 | |||
68 | // FIXME: use bulk delete when s3ninja will support this operation | ||
69 | // const deleteParams = { | ||
70 | // Bucket: bucketInfo.BUCKET_NAME, | ||
71 | // Delete: { Objects: [] } | ||
72 | // } | ||
73 | |||
74 | if (isArray(listedObjects.Contents) !== true) { | ||
75 | const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
76 | |||
77 | logger.error(message, { response: listedObjects, ...lTags() }) | ||
78 | throw new Error(message) | ||
79 | } | ||
80 | |||
81 | for (const object of listedObjects.Contents) { | ||
82 | const command = new DeleteObjectCommand({ | ||
83 | Bucket: bucketInfo.BUCKET_NAME, | ||
84 | Key: object.Key | ||
85 | }) | ||
86 | |||
87 | await s3Client.send(command) | ||
88 | |||
89 | // FIXME: use bulk delete when s3ninja will support this operation | ||
90 | // deleteParams.Delete.Objects.push({ Key: object.Key }) | ||
91 | } | ||
92 | |||
93 | // FIXME: use bulk delete when s3ninja will support this operation | ||
94 | // const deleteCommand = new DeleteObjectsCommand(deleteParams) | ||
95 | // await s3Client.send(deleteCommand) | ||
96 | |||
97 | // Repeat if not all objects could be listed at once (limit of 1000?) | ||
98 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) | ||
99 | } | ||
100 | |||
101 | async function makeAvailable (options: { | ||
102 | key: string | ||
103 | destination: string | ||
104 | bucketInfo: BucketInfo | ||
105 | }) { | ||
106 | const { key, destination, bucketInfo } = options | ||
107 | |||
108 | await ensureDir(dirname(options.destination)) | ||
109 | |||
110 | const command = new GetObjectCommand({ | ||
111 | Bucket: bucketInfo.BUCKET_NAME, | ||
112 | Key: buildKey(key, bucketInfo) | ||
113 | }) | ||
114 | const response = await getClient().send(command) | ||
115 | |||
116 | const file = createWriteStream(destination) | ||
117 | await pipelinePromise(response.Body as Readable, file) | ||
118 | |||
119 | file.close() | ||
120 | } | ||
121 | |||
122 | function buildKey (key: string, bucketInfo: BucketInfo) { | ||
123 | return bucketInfo.PREFIX + key | ||
124 | } | ||
125 | |||
126 | // --------------------------------------------------------------------------- | ||
127 | |||
128 | export { | ||
129 | BucketInfo, | ||
130 | buildKey, | ||
131 | storeObject, | ||
132 | removeObject, | ||
133 | removePrefix, | ||
134 | makeAvailable | ||
135 | } | ||
136 | |||
137 | // --------------------------------------------------------------------------- | ||
138 | |||
139 | async function objectStoragePut (options: { | ||
140 | objectStorageKey: string | ||
141 | content: ReadStream | ||
142 | bucketInfo: BucketInfo | ||
143 | }) { | ||
144 | const { objectStorageKey, content, bucketInfo } = options | ||
145 | |||
146 | const command = new PutObjectCommand({ | ||
147 | Bucket: bucketInfo.BUCKET_NAME, | ||
148 | Key: buildKey(objectStorageKey, bucketInfo), | ||
149 | Body: content | ||
150 | }) | ||
151 | |||
152 | await getClient().send(command) | ||
153 | |||
154 | return getPrivateUrl(bucketInfo, objectStorageKey) | ||
155 | } | ||
156 | |||
157 | async function multiPartUpload (options: { | ||
158 | inputPath: string | ||
159 | objectStorageKey: string | ||
160 | bucketInfo: BucketInfo | ||
161 | }) { | ||
162 | const { objectStorageKey, inputPath, bucketInfo } = options | ||
163 | |||
164 | const key = buildKey(objectStorageKey, bucketInfo) | ||
165 | const s3Client = getClient() | ||
166 | |||
167 | const statResult = await stat(inputPath) | ||
168 | |||
169 | const createMultipartCommand = new CreateMultipartUploadCommand({ | ||
170 | Bucket: bucketInfo.BUCKET_NAME, | ||
171 | Key: key | ||
172 | }) | ||
173 | const createResponse = await s3Client.send(createMultipartCommand) | ||
174 | |||
175 | const fd = await open(inputPath, 'r') | ||
176 | let partNumber = 1 | ||
177 | const parts: CompletedPart[] = [] | ||
178 | const partSize = CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART | ||
179 | |||
180 | for (let start = 0; start < statResult.size; start += partSize) { | ||
181 | logger.debug( | ||
182 | 'Uploading part %d of file to %s%s in bucket %s', | ||
183 | partNumber, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() | ||
184 | ) | ||
185 | |||
186 | // FIXME: Remove when https://github.com/aws/aws-sdk-js-v3/pull/2637 is released | ||
187 | // The s3 sdk needs to know the length of the http body beforehand, but doesn't support | ||
188 | // streams with start and end set, so it just tries to stat the file in stream.path. | ||
189 | // This fails for us because we only want to send part of the file. The stream type | ||
190 | // is modified so we can set the byteLength here, which s3 detects because array buffers | ||
191 | // have this field set | ||
192 | const stream: ReadStream & { byteLength: number } = | ||
193 | createReadStream( | ||
194 | inputPath, | ||
195 | { fd, autoClose: false, start, end: (start + partSize) - 1 } | ||
196 | ) as ReadStream & { byteLength: number } | ||
197 | |||
198 | // Calculate if the part size is more than what's left over, and in that case use left over bytes for byteLength | ||
199 | stream.byteLength = min([ statResult.size - start, partSize ]) | ||
200 | |||
201 | const uploadPartCommand = new UploadPartCommand({ | ||
202 | Bucket: bucketInfo.BUCKET_NAME, | ||
203 | Key: key, | ||
204 | UploadId: createResponse.UploadId, | ||
205 | PartNumber: partNumber, | ||
206 | Body: stream | ||
207 | }) | ||
208 | const uploadResponse = await s3Client.send(uploadPartCommand) | ||
209 | |||
210 | parts.push({ ETag: uploadResponse.ETag, PartNumber: partNumber }) | ||
211 | partNumber += 1 | ||
212 | } | ||
213 | await close(fd) | ||
214 | |||
215 | const completeUploadCommand = new CompleteMultipartUploadCommand({ | ||
216 | Bucket: bucketInfo.BUCKET_NAME, | ||
217 | Key: objectStorageKey, | ||
218 | UploadId: createResponse.UploadId, | ||
219 | MultipartUpload: { Parts: parts } | ||
220 | }) | ||
221 | await s3Client.send(completeUploadCommand) | ||
222 | |||
223 | logger.debug( | ||
224 | 'Completed %s%s in bucket %s in %d parts', | ||
225 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, partNumber - 1, lTags() | ||
226 | ) | ||
227 | |||
228 | return getPrivateUrl(bucketInfo, objectStorageKey) | ||
229 | } | ||
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts new file mode 100644 index 000000000..2a889190b --- /dev/null +++ b/server/lib/object-storage/urls.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' | ||
3 | |||
4 | function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { | ||
5 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) | ||
6 | } | ||
7 | |||
8 | function getWebTorrentPublicFileUrl (fileUrl: string) { | ||
9 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL | ||
10 | if (!baseUrl) return fileUrl | ||
11 | |||
12 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
13 | } | ||
14 | |||
15 | function getHLSPublicFileUrl (fileUrl: string) { | ||
16 | const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL | ||
17 | if (!baseUrl) return fileUrl | ||
18 | |||
19 | return replaceByBaseUrl(fileUrl, baseUrl) | ||
20 | } | ||
21 | |||
22 | export { | ||
23 | getPrivateUrl, | ||
24 | getWebTorrentPublicFileUrl, | ||
25 | replaceByBaseUrl, | ||
26 | getHLSPublicFileUrl | ||
27 | } | ||
28 | |||
29 | // --------------------------------------------------------------------------- | ||
30 | |||
31 | function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { | ||
32 | if (baseUrl) return baseUrl | ||
33 | |||
34 | return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` | ||
35 | } | ||
36 | |||
37 | const regex = new RegExp('https?://[^/]+') | ||
38 | function replaceByBaseUrl (fileUrl: string, baseUrl: string) { | ||
39 | return fileUrl.replace(regex, baseUrl) | ||
40 | } | ||
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts new file mode 100644 index 000000000..15b8f58d5 --- /dev/null +++ b/server/lib/object-storage/videos.ts | |||
@@ -0,0 +1,72 @@ | |||
1 | import { join } from 'path' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models' | ||
5 | import { getHLSDirectory } from '../paths' | ||
6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | ||
7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | ||
8 | |||
9 | function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { | ||
10 | const baseHlsDirectory = getHLSDirectory(video) | ||
11 | |||
12 | return storeObject({ | ||
13 | inputPath: join(baseHlsDirectory, filename), | ||
14 | objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename), | ||
15 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
16 | }) | ||
17 | } | ||
18 | |||
19 | function storeWebTorrentFile (filename: string) { | ||
20 | return storeObject({ | ||
21 | inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename), | ||
22 | objectStorageKey: generateWebTorrentObjectStorageKey(filename), | ||
23 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) { | ||
28 | return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | ||
29 | } | ||
30 | |||
31 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { | ||
32 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) | ||
33 | } | ||
34 | |||
35 | async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) { | ||
36 | const key = generateHLSObjectStorageKey(playlist, video, filename) | ||
37 | |||
38 | logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) | ||
39 | |||
40 | await makeAvailable({ | ||
41 | key, | ||
42 | destination, | ||
43 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | ||
44 | }) | ||
45 | |||
46 | return destination | ||
47 | } | ||
48 | |||
49 | async function makeWebTorrentFileAvailable (filename: string, destination: string) { | ||
50 | const key = generateWebTorrentObjectStorageKey(filename) | ||
51 | |||
52 | logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags()) | ||
53 | |||
54 | await makeAvailable({ | ||
55 | key, | ||
56 | destination, | ||
57 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | ||
58 | }) | ||
59 | |||
60 | return destination | ||
61 | } | ||
62 | |||
63 | export { | ||
64 | storeWebTorrentFile, | ||
65 | storeHLSFile, | ||
66 | |||
67 | removeHLSObjectStorage, | ||
68 | removeWebTorrentObjectStorage, | ||
69 | |||
70 | makeWebTorrentFileAvailable, | ||
71 | makeHLSFileAvailable | ||
72 | } | ||
diff --git a/server/lib/video-paths.ts b/server/lib/paths.ts index 1e4382108..434e637c6 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/paths.ts | |||
@@ -1,9 +1,8 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { extractVideo } from '@server/helpers/video' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
5 | import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
6 | import { buildUUID } from '@server/helpers/uuid' | 2 | import { buildUUID } from '@server/helpers/uuid' |
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' | ||
5 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
7 | import { removeFragmentedMP4Ext } from '@shared/core-utils' | 6 | import { removeFragmentedMP4Ext } from '@shared/core-utils' |
8 | 7 | ||
9 | // ################## Video file name ################## | 8 | // ################## Video file name ################## |
@@ -16,39 +15,18 @@ function generateHLSVideoFilename (resolution: number) { | |||
16 | return `${buildUUID()}-${resolution}-fragmented.mp4` | 15 | return `${buildUUID()}-${resolution}-fragmented.mp4` |
17 | } | 16 | } |
18 | 17 | ||
19 | function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { | 18 | // ################## Streaming playlist ################## |
20 | if (videoFile.isHLS()) { | ||
21 | const video = extractVideo(videoOrPlaylist) | ||
22 | |||
23 | return join(getHLSDirectory(video), videoFile.filename) | ||
24 | } | ||
25 | |||
26 | const baseDir = isRedundancy | ||
27 | ? CONFIG.STORAGE.REDUNDANCY_DIR | ||
28 | : CONFIG.STORAGE.VIDEOS_DIR | ||
29 | |||
30 | return join(baseDir, videoFile.filename) | ||
31 | } | ||
32 | |||
33 | // ################## Redundancy ################## | ||
34 | 19 | ||
35 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | 20 | function getLiveDirectory (video: MVideoUUID) { |
36 | // Base URL used by our HLS player | 21 | return getHLSDirectory(video) |
37 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
38 | } | 22 | } |
39 | 23 | ||
40 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | 24 | function getHLSDirectory (video: MVideoUUID) { |
41 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | 25 | return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) |
42 | } | 26 | } |
43 | 27 | ||
44 | // ################## Streaming playlist ################## | 28 | function getHLSRedundancyDirectory (video: MVideoUUID) { |
45 | 29 | return join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | |
46 | function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { | ||
47 | const baseDir = isRedundancy | ||
48 | ? HLS_REDUNDANCY_DIRECTORY | ||
49 | : HLS_STREAMING_PLAYLIST_DIRECTORY | ||
50 | |||
51 | return join(baseDir, video.uuid) | ||
52 | } | 30 | } |
53 | 31 | ||
54 | function getHlsResolutionPlaylistFilename (videoFilename: string) { | 32 | function getHlsResolutionPlaylistFilename (videoFilename: string) { |
@@ -81,36 +59,24 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi | |||
81 | return uuid + '-' + resolution + extension | 59 | return uuid + '-' + resolution + extension |
82 | } | 60 | } |
83 | 61 | ||
84 | function getTorrentFilePath (videoFile: MVideoFile) { | 62 | function getFSTorrentFilePath (videoFile: MVideoFile) { |
85 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) | 63 | return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) |
86 | } | 64 | } |
87 | 65 | ||
88 | // ################## Meta data ################## | ||
89 | |||
90 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
91 | const path = '/api/v1/videos/' | ||
92 | |||
93 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
94 | } | ||
95 | |||
96 | // --------------------------------------------------------------------------- | 66 | // --------------------------------------------------------------------------- |
97 | 67 | ||
98 | export { | 68 | export { |
99 | generateHLSVideoFilename, | 69 | generateHLSVideoFilename, |
100 | generateWebTorrentVideoFilename, | 70 | generateWebTorrentVideoFilename, |
101 | 71 | ||
102 | getVideoFilePath, | ||
103 | |||
104 | generateTorrentFileName, | 72 | generateTorrentFileName, |
105 | getTorrentFilePath, | 73 | getFSTorrentFilePath, |
106 | 74 | ||
107 | getHLSDirectory, | 75 | getHLSDirectory, |
76 | getLiveDirectory, | ||
77 | getHLSRedundancyDirectory, | ||
78 | |||
108 | generateHLSMasterPlaylistFilename, | 79 | generateHLSMasterPlaylistFilename, |
109 | generateHlsSha256SegmentsFilename, | 80 | generateHlsSha256SegmentsFilename, |
110 | getHlsResolutionPlaylistFilename, | 81 | getHlsResolutionPlaylistFilename |
111 | |||
112 | getLocalVideoFileMetadataUrl, | ||
113 | |||
114 | generateWebTorrentRedundancyUrl, | ||
115 | generateHLSRedundancyUrl | ||
116 | } | 82 | } |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 137ae53a0..ebfd015b5 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -24,7 +24,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli | |||
24 | import { getOrCreateAPVideo } from '../activitypub/videos' | 24 | import { getOrCreateAPVideo } from '../activitypub/videos' |
25 | import { downloadPlaylistSegments } from '../hls' | 25 | import { downloadPlaylistSegments } from '../hls' |
26 | import { removeVideoRedundancy } from '../redundancy' | 26 | import { removeVideoRedundancy } from '../redundancy' |
27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' | 27 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls' |
28 | import { AbstractScheduler } from './abstract-scheduler' | 28 | import { AbstractScheduler } from './abstract-scheduler' |
29 | 29 | ||
30 | type CandidateToDuplicate = { | 30 | type CandidateToDuplicate = { |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index c08523988..d2384f53c 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | |||
3 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' | 2 | import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' |
4 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' | 3 | import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' |
5 | import { generateImageFilename, processImage } from '../helpers/image-utils' | 4 | import { generateImageFilename, processImage } from '../helpers/image-utils' |
@@ -10,7 +9,7 @@ import { ThumbnailModel } from '../models/video/thumbnail' | |||
10 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' | 9 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' |
11 | import { MThumbnail } from '../types/models/video/thumbnail' | 10 | import { MThumbnail } from '../types/models/video/thumbnail' |
12 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | 11 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' |
13 | import { getVideoFilePath } from './video-paths' | 12 | import { VideoPathManager } from './video-path-manager' |
14 | 13 | ||
15 | type ImageSize = { height?: number, width?: number } | 14 | type ImageSize = { height?: number, width?: number } |
16 | 15 | ||
@@ -116,21 +115,22 @@ function generateVideoMiniature (options: { | |||
116 | }) { | 115 | }) { |
117 | const { video, videoFile, type } = options | 116 | const { video, videoFile, type } = options |
118 | 117 | ||
119 | const input = getVideoFilePath(video, videoFile) | 118 | return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => { |
119 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | ||
120 | 120 | ||
121 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | 121 | const thumbnailCreator = videoFile.isAudio() |
122 | const thumbnailCreator = videoFile.isAudio() | 122 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) |
123 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) | 123 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) |
124 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | ||
125 | 124 | ||
126 | return updateThumbnailFromFunction({ | 125 | return updateThumbnailFromFunction({ |
127 | thumbnailCreator, | 126 | thumbnailCreator, |
128 | filename, | 127 | filename, |
129 | height, | 128 | height, |
130 | width, | 129 | width, |
131 | type, | 130 | type, |
132 | automaticallyGenerated: true, | 131 | automaticallyGenerated: true, |
133 | existingThumbnail | 132 | existingThumbnail |
133 | }) | ||
134 | }) | 134 | }) |
135 | } | 135 | } |
136 | 136 | ||
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index d2a556360..ee228c011 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts | |||
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path' | |||
4 | import { toEven } from '@server/helpers/core-utils' | 4 | import { toEven } from '@server/helpers/core-utils' |
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 6 | import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
7 | import { VideoResolution } from '../../../shared/models/videos' | 7 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' |
8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' | 9 | import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' |
10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' | 10 | import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' |
11 | import { logger } from '../../helpers/logger' | 11 | import { logger } from '../../helpers/logger' |
12 | import { CONFIG } from '../../initializers/config' | 12 | import { CONFIG } from '../../initializers/config' |
13 | import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' | 13 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' |
14 | import { VideoFileModel } from '../../models/video/video-file' | 14 | import { VideoFileModel } from '../../models/video/video-file' |
15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | 15 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' |
16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' | 16 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' |
@@ -19,9 +19,9 @@ import { | |||
19 | generateHlsSha256SegmentsFilename, | 19 | generateHlsSha256SegmentsFilename, |
20 | generateHLSVideoFilename, | 20 | generateHLSVideoFilename, |
21 | generateWebTorrentVideoFilename, | 21 | generateWebTorrentVideoFilename, |
22 | getHlsResolutionPlaylistFilename, | 22 | getHlsResolutionPlaylistFilename |
23 | getVideoFilePath | 23 | } from '../paths' |
24 | } from '../video-paths' | 24 | import { VideoPathManager } from '../video-path-manager' |
25 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | 25 | import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' |
26 | 26 | ||
27 | /** | 27 | /** |
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' | |||
32 | */ | 32 | */ |
33 | 33 | ||
34 | // Optimize the original video file and replace it. The resolution is not changed. | 34 | // Optimize the original video file and replace it. The resolution is not changed. |
35 | async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { | 35 | function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { |
36 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 36 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
37 | const newExtname = '.mp4' | 37 | const newExtname = '.mp4' |
38 | 38 | ||
39 | const videoInputPath = getVideoFilePath(video, inputVideoFile) | 39 | return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => { |
40 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 40 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
41 | 41 | ||
42 | const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) | 42 | const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) |
43 | ? 'quick-transcode' | 43 | ? 'quick-transcode' |
44 | : 'video' | 44 | : 'video' |
45 | 45 | ||
46 | const resolution = toEven(inputVideoFile.resolution) | 46 | const resolution = toEven(inputVideoFile.resolution) |
47 | 47 | ||
48 | const transcodeOptions: TranscodeOptions = { | 48 | const transcodeOptions: TranscodeOptions = { |
49 | type: transcodeType, | 49 | type: transcodeType, |
50 | 50 | ||
51 | inputPath: videoInputPath, | 51 | inputPath: videoInputPath, |
52 | outputPath: videoTranscodedPath, | 52 | outputPath: videoTranscodedPath, |
53 | 53 | ||
54 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 54 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
55 | profile: CONFIG.TRANSCODING.PROFILE, | 55 | profile: CONFIG.TRANSCODING.PROFILE, |
56 | 56 | ||
57 | resolution, | 57 | resolution, |
58 | 58 | ||
59 | job | 59 | job |
60 | } | 60 | } |
61 | 61 | ||
62 | // Could be very long! | 62 | // Could be very long! |
63 | await transcode(transcodeOptions) | 63 | await transcode(transcodeOptions) |
64 | 64 | ||
65 | try { | 65 | try { |
66 | await remove(videoInputPath) | 66 | await remove(videoInputPath) |
67 | 67 | ||
68 | // Important to do this before getVideoFilename() to take in account the new filename | 68 | // Important to do this before getVideoFilename() to take in account the new filename |
69 | inputVideoFile.extname = newExtname | 69 | inputVideoFile.extname = newExtname |
70 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | 70 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) |
71 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
71 | 72 | ||
72 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 73 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) |
73 | 74 | ||
74 | await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 75 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) |
75 | 76 | ||
76 | return transcodeType | 77 | return { transcodeType, videoFile } |
77 | } catch (err) { | 78 | } catch (err) { |
78 | // Auto destruction... | 79 | // Auto destruction... |
79 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) | 80 | video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) |
80 | 81 | ||
81 | throw err | 82 | throw err |
82 | } | 83 | } |
84 | }) | ||
83 | } | 85 | } |
84 | 86 | ||
85 | // Transcode the original video file to a lower resolution. | 87 | // Transcode the original video file to a lower resolution |
86 | async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { | 88 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed |
89 | function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { | ||
87 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 90 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
88 | const extname = '.mp4' | 91 | const extname = '.mp4' |
89 | 92 | ||
90 | // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed | 93 | return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => { |
91 | const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) | 94 | const newVideoFile = new VideoFileModel({ |
95 | resolution, | ||
96 | extname, | ||
97 | filename: generateWebTorrentVideoFilename(resolution, extname), | ||
98 | size: 0, | ||
99 | videoId: video.id | ||
100 | }) | ||
92 | 101 | ||
93 | const newVideoFile = new VideoFileModel({ | 102 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) |
94 | resolution, | 103 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) |
95 | extname, | ||
96 | filename: generateWebTorrentVideoFilename(resolution, extname), | ||
97 | size: 0, | ||
98 | videoId: video.id | ||
99 | }) | ||
100 | 104 | ||
101 | const videoOutputPath = getVideoFilePath(video, newVideoFile) | 105 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO |
102 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) | 106 | ? { |
107 | type: 'only-audio' as 'only-audio', | ||
103 | 108 | ||
104 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO | 109 | inputPath: videoInputPath, |
105 | ? { | 110 | outputPath: videoTranscodedPath, |
106 | type: 'only-audio' as 'only-audio', | ||
107 | 111 | ||
108 | inputPath: videoInputPath, | 112 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
109 | outputPath: videoTranscodedPath, | 113 | profile: CONFIG.TRANSCODING.PROFILE, |
110 | 114 | ||
111 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 115 | resolution, |
112 | profile: CONFIG.TRANSCODING.PROFILE, | ||
113 | 116 | ||
114 | resolution, | 117 | job |
118 | } | ||
119 | : { | ||
120 | type: 'video' as 'video', | ||
121 | inputPath: videoInputPath, | ||
122 | outputPath: videoTranscodedPath, | ||
115 | 123 | ||
116 | job | 124 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
117 | } | 125 | profile: CONFIG.TRANSCODING.PROFILE, |
118 | : { | ||
119 | type: 'video' as 'video', | ||
120 | inputPath: videoInputPath, | ||
121 | outputPath: videoTranscodedPath, | ||
122 | 126 | ||
123 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 127 | resolution, |
124 | profile: CONFIG.TRANSCODING.PROFILE, | 128 | isPortraitMode: isPortrait, |
125 | 129 | ||
126 | resolution, | 130 | job |
127 | isPortraitMode: isPortrait, | 131 | } |
128 | 132 | ||
129 | job | 133 | await transcode(transcodeOptions) |
130 | } | ||
131 | |||
132 | await transcode(transcodeOptions) | ||
133 | 134 | ||
134 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) | 135 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) |
136 | }) | ||
135 | } | 137 | } |
136 | 138 | ||
137 | // Merge an image with an audio file to create a video | 139 | // Merge an image with an audio file to create a video |
138 | async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { | 140 | function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { |
139 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | 141 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR |
140 | const newExtname = '.mp4' | 142 | const newExtname = '.mp4' |
141 | 143 | ||
142 | const inputVideoFile = video.getMinQualityFile() | 144 | const inputVideoFile = video.getMinQualityFile() |
143 | 145 | ||
144 | const audioInputPath = getVideoFilePath(video, inputVideoFile) | 146 | return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => { |
145 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | 147 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) |
146 | 148 | ||
147 | // If the user updates the video preview during transcoding | 149 | // If the user updates the video preview during transcoding |
148 | const previewPath = video.getPreview().getPath() | 150 | const previewPath = video.getPreview().getPath() |
149 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | 151 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) |
150 | await copyFile(previewPath, tmpPreviewPath) | 152 | await copyFile(previewPath, tmpPreviewPath) |
151 | 153 | ||
152 | const transcodeOptions = { | 154 | const transcodeOptions = { |
153 | type: 'merge-audio' as 'merge-audio', | 155 | type: 'merge-audio' as 'merge-audio', |
154 | 156 | ||
155 | inputPath: tmpPreviewPath, | 157 | inputPath: tmpPreviewPath, |
156 | outputPath: videoTranscodedPath, | 158 | outputPath: videoTranscodedPath, |
157 | 159 | ||
158 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 160 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), |
159 | profile: CONFIG.TRANSCODING.PROFILE, | 161 | profile: CONFIG.TRANSCODING.PROFILE, |
160 | 162 | ||
161 | audioPath: audioInputPath, | 163 | audioPath: audioInputPath, |
162 | resolution, | 164 | resolution, |
163 | 165 | ||
164 | job | 166 | job |
165 | } | 167 | } |
166 | 168 | ||
167 | try { | 169 | try { |
168 | await transcode(transcodeOptions) | 170 | await transcode(transcodeOptions) |
169 | 171 | ||
170 | await remove(audioInputPath) | 172 | await remove(audioInputPath) |
171 | await remove(tmpPreviewPath) | 173 | await remove(tmpPreviewPath) |
172 | } catch (err) { | 174 | } catch (err) { |
173 | await remove(tmpPreviewPath) | 175 | await remove(tmpPreviewPath) |
174 | throw err | 176 | throw err |
175 | } | 177 | } |
176 | 178 | ||
177 | // Important to do this before getVideoFilename() to take in account the new file extension | 179 | // Important to do this before getVideoFilename() to take in account the new file extension |
178 | inputVideoFile.extname = newExtname | 180 | inputVideoFile.extname = newExtname |
179 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | 181 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) |
180 | 182 | ||
181 | const videoOutputPath = getVideoFilePath(video, inputVideoFile) | 183 | const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile) |
182 | // ffmpeg generated a new video file, so update the video duration | 184 | // ffmpeg generated a new video file, so update the video duration |
183 | // See https://trac.ffmpeg.org/ticket/5456 | 185 | // See https://trac.ffmpeg.org/ticket/5456 |
184 | video.duration = await getDurationFromVideoFile(videoTranscodedPath) | 186 | video.duration = await getDurationFromVideoFile(videoTranscodedPath) |
185 | await video.save() | 187 | await video.save() |
186 | 188 | ||
187 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) | 189 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) |
190 | }) | ||
188 | } | 191 | } |
189 | 192 | ||
190 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | 193 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist |
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding ( | |||
258 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | 261 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) |
259 | video.VideoFiles = await video.$get('VideoFiles') | 262 | video.VideoFiles = await video.$get('VideoFiles') |
260 | 263 | ||
261 | return video | 264 | return { video, videoFile } |
262 | } | 265 | } |
263 | 266 | ||
264 | async function generateHlsPlaylistCommon (options: { | 267 | async function generateHlsPlaylistCommon (options: { |
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: { | |||
335 | videoStreamingPlaylistId: playlist.id | 338 | videoStreamingPlaylistId: playlist.id |
336 | }) | 339 | }) |
337 | 340 | ||
338 | const videoFilePath = getVideoFilePath(playlist, newVideoFile) | 341 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) |
339 | 342 | ||
340 | // Move files from tmp transcoded directory to the appropriate place | 343 | // Move files from tmp transcoded directory to the appropriate place |
341 | const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) | 344 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) |
342 | await ensureDir(baseHlsDirectory) | ||
343 | 345 | ||
344 | // Move playlist file | 346 | // Move playlist file |
345 | const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) | 347 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) |
346 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | 348 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) |
347 | // Move video file | 349 | // Move video file |
348 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | 350 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) |
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: { | |||
355 | 357 | ||
356 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | 358 | await createTorrentAndSetInfoHash(playlist, newVideoFile) |
357 | 359 | ||
358 | await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | 360 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) |
359 | 361 | ||
360 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo | 362 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo |
361 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') | 363 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') |
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: { | |||
368 | await updateMasterHLSPlaylist(video, playlistWithFiles) | 370 | await updateMasterHLSPlaylist(video, playlistWithFiles) |
369 | await updateSha256VODSegments(video, playlistWithFiles) | 371 | await updateSha256VODSegments(video, playlistWithFiles) |
370 | 372 | ||
371 | return resolutionPlaylistPath | 373 | return { resolutionPlaylistPath, videoFile: savedVideoFile } |
372 | } | 374 | } |
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts new file mode 100644 index 000000000..4c5d0c89d --- /dev/null +++ b/server/lib/video-path-manager.ts | |||
@@ -0,0 +1,139 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { extname, join } from 'path' | ||
3 | import { buildUUID } from '@server/helpers/uuid' | ||
4 | import { extractVideo } from '@server/helpers/video' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
7 | import { VideoStorage } from '@shared/models' | ||
8 | import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' | ||
9 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' | ||
10 | |||
11 | type MakeAvailableCB <T> = (path: string) => Promise<T> | T | ||
12 | |||
13 | class VideoPathManager { | ||
14 | |||
15 | private static instance: VideoPathManager | ||
16 | |||
17 | private constructor () {} | ||
18 | |||
19 | getFSHLSOutputPath (video: MVideoUUID, filename?: string) { | ||
20 | const base = getHLSDirectory(video) | ||
21 | if (!filename) return base | ||
22 | |||
23 | return join(base, filename) | ||
24 | } | ||
25 | |||
26 | getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
27 | if (videoFile.isHLS()) { | ||
28 | const video = extractVideo(videoOrPlaylist) | ||
29 | |||
30 | return join(getHLSRedundancyDirectory(video), videoFile.filename) | ||
31 | } | ||
32 | |||
33 | return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) | ||
34 | } | ||
35 | |||
36 | getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
37 | if (videoFile.isHLS()) { | ||
38 | const video = extractVideo(videoOrPlaylist) | ||
39 | |||
40 | return join(getHLSDirectory(video), videoFile.filename) | ||
41 | } | ||
42 | |||
43 | return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) | ||
44 | } | ||
45 | |||
46 | async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { | ||
47 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
48 | return this.makeAvailableFactory( | ||
49 | () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile), | ||
50 | false, | ||
51 | cb | ||
52 | ) | ||
53 | } | ||
54 | |||
55 | const destination = this.buildTMPDestination(videoFile.filename) | ||
56 | |||
57 | if (videoFile.isHLS()) { | ||
58 | const video = extractVideo(videoOrPlaylist) | ||
59 | |||
60 | return this.makeAvailableFactory( | ||
61 | () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination), | ||
62 | true, | ||
63 | cb | ||
64 | ) | ||
65 | } | ||
66 | |||
67 | return this.makeAvailableFactory( | ||
68 | () => makeWebTorrentFileAvailable(videoFile.filename, destination), | ||
69 | true, | ||
70 | cb | ||
71 | ) | ||
72 | } | ||
73 | |||
74 | async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) { | ||
75 | const filename = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
76 | |||
77 | if (videoFile.storage === VideoStorage.FILE_SYSTEM) { | ||
78 | return this.makeAvailableFactory( | ||
79 | () => join(getHLSDirectory(playlist.Video), filename), | ||
80 | false, | ||
81 | cb | ||
82 | ) | ||
83 | } | ||
84 | |||
85 | return this.makeAvailableFactory( | ||
86 | () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), | ||
87 | true, | ||
88 | cb | ||
89 | ) | ||
90 | } | ||
91 | |||
92 | async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) { | ||
93 | if (playlist.storage === VideoStorage.FILE_SYSTEM) { | ||
94 | return this.makeAvailableFactory( | ||
95 | () => join(getHLSDirectory(playlist.Video), filename), | ||
96 | false, | ||
97 | cb | ||
98 | ) | ||
99 | } | ||
100 | |||
101 | return this.makeAvailableFactory( | ||
102 | () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), | ||
103 | true, | ||
104 | cb | ||
105 | ) | ||
106 | } | ||
107 | |||
108 | private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) { | ||
109 | let result: T | ||
110 | |||
111 | const destination = await method() | ||
112 | |||
113 | try { | ||
114 | result = await cb(destination) | ||
115 | } catch (err) { | ||
116 | if (destination && clean) await remove(destination) | ||
117 | throw err | ||
118 | } | ||
119 | |||
120 | if (clean) await remove(destination) | ||
121 | |||
122 | return result | ||
123 | } | ||
124 | |||
125 | private buildTMPDestination (filename: string) { | ||
126 | return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) | ||
127 | |||
128 | } | ||
129 | |||
130 | static get Instance () { | ||
131 | return this.instance || (this.instance = new this()) | ||
132 | } | ||
133 | } | ||
134 | |||
135 | // --------------------------------------------------------------------------- | ||
136 | |||
137 | export { | ||
138 | VideoPathManager | ||
139 | } | ||
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts new file mode 100644 index 000000000..0613d94bf --- /dev/null +++ b/server/lib/video-state.ts | |||
@@ -0,0 +1,99 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { VideoModel } from '@server/models/video/video' | ||
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
7 | import { MVideoFullLight, MVideoUUID } from '@server/types/models' | ||
8 | import { VideoState } from '@shared/models' | ||
9 | import { federateVideoIfNeeded } from './activitypub/videos' | ||
10 | import { Notifier } from './notifier' | ||
11 | import { addMoveToObjectStorageJob } from './video' | ||
12 | |||
13 | function buildNextVideoState (currentState?: VideoState) { | ||
14 | if (currentState === VideoState.PUBLISHED) { | ||
15 | throw new Error('Video is already in its final state') | ||
16 | } | ||
17 | |||
18 | if ( | ||
19 | currentState !== VideoState.TO_TRANSCODE && | ||
20 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
21 | CONFIG.TRANSCODING.ENABLED | ||
22 | ) { | ||
23 | return VideoState.TO_TRANSCODE | ||
24 | } | ||
25 | |||
26 | if ( | ||
27 | currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && | ||
28 | CONFIG.OBJECT_STORAGE.ENABLED | ||
29 | ) { | ||
30 | return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | ||
31 | } | ||
32 | |||
33 | return VideoState.PUBLISHED | ||
34 | } | ||
35 | |||
36 | function moveToNextState (video: MVideoUUID, isNewVideo = true) { | ||
37 | return sequelizeTypescript.transaction(async t => { | ||
38 | // Maybe the video changed in database, refresh it | ||
39 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
40 | // Video does not exist anymore | ||
41 | if (!videoDatabase) return undefined | ||
42 | |||
43 | // Already in its final state | ||
44 | if (videoDatabase.state === VideoState.PUBLISHED) { | ||
45 | return federateVideoIfNeeded(videoDatabase, false, t) | ||
46 | } | ||
47 | |||
48 | const newState = buildNextVideoState(videoDatabase.state) | ||
49 | |||
50 | if (newState === VideoState.PUBLISHED) { | ||
51 | return moveToPublishedState(videoDatabase, isNewVideo, t) | ||
52 | } | ||
53 | |||
54 | if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | ||
55 | return moveToExternalStorageState(videoDatabase, isNewVideo, t) | ||
56 | } | ||
57 | }) | ||
58 | } | ||
59 | |||
60 | // --------------------------------------------------------------------------- | ||
61 | |||
62 | export { | ||
63 | buildNextVideoState, | ||
64 | moveToNextState | ||
65 | } | ||
66 | |||
67 | // --------------------------------------------------------------------------- | ||
68 | |||
69 | async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
70 | logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
71 | |||
72 | const previousState = video.state | ||
73 | await video.setNewState(VideoState.PUBLISHED, transaction) | ||
74 | |||
75 | // If the video was not published, we consider it is a new one for other instances | ||
76 | // Live videos are always federated, so it's not a new video | ||
77 | await federateVideoIfNeeded(video, isNewVideo, transaction) | ||
78 | |||
79 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | ||
80 | |||
81 | if (previousState === VideoState.TO_TRANSCODE) { | ||
82 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) | ||
83 | } | ||
84 | } | ||
85 | |||
86 | async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) { | ||
87 | const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) | ||
88 | const pendingTranscode = videoJobInfo?.pendingTranscode || 0 | ||
89 | |||
90 | // We want to wait all transcoding jobs before moving the video on an external storage | ||
91 | if (pendingTranscode !== 0) return | ||
92 | |||
93 | await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction) | ||
94 | |||
95 | logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) | ||
96 | |||
97 | addMoveToObjectStorageJob(video, isNewVideo) | ||
98 | .catch(err => logger.error('Cannot add move to object storage job', { err })) | ||
99 | } | ||
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts new file mode 100644 index 000000000..64c2c9bf9 --- /dev/null +++ b/server/lib/video-urls.ts | |||
@@ -0,0 +1,31 @@ | |||
1 | |||
2 | import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
3 | import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' | ||
4 | |||
5 | // ################## Redundancy ################## | ||
6 | |||
7 | function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { | ||
8 | // Base URL used by our HLS player | ||
9 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | ||
10 | } | ||
11 | |||
12 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | ||
13 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | ||
14 | } | ||
15 | |||
16 | // ################## Meta data ################## | ||
17 | |||
18 | function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { | ||
19 | const path = '/api/v1/videos/' | ||
20 | |||
21 | return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id | ||
22 | } | ||
23 | |||
24 | // --------------------------------------------------------------------------- | ||
25 | |||
26 | export { | ||
27 | getLocalVideoFileMetadataUrl, | ||
28 | |||
29 | generateWebTorrentRedundancyUrl, | ||
30 | generateHLSRedundancyUrl | ||
31 | } | ||
diff --git a/server/lib/video.ts b/server/lib/video.ts index 61fee4949..0a2b93cc0 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -1,15 +1,13 @@ | |||
1 | import { UploadFiles } from 'express' | 1 | import { UploadFiles } from 'express' |
2 | import { Transaction } from 'sequelize/types' | 2 | import { Transaction } from 'sequelize/types' |
3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' | 3 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' |
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { TagModel } from '@server/models/video/tag' | 4 | import { TagModel } from '@server/models/video/tag' |
6 | import { VideoModel } from '@server/models/video/video' | 5 | import { VideoModel } from '@server/models/video/video' |
6 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
7 | import { FilteredModelAttributes } from '@server/types' | 7 | import { FilteredModelAttributes } from '@server/types' |
8 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | 8 | import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' |
9 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' | 9 | import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' |
10 | import { federateVideoIfNeeded } from './activitypub/videos' | 10 | import { CreateJobOptions, JobQueue } from './job-queue/job-queue' |
11 | import { JobQueue } from './job-queue/job-queue' | ||
12 | import { Notifier } from './notifier' | ||
13 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 11 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
14 | 12 | ||
15 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 13 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
@@ -82,29 +80,6 @@ async function setVideoTags (options: { | |||
82 | video.Tags = tagInstances | 80 | video.Tags = tagInstances |
83 | } | 81 | } |
84 | 82 | ||
85 | async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) { | ||
86 | const result = await sequelizeTypescript.transaction(async t => { | ||
87 | // Maybe the video changed in database, refresh it | ||
88 | const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
89 | // Video does not exist anymore | ||
90 | if (!videoDatabase) return undefined | ||
91 | |||
92 | // We transcoded the video file in another format, now we can publish it | ||
93 | const videoPublished = await videoDatabase.publishIfNeededAndSave(t) | ||
94 | |||
95 | // If the video was not published, we consider it is a new one for other instances | ||
96 | // Live videos are always federated, so it's not a new video | ||
97 | await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t) | ||
98 | |||
99 | return { videoDatabase, videoPublished } | ||
100 | }) | ||
101 | |||
102 | if (result?.videoPublished) { | ||
103 | Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase) | ||
104 | Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase) | ||
105 | } | ||
106 | } | ||
107 | |||
108 | async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { | 83 | async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { |
109 | let dataInput: VideoTranscodingPayload | 84 | let dataInput: VideoTranscodingPayload |
110 | 85 | ||
@@ -127,7 +102,20 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF | |||
127 | priority: await getTranscodingJobPriority(user) | 102 | priority: await getTranscodingJobPriority(user) |
128 | } | 103 | } |
129 | 104 | ||
130 | return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions) | 105 | return addTranscodingJob(dataInput, jobOptions) |
106 | } | ||
107 | |||
108 | async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) { | ||
109 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
110 | |||
111 | return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options) | ||
112 | } | ||
113 | |||
114 | async function addMoveToObjectStorageJob (video: MVideoUUID, isNewVideo = true) { | ||
115 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') | ||
116 | |||
117 | const dataInput = { videoUUID: video.uuid, isNewVideo } | ||
118 | return JobQueue.Instance.createJobWithPromise({ type: 'move-to-object-storage', payload: dataInput }) | ||
131 | } | 119 | } |
132 | 120 | ||
133 | async function getTranscodingJobPriority (user: MUserId) { | 121 | async function getTranscodingJobPriority (user: MUserId) { |
@@ -143,9 +131,10 @@ async function getTranscodingJobPriority (user: MUserId) { | |||
143 | 131 | ||
144 | export { | 132 | export { |
145 | buildLocalVideoFromReq, | 133 | buildLocalVideoFromReq, |
146 | publishAndFederateIfNeeded, | ||
147 | buildVideoThumbnailsFromReq, | 134 | buildVideoThumbnailsFromReq, |
148 | setVideoTags, | 135 | setVideoTags, |
149 | addOptimizeOrMergeAudioJob, | 136 | addOptimizeOrMergeAudioJob, |
137 | addTranscodingJob, | ||
138 | addMoveToObjectStorageJob, | ||
150 | getTranscodingJobPriority | 139 | getTranscodingJobPriority |
151 | } | 140 | } |
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 8a54de3b0..b3c4f390d 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { uuidToShort } from '@server/helpers/uuid' | 1 | import { uuidToShort } from '@server/helpers/uuid' |
2 | import { generateMagnetUri } from '@server/helpers/webtorrent' | 2 | import { generateMagnetUri } from '@server/helpers/webtorrent' |
3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' | 3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' |
4 | import { VideoFile } from '@shared/models/videos/video-file.model' | 4 | import { VideoFile } from '@shared/models/videos/video-file.model' |
5 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' | 5 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' |
6 | import { Video, VideoDetails } from '../../../../shared/models/videos' | 6 | import { Video, VideoDetails } from '../../../../shared/models/videos' |
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts index 742d19099..75823864d 100644 --- a/server/models/video/sql/shared/video-tables.ts +++ b/server/models/video/sql/shared/video-tables.ts | |||
@@ -87,7 +87,8 @@ export class VideoTables { | |||
87 | 'fps', | 87 | 'fps', |
88 | 'metadataUrl', | 88 | 'metadataUrl', |
89 | 'videoStreamingPlaylistId', | 89 | 'videoStreamingPlaylistId', |
90 | 'videoId' | 90 | 'videoId', |
91 | 'storage' | ||
91 | ] | 92 | ] |
92 | } | 93 | } |
93 | 94 | ||
@@ -102,7 +103,8 @@ export class VideoTables { | |||
102 | 'segmentsSha256Url', | 103 | 'segmentsSha256Url', |
103 | 'videoId', | 104 | 'videoId', |
104 | 'createdAt', | 105 | 'createdAt', |
105 | 'updatedAt' | 106 | 'updatedAt', |
107 | 'storage' | ||
106 | ]) | 108 | ]) |
107 | } | 109 | } |
108 | 110 | ||
@@ -258,7 +260,8 @@ export class VideoTables { | |||
258 | 'originallyPublishedAt', | 260 | 'originallyPublishedAt', |
259 | 'channelId', | 261 | 'channelId', |
260 | 'createdAt', | 262 | 'createdAt', |
261 | 'updatedAt' | 263 | 'updatedAt', |
264 | 'moveJobsRunning' | ||
262 | ] | 265 | ] |
263 | } | 266 | } |
264 | } | 267 | } |
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 09fc5288b..627c95763 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -23,9 +23,11 @@ import validator from 'validator' | |||
23 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' | 23 | import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' |
24 | import { logger } from '@server/helpers/logger' | 24 | import { logger } from '@server/helpers/logger' |
25 | import { extractVideo } from '@server/helpers/video' | 25 | import { extractVideo } from '@server/helpers/video' |
26 | import { getTorrentFilePath } from '@server/lib/video-paths' | 26 | import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' |
27 | import { getFSTorrentFilePath } from '@server/lib/paths' | ||
27 | import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' | 28 | import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' |
28 | import { AttributesOnly } from '@shared/core-utils' | 29 | import { AttributesOnly } from '@shared/core-utils' |
30 | import { VideoStorage } from '@shared/models' | ||
29 | import { | 31 | import { |
30 | isVideoFileExtnameValid, | 32 | isVideoFileExtnameValid, |
31 | isVideoFileInfoHashValid, | 33 | isVideoFileInfoHashValid, |
@@ -214,6 +216,11 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
214 | @Column | 216 | @Column |
215 | videoId: number | 217 | videoId: number |
216 | 218 | ||
219 | @AllowNull(false) | ||
220 | @Default(VideoStorage.FILE_SYSTEM) | ||
221 | @Column | ||
222 | storage: VideoStorage | ||
223 | |||
217 | @BelongsTo(() => VideoModel, { | 224 | @BelongsTo(() => VideoModel, { |
218 | foreignKey: { | 225 | foreignKey: { |
219 | allowNull: true | 226 | allowNull: true |
@@ -273,7 +280,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
273 | 280 | ||
274 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { | 281 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { |
275 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + | 282 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + |
276 | 'WHERE "filename" = $filename LIMIT 1' | 283 | `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` |
277 | 284 | ||
278 | return doesExist(query, { filename }) | 285 | return doesExist(query, { filename }) |
279 | } | 286 | } |
@@ -450,9 +457,20 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
450 | return !!this.videoStreamingPlaylistId | 457 | return !!this.videoStreamingPlaylistId |
451 | } | 458 | } |
452 | 459 | ||
460 | getObjectStorageUrl () { | ||
461 | if (this.isHLS()) { | ||
462 | return getHLSPublicFileUrl(this.fileUrl) | ||
463 | } | ||
464 | |||
465 | return getWebTorrentPublicFileUrl(this.fileUrl) | ||
466 | } | ||
467 | |||
453 | getFileUrl (video: MVideo) { | 468 | getFileUrl (video: MVideo) { |
454 | if (!this.Video) this.Video = video as VideoModel | 469 | if (this.storage === VideoStorage.OBJECT_STORAGE) { |
470 | return this.getObjectStorageUrl() | ||
471 | } | ||
455 | 472 | ||
473 | if (!this.Video) this.Video = video as VideoModel | ||
456 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) | 474 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) |
457 | 475 | ||
458 | return this.fileUrl | 476 | return this.fileUrl |
@@ -503,7 +521,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
503 | removeTorrent () { | 521 | removeTorrent () { |
504 | if (!this.torrentFilename) return null | 522 | if (!this.torrentFilename) return null |
505 | 523 | ||
506 | const torrentPath = getTorrentFilePath(this) | 524 | const torrentPath = getFSTorrentFilePath(this) |
507 | return remove(torrentPath) | 525 | return remove(torrentPath) |
508 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | 526 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) |
509 | } | 527 | } |
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts new file mode 100644 index 000000000..7c1fe6734 --- /dev/null +++ b/server/models/video/video-job-info.ts | |||
@@ -0,0 +1,100 @@ | |||
1 | import { Op, QueryTypes, Transaction } from 'sequelize' | ||
2 | import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' | ||
3 | import { AttributesOnly } from '@shared/core-utils' | ||
4 | import { VideoModel } from './video' | ||
5 | |||
6 | @Table({ | ||
7 | tableName: 'videoJobInfo', | ||
8 | indexes: [ | ||
9 | { | ||
10 | fields: [ 'videoId' ], | ||
11 | where: { | ||
12 | videoId: { | ||
13 | [Op.ne]: null | ||
14 | } | ||
15 | } | ||
16 | } | ||
17 | ] | ||
18 | }) | ||
19 | |||
20 | export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfoModel>>> { | ||
21 | @CreatedAt | ||
22 | createdAt: Date | ||
23 | |||
24 | @UpdatedAt | ||
25 | updatedAt: Date | ||
26 | |||
27 | @AllowNull(false) | ||
28 | @Default(0) | ||
29 | @IsInt | ||
30 | @Column | ||
31 | pendingMove: number | ||
32 | |||
33 | @AllowNull(false) | ||
34 | @Default(0) | ||
35 | @IsInt | ||
36 | @Column | ||
37 | pendingTranscode: number | ||
38 | |||
39 | @ForeignKey(() => VideoModel) | ||
40 | @Unique | ||
41 | @Column | ||
42 | videoId: number | ||
43 | |||
44 | @BelongsTo(() => VideoModel, { | ||
45 | foreignKey: { | ||
46 | allowNull: false | ||
47 | }, | ||
48 | onDelete: 'cascade' | ||
49 | }) | ||
50 | Video: VideoModel | ||
51 | |||
52 | static load (videoId: number, transaction: Transaction) { | ||
53 | const where = { | ||
54 | videoId | ||
55 | } | ||
56 | |||
57 | return VideoJobInfoModel.findOne({ where, transaction }) | ||
58 | } | ||
59 | |||
60 | static async increaseOrCreate (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> { | ||
61 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | ||
62 | |||
63 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | ||
64 | INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") | ||
65 | SELECT | ||
66 | "video"."id" AS "videoId", 1, NOW(), NOW() | ||
67 | FROM | ||
68 | "video" | ||
69 | WHERE | ||
70 | "video"."uuid" = $videoUUID | ||
71 | ON CONFLICT ("videoId") DO UPDATE | ||
72 | SET | ||
73 | "${column}" = "videoJobInfo"."${column}" + 1, | ||
74 | "updatedAt" = NOW() | ||
75 | RETURNING | ||
76 | "${column}" | ||
77 | `, options) | ||
78 | |||
79 | return pendingMove | ||
80 | } | ||
81 | |||
82 | static async decrease (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> { | ||
83 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | ||
84 | |||
85 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | ||
86 | UPDATE | ||
87 | "videoJobInfo" | ||
88 | SET | ||
89 | "${column}" = "videoJobInfo"."${column}" - 1, | ||
90 | "updatedAt" = NOW() | ||
91 | FROM "video" | ||
92 | WHERE | ||
93 | "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID | ||
94 | RETURNING | ||
95 | "${column}"; | ||
96 | `, options) | ||
97 | |||
98 | return pendingMove | ||
99 | } | ||
100 | } | ||
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index d591a3134..3e9fd97c7 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -1,10 +1,25 @@ | |||
1 | import * as memoizee from 'memoizee' | 1 | import * as memoizee from 'memoizee' |
2 | import { join } from 'path' | 2 | import { join } from 'path' |
3 | import { Op } from 'sequelize' | 3 | import { Op } from 'sequelize' |
4 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' | 4 | import { |
5 | AllowNull, | ||
6 | BelongsTo, | ||
7 | Column, | ||
8 | CreatedAt, | ||
9 | DataType, | ||
10 | Default, | ||
11 | ForeignKey, | ||
12 | HasMany, | ||
13 | Is, | ||
14 | Model, | ||
15 | Table, | ||
16 | UpdatedAt | ||
17 | } from 'sequelize-typescript' | ||
18 | import { getHLSPublicFileUrl } from '@server/lib/object-storage' | ||
5 | import { VideoFileModel } from '@server/models/video/video-file' | 19 | import { VideoFileModel } from '@server/models/video/video-file' |
6 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 20 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
7 | import { AttributesOnly } from '@shared/core-utils' | 21 | import { AttributesOnly } from '@shared/core-utils' |
22 | import { VideoStorage } from '@shared/models' | ||
8 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 23 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
9 | import { sha1 } from '../../helpers/core-utils' | 24 | import { sha1 } from '../../helpers/core-utils' |
10 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 25 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
@@ -81,6 +96,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
81 | @Column | 96 | @Column |
82 | videoId: number | 97 | videoId: number |
83 | 98 | ||
99 | @AllowNull(false) | ||
100 | @Default(VideoStorage.FILE_SYSTEM) | ||
101 | @Column | ||
102 | storage: VideoStorage | ||
103 | |||
84 | @BelongsTo(() => VideoModel, { | 104 | @BelongsTo(() => VideoModel, { |
85 | foreignKey: { | 105 | foreignKey: { |
86 | allowNull: false | 106 | allowNull: false |
@@ -185,12 +205,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
185 | } | 205 | } |
186 | 206 | ||
187 | getMasterPlaylistUrl (video: MVideo) { | 207 | getMasterPlaylistUrl (video: MVideo) { |
208 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
209 | return getHLSPublicFileUrl(this.playlistUrl) | ||
210 | } | ||
211 | |||
188 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) | 212 | if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) |
189 | 213 | ||
190 | return this.playlistUrl | 214 | return this.playlistUrl |
191 | } | 215 | } |
192 | 216 | ||
193 | getSha256SegmentsUrl (video: MVideo) { | 217 | getSha256SegmentsUrl (video: MVideo) { |
218 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | ||
219 | return getHLSPublicFileUrl(this.segmentsSha256Url) | ||
220 | } | ||
221 | |||
194 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) | 222 | if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) |
195 | 223 | ||
196 | return this.segmentsSha256Url | 224 | return this.segmentsSha256Url |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 56a5b0e18..874ad168a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -28,14 +28,16 @@ import { buildNSFWFilter } from '@server/helpers/express-utils' | |||
28 | import { uuidToShort } from '@server/helpers/uuid' | 28 | import { uuidToShort } from '@server/helpers/uuid' |
29 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' | 29 | import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' |
30 | import { LiveManager } from '@server/lib/live/live-manager' | 30 | import { LiveManager } from '@server/lib/live/live-manager' |
31 | import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' | 31 | import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' |
32 | import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths' | ||
33 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
32 | import { getServerActor } from '@server/models/application/application' | 34 | import { getServerActor } from '@server/models/application/application' |
33 | import { ModelCache } from '@server/models/model-cache' | 35 | import { ModelCache } from '@server/models/model-cache' |
34 | import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' | 36 | import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' |
35 | import { VideoFile } from '@shared/models/videos/video-file.model' | 37 | import { VideoFile } from '@shared/models/videos/video-file.model' |
36 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' | 38 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' |
37 | import { VideoObject } from '../../../shared/models/activitypub/objects' | 39 | import { VideoObject } from '../../../shared/models/activitypub/objects' |
38 | import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos' | 40 | import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos' |
39 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' | 41 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' |
40 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 42 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
41 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 43 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
@@ -114,6 +116,7 @@ import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel | |||
114 | import { VideoCommentModel } from './video-comment' | 116 | import { VideoCommentModel } from './video-comment' |
115 | import { VideoFileModel } from './video-file' | 117 | import { VideoFileModel } from './video-file' |
116 | import { VideoImportModel } from './video-import' | 118 | import { VideoImportModel } from './video-import' |
119 | import { VideoJobInfoModel } from './video-job-info' | ||
117 | import { VideoLiveModel } from './video-live' | 120 | import { VideoLiveModel } from './video-live' |
118 | import { VideoPlaylistElementModel } from './video-playlist-element' | 121 | import { VideoPlaylistElementModel } from './video-playlist-element' |
119 | import { VideoShareModel } from './video-share' | 122 | import { VideoShareModel } from './video-share' |
@@ -732,6 +735,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
732 | }) | 735 | }) |
733 | VideoCaptions: VideoCaptionModel[] | 736 | VideoCaptions: VideoCaptionModel[] |
734 | 737 | ||
738 | @HasOne(() => VideoJobInfoModel, { | ||
739 | foreignKey: { | ||
740 | name: 'videoId', | ||
741 | allowNull: false | ||
742 | }, | ||
743 | onDelete: 'cascade' | ||
744 | }) | ||
745 | VideoJobInfo: VideoJobInfoModel | ||
746 | |||
735 | @BeforeDestroy | 747 | @BeforeDestroy |
736 | static async sendDelete (instance: MVideoAccountLight, options) { | 748 | static async sendDelete (instance: MVideoAccountLight, options) { |
737 | if (!instance.isOwned()) return undefined | 749 | if (!instance.isOwned()) return undefined |
@@ -1641,9 +1653,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1641 | getMaxQualityResolution () { | 1653 | getMaxQualityResolution () { |
1642 | const file = this.getMaxQualityFile() | 1654 | const file = this.getMaxQualityFile() |
1643 | const videoOrPlaylist = file.getVideoOrStreamingPlaylist() | 1655 | const videoOrPlaylist = file.getVideoOrStreamingPlaylist() |
1644 | const originalFilePath = getVideoFilePath(videoOrPlaylist, file) | ||
1645 | 1656 | ||
1646 | return getVideoFileResolution(originalFilePath) | 1657 | return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => { |
1658 | return getVideoFileResolution(originalFilePath) | ||
1659 | }) | ||
1647 | } | 1660 | } |
1648 | 1661 | ||
1649 | getDescriptionAPIPath () { | 1662 | getDescriptionAPIPath () { |
@@ -1673,16 +1686,24 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1673 | } | 1686 | } |
1674 | 1687 | ||
1675 | removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { | 1688 | removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { |
1676 | const filePath = getVideoFilePath(this, videoFile, isRedundancy) | 1689 | const filePath = isRedundancy |
1690 | ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) | ||
1691 | : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) | ||
1677 | 1692 | ||
1678 | const promises: Promise<any>[] = [ remove(filePath) ] | 1693 | const promises: Promise<any>[] = [ remove(filePath) ] |
1679 | if (!isRedundancy) promises.push(videoFile.removeTorrent()) | 1694 | if (!isRedundancy) promises.push(videoFile.removeTorrent()) |
1680 | 1695 | ||
1696 | if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { | ||
1697 | promises.push(removeWebTorrentObjectStorage(videoFile)) | ||
1698 | } | ||
1699 | |||
1681 | return Promise.all(promises) | 1700 | return Promise.all(promises) |
1682 | } | 1701 | } |
1683 | 1702 | ||
1684 | async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { | 1703 | async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { |
1685 | const directoryPath = getHLSDirectory(this, isRedundancy) | 1704 | const directoryPath = isRedundancy |
1705 | ? getHLSRedundancyDirectory(this) | ||
1706 | : getHLSDirectory(this) | ||
1686 | 1707 | ||
1687 | await remove(directoryPath) | 1708 | await remove(directoryPath) |
1688 | 1709 | ||
@@ -1698,6 +1719,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1698 | await Promise.all( | 1719 | await Promise.all( |
1699 | streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) | 1720 | streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) |
1700 | ) | 1721 | ) |
1722 | |||
1723 | if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { | ||
1724 | await removeHLSObjectStorage(streamingPlaylist, this) | ||
1725 | } | ||
1701 | } | 1726 | } |
1702 | } | 1727 | } |
1703 | 1728 | ||
@@ -1741,16 +1766,16 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1741 | this.privacy === VideoPrivacy.INTERNAL | 1766 | this.privacy === VideoPrivacy.INTERNAL |
1742 | } | 1767 | } |
1743 | 1768 | ||
1744 | async publishIfNeededAndSave (t: Transaction) { | 1769 | async setNewState (newState: VideoState, transaction: Transaction) { |
1745 | if (this.state !== VideoState.PUBLISHED) { | 1770 | if (this.state === newState) throw new Error('Cannot use same state ' + newState) |
1746 | this.state = VideoState.PUBLISHED | 1771 | |
1747 | this.publishedAt = new Date() | 1772 | this.state = newState |
1748 | await this.save({ transaction: t }) | ||
1749 | 1773 | ||
1750 | return true | 1774 | if (this.state === VideoState.PUBLISHED) { |
1775 | this.publishedAt = new Date() | ||
1751 | } | 1776 | } |
1752 | 1777 | ||
1753 | return false | 1778 | await this.save({ transaction }) |
1754 | } | 1779 | } |
1755 | 1780 | ||
1756 | getBandwidthBits (videoFile: MVideoFile) { | 1781 | getBandwidthBits (videoFile: MVideoFile) { |
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index b62e2f5f7..19301c0b9 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts | |||
@@ -2,6 +2,7 @@ | |||
2 | import './activitypub' | 2 | import './activitypub' |
3 | import './check-params' | 3 | import './check-params' |
4 | import './moderation' | 4 | import './moderation' |
5 | import './object-storage' | ||
5 | import './notifications' | 6 | import './notifications' |
6 | import './redundancy' | 7 | import './redundancy' |
7 | import './search' | 8 | import './search' |
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index 8f1fb78a5..6c4ea90ca 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts | |||
@@ -15,7 +15,9 @@ import { | |||
15 | stopFfmpeg, | 15 | stopFfmpeg, |
16 | testFfmpegStreamError, | 16 | testFfmpegStreamError, |
17 | wait, | 17 | wait, |
18 | waitJobs | 18 | waitJobs, |
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveSavedOnAllServers | ||
19 | } from '@shared/extra-utils' | 21 | } from '@shared/extra-utils' |
20 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' | 22 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' |
21 | 23 | ||
@@ -66,18 +68,6 @@ describe('Save replay setting', function () { | |||
66 | } | 68 | } |
67 | } | 69 | } |
68 | 70 | ||
69 | async function waitUntilLivePublishedOnAllServers (videoId: string) { | ||
70 | for (const server of servers) { | ||
71 | await server.live.waitUntilPublished({ videoId }) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | async function waitUntilLiveSavedOnAllServers (videoId: string) { | ||
76 | for (const server of servers) { | ||
77 | await server.live.waitUntilSaved({ videoId }) | ||
78 | } | ||
79 | } | ||
80 | |||
81 | before(async function () { | 71 | before(async function () { |
82 | this.timeout(120000) | 72 | this.timeout(120000) |
83 | 73 | ||
@@ -127,7 +117,7 @@ describe('Save replay setting', function () { | |||
127 | 117 | ||
128 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 118 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
129 | 119 | ||
130 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 120 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
131 | 121 | ||
132 | await waitJobs(servers) | 122 | await waitJobs(servers) |
133 | 123 | ||
@@ -160,7 +150,7 @@ describe('Save replay setting', function () { | |||
160 | 150 | ||
161 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 151 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
162 | 152 | ||
163 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 153 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
164 | 154 | ||
165 | await waitJobs(servers) | 155 | await waitJobs(servers) |
166 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 156 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -189,7 +179,7 @@ describe('Save replay setting', function () { | |||
189 | 179 | ||
190 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 180 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
191 | 181 | ||
192 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 182 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
193 | 183 | ||
194 | await waitJobs(servers) | 184 | await waitJobs(servers) |
195 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 185 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -224,7 +214,7 @@ describe('Save replay setting', function () { | |||
224 | this.timeout(20000) | 214 | this.timeout(20000) |
225 | 215 | ||
226 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 216 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
227 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 217 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
228 | 218 | ||
229 | await waitJobs(servers) | 219 | await waitJobs(servers) |
230 | 220 | ||
@@ -237,7 +227,7 @@ describe('Save replay setting', function () { | |||
237 | 227 | ||
238 | await stopFfmpeg(ffmpegCommand) | 228 | await stopFfmpeg(ffmpegCommand) |
239 | 229 | ||
240 | await waitUntilLiveSavedOnAllServers(liveVideoUUID) | 230 | await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID) |
241 | await waitJobs(servers) | 231 | await waitJobs(servers) |
242 | 232 | ||
243 | // Live has been transcoded | 233 | // Live has been transcoded |
@@ -268,7 +258,7 @@ describe('Save replay setting', function () { | |||
268 | liveVideoUUID = await createLiveWrapper(true) | 258 | liveVideoUUID = await createLiveWrapper(true) |
269 | 259 | ||
270 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 260 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
271 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 261 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
272 | 262 | ||
273 | await waitJobs(servers) | 263 | await waitJobs(servers) |
274 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 264 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
@@ -296,7 +286,7 @@ describe('Save replay setting', function () { | |||
296 | liveVideoUUID = await createLiveWrapper(true) | 286 | liveVideoUUID = await createLiveWrapper(true) |
297 | 287 | ||
298 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) | 288 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) |
299 | await waitUntilLivePublishedOnAllServers(liveVideoUUID) | 289 | await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) |
300 | 290 | ||
301 | await waitJobs(servers) | 291 | await waitJobs(servers) |
302 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) | 292 | await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) |
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts new file mode 100644 index 000000000..f319d6ef5 --- /dev/null +++ b/server/tests/api/object-storage/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './live' | ||
2 | export * from './video-imports' | ||
3 | export * from './videos' | ||
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts new file mode 100644 index 000000000..d3e6777f2 --- /dev/null +++ b/server/tests/api/object-storage/live.ts | |||
@@ -0,0 +1,136 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { | ||
7 | areObjectStorageTestsDisabled, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | killallServers, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
14 | PeerTubeServer, | ||
15 | setAccessTokensToServers, | ||
16 | setDefaultVideoChannel, | ||
17 | stopFfmpeg, | ||
18 | waitJobs, | ||
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveSavedOnAllServers | ||
21 | } from '@shared/extra-utils' | ||
22 | import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models' | ||
23 | |||
24 | const expect = chai.expect | ||
25 | |||
26 | async function createLive (server: PeerTubeServer) { | ||
27 | const attributes: LiveVideoCreate = { | ||
28 | channelId: server.store.channel.id, | ||
29 | privacy: VideoPrivacy.PUBLIC, | ||
30 | name: 'my super live', | ||
31 | saveReplay: true | ||
32 | } | ||
33 | |||
34 | const { uuid } = await server.live.create({ fields: attributes }) | ||
35 | |||
36 | return uuid | ||
37 | } | ||
38 | |||
39 | async function checkFiles (files: VideoFile[]) { | ||
40 | for (const file of files) { | ||
41 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
42 | |||
43 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
44 | } | ||
45 | } | ||
46 | |||
47 | describe('Object storage for lives', function () { | ||
48 | if (areObjectStorageTestsDisabled()) return | ||
49 | |||
50 | let ffmpegCommand: FfmpegCommand | ||
51 | let servers: PeerTubeServer[] | ||
52 | let videoUUID: string | ||
53 | |||
54 | before(async function () { | ||
55 | this.timeout(120000) | ||
56 | |||
57 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
58 | |||
59 | servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) | ||
60 | |||
61 | await setAccessTokensToServers(servers) | ||
62 | await setDefaultVideoChannel(servers) | ||
63 | await doubleFollow(servers[0], servers[1]) | ||
64 | |||
65 | await servers[0].config.enableTranscoding() | ||
66 | }) | ||
67 | |||
68 | describe('Without live transcoding', async function () { | ||
69 | |||
70 | before(async function () { | ||
71 | await servers[0].config.enableLive({ transcoding: false }) | ||
72 | |||
73 | videoUUID = await createLive(servers[0]) | ||
74 | }) | ||
75 | |||
76 | it('Should create a live and save the replay on object storage', async function () { | ||
77 | this.timeout(220000) | ||
78 | |||
79 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
80 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
81 | |||
82 | await stopFfmpeg(ffmpegCommand) | ||
83 | |||
84 | await waitUntilLiveSavedOnAllServers(servers, videoUUID) | ||
85 | await waitJobs(servers) | ||
86 | |||
87 | for (const server of servers) { | ||
88 | const video = await server.videos.get({ id: videoUUID }) | ||
89 | |||
90 | expect(video.files).to.have.lengthOf(0) | ||
91 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
92 | |||
93 | const files = video.streamingPlaylists[0].files | ||
94 | |||
95 | await checkFiles(files) | ||
96 | } | ||
97 | }) | ||
98 | }) | ||
99 | |||
100 | describe('With live transcoding', async function () { | ||
101 | |||
102 | before(async function () { | ||
103 | await servers[0].config.enableLive({ transcoding: true }) | ||
104 | |||
105 | videoUUID = await createLive(servers[0]) | ||
106 | }) | ||
107 | |||
108 | it('Should import a video and have sent it to object storage', async function () { | ||
109 | this.timeout(240000) | ||
110 | |||
111 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
112 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
113 | |||
114 | await stopFfmpeg(ffmpegCommand) | ||
115 | |||
116 | await waitUntilLiveSavedOnAllServers(servers, videoUUID) | ||
117 | await waitJobs(servers) | ||
118 | |||
119 | for (const server of servers) { | ||
120 | const video = await server.videos.get({ id: videoUUID }) | ||
121 | |||
122 | expect(video.files).to.have.lengthOf(0) | ||
123 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
124 | |||
125 | const files = video.streamingPlaylists[0].files | ||
126 | expect(files).to.have.lengthOf(4) | ||
127 | |||
128 | await checkFiles(files) | ||
129 | } | ||
130 | }) | ||
131 | }) | ||
132 | |||
133 | after(async function () { | ||
134 | await killallServers(servers) | ||
135 | }) | ||
136 | }) | ||
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts new file mode 100644 index 000000000..efc01f550 --- /dev/null +++ b/server/tests/api/object-storage/video-imports.ts | |||
@@ -0,0 +1,112 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { | ||
6 | areObjectStorageTestsDisabled, | ||
7 | createSingleServer, | ||
8 | expectStartWith, | ||
9 | FIXTURE_URLS, | ||
10 | killallServers, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | setDefaultVideoChannel, | ||
16 | waitJobs | ||
17 | } from '@shared/extra-utils' | ||
18 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | ||
19 | |||
20 | const expect = chai.expect | ||
21 | |||
22 | async function importVideo (server: PeerTubeServer) { | ||
23 | const attributes = { | ||
24 | name: 'import 2', | ||
25 | privacy: VideoPrivacy.PUBLIC, | ||
26 | channelId: server.store.channel.id, | ||
27 | targetUrl: FIXTURE_URLS.goodVideo720 | ||
28 | } | ||
29 | |||
30 | const { video: { uuid } } = await server.imports.importVideo({ attributes }) | ||
31 | |||
32 | return uuid | ||
33 | } | ||
34 | |||
35 | describe('Object storage for video import', function () { | ||
36 | if (areObjectStorageTestsDisabled()) return | ||
37 | |||
38 | let server: PeerTubeServer | ||
39 | |||
40 | before(async function () { | ||
41 | this.timeout(120000) | ||
42 | |||
43 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
44 | |||
45 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) | ||
46 | |||
47 | await setAccessTokensToServers([ server ]) | ||
48 | await setDefaultVideoChannel([ server ]) | ||
49 | |||
50 | await server.config.enableImports() | ||
51 | }) | ||
52 | |||
53 | describe('Without transcoding', async function () { | ||
54 | |||
55 | before(async function () { | ||
56 | await server.config.disableTranscoding() | ||
57 | }) | ||
58 | |||
59 | it('Should import a video and have sent it to object storage', async function () { | ||
60 | this.timeout(120000) | ||
61 | |||
62 | const uuid = await importVideo(server) | ||
63 | await waitJobs(server) | ||
64 | |||
65 | const video = await server.videos.get({ id: uuid }) | ||
66 | |||
67 | expect(video.files).to.have.lengthOf(1) | ||
68 | expect(video.streamingPlaylists).to.have.lengthOf(0) | ||
69 | |||
70 | const fileUrl = video.files[0].fileUrl | ||
71 | expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
72 | |||
73 | await makeRawRequest(fileUrl, HttpStatusCode.OK_200) | ||
74 | }) | ||
75 | }) | ||
76 | |||
77 | describe('With transcoding', async function () { | ||
78 | |||
79 | before(async function () { | ||
80 | await server.config.enableTranscoding() | ||
81 | }) | ||
82 | |||
83 | it('Should import a video and have sent it to object storage', async function () { | ||
84 | this.timeout(120000) | ||
85 | |||
86 | const uuid = await importVideo(server) | ||
87 | await waitJobs(server) | ||
88 | |||
89 | const video = await server.videos.get({ id: uuid }) | ||
90 | |||
91 | expect(video.files).to.have.lengthOf(4) | ||
92 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
93 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(4) | ||
94 | |||
95 | for (const file of video.files) { | ||
96 | expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
97 | |||
98 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
99 | } | ||
100 | |||
101 | for (const file of video.streamingPlaylists[0].files) { | ||
102 | expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
103 | |||
104 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
105 | } | ||
106 | }) | ||
107 | }) | ||
108 | |||
109 | after(async function () { | ||
110 | await killallServers([ server ]) | ||
111 | }) | ||
112 | }) | ||
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts new file mode 100644 index 000000000..3958bd3d7 --- /dev/null +++ b/server/tests/api/object-storage/videos.ts | |||
@@ -0,0 +1,391 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { merge } from 'lodash' | ||
6 | import { | ||
7 | areObjectStorageTestsDisabled, | ||
8 | checkTmpIsEmpty, | ||
9 | cleanupTests, | ||
10 | createMultipleServers, | ||
11 | createSingleServer, | ||
12 | doubleFollow, | ||
13 | expectStartWith, | ||
14 | killallServers, | ||
15 | makeRawRequest, | ||
16 | MockObjectStorage, | ||
17 | ObjectStorageCommand, | ||
18 | PeerTubeServer, | ||
19 | setAccessTokensToServers, | ||
20 | waitJobs, | ||
21 | webtorrentAdd | ||
22 | } from '@shared/extra-utils' | ||
23 | import { HttpStatusCode, VideoDetails } from '@shared/models' | ||
24 | |||
25 | const expect = chai.expect | ||
26 | |||
27 | async function checkFiles (options: { | ||
28 | video: VideoDetails | ||
29 | |||
30 | baseMockUrl?: string | ||
31 | |||
32 | playlistBucket: string | ||
33 | playlistPrefix?: string | ||
34 | |||
35 | webtorrentBucket: string | ||
36 | webtorrentPrefix?: string | ||
37 | }) { | ||
38 | const { | ||
39 | video, | ||
40 | playlistBucket, | ||
41 | webtorrentBucket, | ||
42 | baseMockUrl, | ||
43 | playlistPrefix, | ||
44 | webtorrentPrefix | ||
45 | } = options | ||
46 | |||
47 | let allFiles = video.files | ||
48 | |||
49 | for (const file of video.files) { | ||
50 | const baseUrl = baseMockUrl | ||
51 | ? `${baseMockUrl}/${webtorrentBucket}/` | ||
52 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | ||
53 | |||
54 | const prefix = webtorrentPrefix || '' | ||
55 | const start = baseUrl + prefix | ||
56 | |||
57 | expectStartWith(file.fileUrl, start) | ||
58 | |||
59 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | ||
60 | const location = res.headers['location'] | ||
61 | expectStartWith(location, start) | ||
62 | |||
63 | await makeRawRequest(location, HttpStatusCode.OK_200) | ||
64 | } | ||
65 | |||
66 | const hls = video.streamingPlaylists[0] | ||
67 | |||
68 | if (hls) { | ||
69 | allFiles = allFiles.concat(hls.files) | ||
70 | |||
71 | const baseUrl = baseMockUrl | ||
72 | ? `${baseMockUrl}/${playlistBucket}/` | ||
73 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | ||
74 | |||
75 | const prefix = playlistPrefix || '' | ||
76 | const start = baseUrl + prefix | ||
77 | |||
78 | expectStartWith(hls.playlistUrl, start) | ||
79 | expectStartWith(hls.segmentsSha256Url, start) | ||
80 | |||
81 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | ||
82 | |||
83 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | ||
84 | expect(JSON.stringify(resSha.body)).to.not.throw | ||
85 | |||
86 | for (const file of hls.files) { | ||
87 | expectStartWith(file.fileUrl, start) | ||
88 | |||
89 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | ||
90 | const location = res.headers['location'] | ||
91 | expectStartWith(location, start) | ||
92 | |||
93 | await makeRawRequest(location, HttpStatusCode.OK_200) | ||
94 | } | ||
95 | } | ||
96 | |||
97 | for (const file of allFiles) { | ||
98 | const torrent = await webtorrentAdd(file.magnetUri, true) | ||
99 | |||
100 | expect(torrent.files).to.be.an('array') | ||
101 | expect(torrent.files.length).to.equal(1) | ||
102 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | ||
103 | |||
104 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | ||
105 | expect(res.body).to.have.length.above(100) | ||
106 | } | ||
107 | |||
108 | return allFiles.map(f => f.fileUrl) | ||
109 | } | ||
110 | |||
111 | function runTestSuite (options: { | ||
112 | playlistBucket: string | ||
113 | playlistPrefix?: string | ||
114 | |||
115 | webtorrentBucket: string | ||
116 | webtorrentPrefix?: string | ||
117 | |||
118 | useMockBaseUrl?: boolean | ||
119 | |||
120 | maxUploadPart?: string | ||
121 | }) { | ||
122 | const mockObjectStorage = new MockObjectStorage() | ||
123 | let baseMockUrl: string | ||
124 | |||
125 | let servers: PeerTubeServer[] | ||
126 | |||
127 | let keptUrls: string[] = [] | ||
128 | |||
129 | const uuidsToDelete: string[] = [] | ||
130 | let deletedUrls: string[] = [] | ||
131 | |||
132 | before(async function () { | ||
133 | this.timeout(120000) | ||
134 | |||
135 | const port = await mockObjectStorage.initialize() | ||
136 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | ||
137 | |||
138 | await ObjectStorageCommand.createBucket(options.playlistBucket) | ||
139 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | ||
140 | |||
141 | const config = { | ||
142 | object_storage: { | ||
143 | enabled: true, | ||
144 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | ||
145 | region: ObjectStorageCommand.getRegion(), | ||
146 | |||
147 | credentials: ObjectStorageCommand.getCredentialsConfig(), | ||
148 | |||
149 | max_upload_part: options.maxUploadPart || '2MB', | ||
150 | |||
151 | streaming_playlists: { | ||
152 | bucket_name: options.playlistBucket, | ||
153 | prefix: options.playlistPrefix, | ||
154 | base_url: baseMockUrl | ||
155 | ? `${baseMockUrl}/${options.playlistBucket}` | ||
156 | : undefined | ||
157 | }, | ||
158 | |||
159 | videos: { | ||
160 | bucket_name: options.webtorrentBucket, | ||
161 | prefix: options.webtorrentPrefix, | ||
162 | base_url: baseMockUrl | ||
163 | ? `${baseMockUrl}/${options.webtorrentBucket}` | ||
164 | : undefined | ||
165 | } | ||
166 | } | ||
167 | } | ||
168 | |||
169 | servers = await createMultipleServers(2, config) | ||
170 | |||
171 | await setAccessTokensToServers(servers) | ||
172 | await doubleFollow(servers[0], servers[1]) | ||
173 | |||
174 | for (const server of servers) { | ||
175 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | ||
176 | await waitJobs(servers) | ||
177 | |||
178 | const files = await server.videos.listFiles({ id: uuid }) | ||
179 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | ||
180 | } | ||
181 | }) | ||
182 | |||
183 | it('Should upload a video and move it to the object storage without transcoding', async function () { | ||
184 | this.timeout(20000) | ||
185 | |||
186 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | ||
187 | uuidsToDelete.push(uuid) | ||
188 | |||
189 | await waitJobs(servers) | ||
190 | |||
191 | for (const server of servers) { | ||
192 | const video = await server.videos.get({ id: uuid }) | ||
193 | const files = await checkFiles({ ...options, video, baseMockUrl }) | ||
194 | |||
195 | deletedUrls = deletedUrls.concat(files) | ||
196 | } | ||
197 | }) | ||
198 | |||
199 | it('Should upload a video and move it to the object storage with transcoding', async function () { | ||
200 | this.timeout(40000) | ||
201 | |||
202 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' }) | ||
203 | uuidsToDelete.push(uuid) | ||
204 | |||
205 | await waitJobs(servers) | ||
206 | |||
207 | for (const server of servers) { | ||
208 | const video = await server.videos.get({ id: uuid }) | ||
209 | const files = await checkFiles({ ...options, video, baseMockUrl }) | ||
210 | |||
211 | deletedUrls = deletedUrls.concat(files) | ||
212 | } | ||
213 | }) | ||
214 | |||
215 | it('Should correctly delete the files', async function () { | ||
216 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | ||
217 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | ||
218 | |||
219 | await waitJobs(servers) | ||
220 | |||
221 | for (const url of deletedUrls) { | ||
222 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | ||
223 | } | ||
224 | }) | ||
225 | |||
226 | it('Should have kept other files', async function () { | ||
227 | for (const url of keptUrls) { | ||
228 | await makeRawRequest(url, HttpStatusCode.OK_200) | ||
229 | } | ||
230 | }) | ||
231 | |||
232 | it('Should have an empty tmp directory', async function () { | ||
233 | for (const server of servers) { | ||
234 | await checkTmpIsEmpty(server) | ||
235 | } | ||
236 | }) | ||
237 | |||
238 | after(async function () { | ||
239 | mockObjectStorage.terminate() | ||
240 | |||
241 | await cleanupTests(servers) | ||
242 | }) | ||
243 | } | ||
244 | |||
245 | describe('Object storage for videos', function () { | ||
246 | if (areObjectStorageTestsDisabled()) return | ||
247 | |||
248 | describe('Test config', function () { | ||
249 | let server: PeerTubeServer | ||
250 | |||
251 | const baseConfig = { | ||
252 | object_storage: { | ||
253 | enabled: true, | ||
254 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | ||
255 | region: ObjectStorageCommand.getRegion(), | ||
256 | |||
257 | credentials: ObjectStorageCommand.getCredentialsConfig(), | ||
258 | |||
259 | streaming_playlists: { | ||
260 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | ||
261 | }, | ||
262 | |||
263 | videos: { | ||
264 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | ||
265 | } | ||
266 | } | ||
267 | } | ||
268 | |||
269 | const badCredentials = { | ||
270 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
271 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
272 | } | ||
273 | |||
274 | it('Should fail with same bucket names without prefix', function (done) { | ||
275 | const config = merge({}, baseConfig, { | ||
276 | object_storage: { | ||
277 | streaming_playlists: { | ||
278 | bucket_name: 'aaa' | ||
279 | }, | ||
280 | |||
281 | videos: { | ||
282 | bucket_name: 'aaa' | ||
283 | } | ||
284 | } | ||
285 | }) | ||
286 | |||
287 | createSingleServer(1, config) | ||
288 | .then(() => done(new Error('Did not throw'))) | ||
289 | .catch(() => done()) | ||
290 | }) | ||
291 | |||
292 | it('Should fail with bad credentials', async function () { | ||
293 | this.timeout(60000) | ||
294 | |||
295 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
296 | |||
297 | const config = merge({}, baseConfig, { | ||
298 | object_storage: { | ||
299 | credentials: badCredentials | ||
300 | } | ||
301 | }) | ||
302 | |||
303 | server = await createSingleServer(1, config) | ||
304 | await setAccessTokensToServers([ server ]) | ||
305 | |||
306 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
307 | |||
308 | await waitJobs([ server ], true) | ||
309 | const video = await server.videos.get({ id: uuid }) | ||
310 | |||
311 | expectStartWith(video.files[0].fileUrl, server.url) | ||
312 | |||
313 | await killallServers([ server ]) | ||
314 | }) | ||
315 | |||
316 | it('Should succeed with credentials from env', async function () { | ||
317 | this.timeout(60000) | ||
318 | |||
319 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
320 | |||
321 | const config = merge({}, baseConfig, { | ||
322 | object_storage: { | ||
323 | credentials: { | ||
324 | access_key_id: '', | ||
325 | secret_access_key: '' | ||
326 | } | ||
327 | } | ||
328 | }) | ||
329 | |||
330 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | ||
331 | |||
332 | server = await createSingleServer(1, config, { | ||
333 | env: { | ||
334 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | ||
335 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | ||
336 | } | ||
337 | }) | ||
338 | |||
339 | await setAccessTokensToServers([ server ]) | ||
340 | |||
341 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
342 | |||
343 | await waitJobs([ server ], true) | ||
344 | const video = await server.videos.get({ id: uuid }) | ||
345 | |||
346 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
347 | }) | ||
348 | |||
349 | after(async function () { | ||
350 | await killallServers([ server ]) | ||
351 | }) | ||
352 | }) | ||
353 | |||
354 | describe('Test simple object storage', function () { | ||
355 | runTestSuite({ | ||
356 | playlistBucket: 'streaming-playlists', | ||
357 | webtorrentBucket: 'videos' | ||
358 | }) | ||
359 | }) | ||
360 | |||
361 | describe('Test object storage with prefix', function () { | ||
362 | runTestSuite({ | ||
363 | playlistBucket: 'mybucket', | ||
364 | webtorrentBucket: 'mybucket', | ||
365 | |||
366 | playlistPrefix: 'streaming-playlists_', | ||
367 | webtorrentPrefix: 'webtorrent_' | ||
368 | }) | ||
369 | }) | ||
370 | |||
371 | describe('Test object storage with prefix and base URL', function () { | ||
372 | runTestSuite({ | ||
373 | playlistBucket: 'mybucket', | ||
374 | webtorrentBucket: 'mybucket', | ||
375 | |||
376 | playlistPrefix: 'streaming-playlists_', | ||
377 | webtorrentPrefix: 'webtorrent_', | ||
378 | |||
379 | useMockBaseUrl: true | ||
380 | }) | ||
381 | }) | ||
382 | |||
383 | describe('Test object storage with small upload part', function () { | ||
384 | runTestSuite({ | ||
385 | playlistBucket: 'streaming-playlists', | ||
386 | webtorrentBucket: 'videos', | ||
387 | |||
388 | maxUploadPart: '5KB' | ||
389 | }) | ||
390 | }) | ||
391 | }) | ||
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index e1a12f5f8..3400b1d9a 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -207,14 +207,14 @@ async function check1PlaylistRedundancies (videoUUID?: string) { | |||
207 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) | 207 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) |
208 | } | 208 | } |
209 | 209 | ||
210 | const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' | 210 | const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID |
211 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls' | 211 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID |
212 | 212 | ||
213 | const video = await servers[0].videos.get({ id: videoUUID }) | 213 | const video = await servers[0].videos.get({ id: videoUUID }) |
214 | const hlsPlaylist = video.streamingPlaylists[0] | 214 | const hlsPlaylist = video.streamingPlaylists[0] |
215 | 215 | ||
216 | for (const resolution of [ 240, 360, 480, 720 ]) { | 216 | for (const resolution of [ 240, 360, 480, 720 ]) { |
217 | await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) | 217 | await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist }) |
218 | } | 218 | } |
219 | 219 | ||
220 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) | 220 | const { hlsFilenames } = await ensureSameFilenames(videoUUID) |
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 961f0e617..2c829f532 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts | |||
@@ -5,6 +5,7 @@ import * as chai from 'chai' | |||
5 | import { basename, join } from 'path' | 5 | import { basename, join } from 'path' |
6 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' | 6 | import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' |
7 | import { | 7 | import { |
8 | areObjectStorageTestsDisabled, | ||
8 | checkDirectoryIsEmpty, | 9 | checkDirectoryIsEmpty, |
9 | checkResolutionsInMasterPlaylist, | 10 | checkResolutionsInMasterPlaylist, |
10 | checkSegmentHash, | 11 | checkSegmentHash, |
@@ -12,7 +13,9 @@ import { | |||
12 | cleanupTests, | 13 | cleanupTests, |
13 | createMultipleServers, | 14 | createMultipleServers, |
14 | doubleFollow, | 15 | doubleFollow, |
16 | expectStartWith, | ||
15 | makeRawRequest, | 17 | makeRawRequest, |
18 | ObjectStorageCommand, | ||
16 | PeerTubeServer, | 19 | PeerTubeServer, |
17 | setAccessTokensToServers, | 20 | setAccessTokensToServers, |
18 | waitJobs, | 21 | waitJobs, |
@@ -23,8 +26,19 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' | |||
23 | 26 | ||
24 | const expect = chai.expect | 27 | const expect = chai.expect |
25 | 28 | ||
26 | async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { | 29 | async function checkHlsPlaylist (options: { |
27 | for (const server of servers) { | 30 | servers: PeerTubeServer[] |
31 | videoUUID: string | ||
32 | hlsOnly: boolean | ||
33 | |||
34 | resolutions?: number[] | ||
35 | objectStorageBaseUrl: string | ||
36 | }) { | ||
37 | const { videoUUID, hlsOnly, objectStorageBaseUrl } = options | ||
38 | |||
39 | const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] | ||
40 | |||
41 | for (const server of options.servers) { | ||
28 | const videoDetails = await server.videos.get({ id: videoUUID }) | 42 | const videoDetails = await server.videos.get({ id: videoUUID }) |
29 | const baseUrl = `http://${videoDetails.account.host}` | 43 | const baseUrl = `http://${videoDetails.account.host}` |
30 | 44 | ||
@@ -48,9 +62,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
48 | expect(file.torrentUrl).to.match( | 62 | expect(file.torrentUrl).to.match( |
49 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) | 63 | new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) |
50 | ) | 64 | ) |
51 | expect(file.fileUrl).to.match( | 65 | |
52 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | 66 | if (objectStorageBaseUrl) { |
53 | ) | 67 | expectStartWith(file.fileUrl, objectStorageBaseUrl) |
68 | } else { | ||
69 | expect(file.fileUrl).to.match( | ||
70 | new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) | ||
71 | ) | ||
72 | } | ||
73 | |||
54 | expect(file.resolution.label).to.equal(resolution + 'p') | 74 | expect(file.resolution.label).to.equal(resolution + 'p') |
55 | 75 | ||
56 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) | 76 | await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) |
@@ -80,9 +100,11 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
80 | const file = hlsFiles.find(f => f.resolution.id === resolution) | 100 | const file = hlsFiles.find(f => f.resolution.id === resolution) |
81 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' | 101 | const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' |
82 | 102 | ||
83 | const subPlaylist = await server.streamingPlaylists.get({ | 103 | const url = objectStorageBaseUrl |
84 | url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` | 104 | ? `${objectStorageBaseUrl}hls_${videoUUID}/${playlistName}` |
85 | }) | 105 | : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` |
106 | |||
107 | const subPlaylist = await server.streamingPlaylists.get({ url }) | ||
86 | 108 | ||
87 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) | 109 | expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) |
88 | expect(subPlaylist).to.contain(basename(file.fileUrl)) | 110 | expect(subPlaylist).to.contain(basename(file.fileUrl)) |
@@ -90,14 +112,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h | |||
90 | } | 112 | } |
91 | 113 | ||
92 | { | 114 | { |
93 | const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' | 115 | const baseUrlAndPath = objectStorageBaseUrl |
116 | ? objectStorageBaseUrl + 'hls_' + videoUUID | ||
117 | : baseUrl + '/static/streaming-playlists/hls/' + videoUUID | ||
94 | 118 | ||
95 | for (const resolution of resolutions) { | 119 | for (const resolution of resolutions) { |
96 | await checkSegmentHash({ | 120 | await checkSegmentHash({ |
97 | server, | 121 | server, |
98 | baseUrlPlaylist: baseUrlAndPath, | 122 | baseUrlPlaylist: baseUrlAndPath, |
99 | baseUrlSegment: baseUrlAndPath, | 123 | baseUrlSegment: baseUrlAndPath, |
100 | videoUUID, | ||
101 | resolution, | 124 | resolution, |
102 | hlsPlaylist | 125 | hlsPlaylist |
103 | }) | 126 | }) |
@@ -111,7 +134,7 @@ describe('Test HLS videos', function () { | |||
111 | let videoUUID = '' | 134 | let videoUUID = '' |
112 | let videoAudioUUID = '' | 135 | let videoAudioUUID = '' |
113 | 136 | ||
114 | function runTestSuite (hlsOnly: boolean) { | 137 | function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { |
115 | 138 | ||
116 | it('Should upload a video and transcode it to HLS', async function () { | 139 | it('Should upload a video and transcode it to HLS', async function () { |
117 | this.timeout(120000) | 140 | this.timeout(120000) |
@@ -121,7 +144,7 @@ describe('Test HLS videos', function () { | |||
121 | 144 | ||
122 | await waitJobs(servers) | 145 | await waitJobs(servers) |
123 | 146 | ||
124 | await checkHlsPlaylist(servers, videoUUID, hlsOnly) | 147 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) |
125 | }) | 148 | }) |
126 | 149 | ||
127 | it('Should upload an audio file and transcode it to HLS', async function () { | 150 | it('Should upload an audio file and transcode it to HLS', async function () { |
@@ -132,7 +155,13 @@ describe('Test HLS videos', function () { | |||
132 | 155 | ||
133 | await waitJobs(servers) | 156 | await waitJobs(servers) |
134 | 157 | ||
135 | await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ]) | 158 | await checkHlsPlaylist({ |
159 | servers, | ||
160 | videoUUID: videoAudioUUID, | ||
161 | hlsOnly, | ||
162 | resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], | ||
163 | objectStorageBaseUrl | ||
164 | }) | ||
136 | }) | 165 | }) |
137 | 166 | ||
138 | it('Should update the video', async function () { | 167 | it('Should update the video', async function () { |
@@ -142,7 +171,7 @@ describe('Test HLS videos', function () { | |||
142 | 171 | ||
143 | await waitJobs(servers) | 172 | await waitJobs(servers) |
144 | 173 | ||
145 | await checkHlsPlaylist(servers, videoUUID, hlsOnly) | 174 | await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl }) |
146 | }) | 175 | }) |
147 | 176 | ||
148 | it('Should delete videos', async function () { | 177 | it('Should delete videos', async function () { |
@@ -229,6 +258,22 @@ describe('Test HLS videos', function () { | |||
229 | runTestSuite(true) | 258 | runTestSuite(true) |
230 | }) | 259 | }) |
231 | 260 | ||
261 | describe('With object storage enabled', function () { | ||
262 | if (areObjectStorageTestsDisabled()) return | ||
263 | |||
264 | before(async function () { | ||
265 | this.timeout(120000) | ||
266 | |||
267 | const configOverride = ObjectStorageCommand.getDefaultConfig() | ||
268 | await ObjectStorageCommand.prepareDefaultBuckets() | ||
269 | |||
270 | await servers[0].kill() | ||
271 | await servers[0].run(configOverride) | ||
272 | }) | ||
273 | |||
274 | runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) | ||
275 | }) | ||
276 | |||
232 | after(async function () { | 277 | after(async function () { |
233 | await cleanupTests(servers) | 278 | await cleanupTests(servers) |
234 | }) | 279 | }) |
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index bddcff5e7..9f1b57a2e 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts | |||
@@ -2,8 +2,19 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' | 5 | import { |
6 | import { VideoFile } from '@shared/models' | 6 | areObjectStorageTestsDisabled, |
7 | cleanupTests, | ||
8 | createMultipleServers, | ||
9 | doubleFollow, | ||
10 | expectStartWith, | ||
11 | makeRawRequest, | ||
12 | ObjectStorageCommand, | ||
13 | PeerTubeServer, | ||
14 | setAccessTokensToServers, | ||
15 | waitJobs | ||
16 | } from '@shared/extra-utils' | ||
17 | import { HttpStatusCode, VideoDetails, VideoFile } from '@shared/models' | ||
7 | 18 | ||
8 | const expect = chai.expect | 19 | const expect = chai.expect |
9 | 20 | ||
@@ -17,22 +28,35 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s | |||
17 | if (size) expect(video.size).to.equal(size) | 28 | if (size) expect(video.size).to.equal(size) |
18 | } | 29 | } |
19 | 30 | ||
20 | describe('Test create import video jobs', function () { | 31 | async function checkFiles (video: VideoDetails, objectStorage: boolean) { |
21 | this.timeout(60000) | 32 | for (const file of video.files) { |
33 | if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | ||
22 | 34 | ||
23 | let servers: PeerTubeServer[] = [] | 35 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
36 | } | ||
37 | } | ||
38 | |||
39 | function runTests (objectStorage: boolean) { | ||
24 | let video1UUID: string | 40 | let video1UUID: string |
25 | let video2UUID: string | 41 | let video2UUID: string |
26 | 42 | ||
43 | let servers: PeerTubeServer[] = [] | ||
44 | |||
27 | before(async function () { | 45 | before(async function () { |
28 | this.timeout(90000) | 46 | this.timeout(90000) |
29 | 47 | ||
48 | const config = objectStorage | ||
49 | ? ObjectStorageCommand.getDefaultConfig() | ||
50 | : {} | ||
51 | |||
30 | // Run server 2 to have transcoding enabled | 52 | // Run server 2 to have transcoding enabled |
31 | servers = await createMultipleServers(2) | 53 | servers = await createMultipleServers(2, config) |
32 | await setAccessTokensToServers(servers) | 54 | await setAccessTokensToServers(servers) |
33 | 55 | ||
34 | await doubleFollow(servers[0], servers[1]) | 56 | await doubleFollow(servers[0], servers[1]) |
35 | 57 | ||
58 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | ||
59 | |||
36 | // Upload two videos for our needs | 60 | // Upload two videos for our needs |
37 | { | 61 | { |
38 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) | 62 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) |
@@ -44,7 +68,6 @@ describe('Test create import video jobs', function () { | |||
44 | video2UUID = uuid | 68 | video2UUID = uuid |
45 | } | 69 | } |
46 | 70 | ||
47 | // Transcoding | ||
48 | await waitJobs(servers) | 71 | await waitJobs(servers) |
49 | }) | 72 | }) |
50 | 73 | ||
@@ -65,6 +88,8 @@ describe('Test create import video jobs', function () { | |||
65 | const [ originalVideo, transcodedVideo ] = videoDetails.files | 88 | const [ originalVideo, transcodedVideo ] = videoDetails.files |
66 | assertVideoProperties(originalVideo, 720, 'webm', 218910) | 89 | assertVideoProperties(originalVideo, 720, 'webm', 218910) |
67 | assertVideoProperties(transcodedVideo, 480, 'webm', 69217) | 90 | assertVideoProperties(transcodedVideo, 480, 'webm', 69217) |
91 | |||
92 | await checkFiles(videoDetails, objectStorage) | ||
68 | } | 93 | } |
69 | }) | 94 | }) |
70 | 95 | ||
@@ -87,6 +112,8 @@ describe('Test create import video jobs', function () { | |||
87 | assertVideoProperties(transcodedVideo420, 480, 'mp4') | 112 | assertVideoProperties(transcodedVideo420, 480, 'mp4') |
88 | assertVideoProperties(transcodedVideo320, 360, 'mp4') | 113 | assertVideoProperties(transcodedVideo320, 360, 'mp4') |
89 | assertVideoProperties(transcodedVideo240, 240, 'mp4') | 114 | assertVideoProperties(transcodedVideo240, 240, 'mp4') |
115 | |||
116 | await checkFiles(videoDetails, objectStorage) | ||
90 | } | 117 | } |
91 | }) | 118 | }) |
92 | 119 | ||
@@ -107,10 +134,25 @@ describe('Test create import video jobs', function () { | |||
107 | const [ video720, video480 ] = videoDetails.files | 134 | const [ video720, video480 ] = videoDetails.files |
108 | assertVideoProperties(video720, 720, 'webm', 942961) | 135 | assertVideoProperties(video720, 720, 'webm', 942961) |
109 | assertVideoProperties(video480, 480, 'webm', 69217) | 136 | assertVideoProperties(video480, 480, 'webm', 69217) |
137 | |||
138 | await checkFiles(videoDetails, objectStorage) | ||
110 | } | 139 | } |
111 | }) | 140 | }) |
112 | 141 | ||
113 | after(async function () { | 142 | after(async function () { |
114 | await cleanupTests(servers) | 143 | await cleanupTests(servers) |
115 | }) | 144 | }) |
145 | } | ||
146 | |||
147 | describe('Test create import video jobs', function () { | ||
148 | |||
149 | describe('On filesystem', function () { | ||
150 | runTests(false) | ||
151 | }) | ||
152 | |||
153 | describe('On object storage', function () { | ||
154 | if (areObjectStorageTestsDisabled()) return | ||
155 | |||
156 | runTests(true) | ||
157 | }) | ||
116 | }) | 158 | }) |
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts index df787ccdc..3313a492f 100644 --- a/server/tests/cli/create-transcoding-job.ts +++ b/server/tests/cli/create-transcoding-job.ts | |||
@@ -2,10 +2,15 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { HttpStatusCode, VideoFile } from '@shared/models' | ||
5 | import { | 6 | import { |
7 | areObjectStorageTestsDisabled, | ||
6 | cleanupTests, | 8 | cleanupTests, |
7 | createMultipleServers, | 9 | createMultipleServers, |
8 | doubleFollow, | 10 | doubleFollow, |
11 | expectStartWith, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
9 | PeerTubeServer, | 14 | PeerTubeServer, |
10 | setAccessTokensToServers, | 15 | setAccessTokensToServers, |
11 | waitJobs | 16 | waitJobs |
@@ -13,39 +18,39 @@ import { | |||
13 | 18 | ||
14 | const expect = chai.expect | 19 | const expect = chai.expect |
15 | 20 | ||
16 | describe('Test create transcoding jobs', function () { | 21 | async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { |
17 | let servers: PeerTubeServer[] = [] | 22 | for (const file of files) { |
18 | const videosUUID: string[] = [] | 23 | const shouldStartWith = type === 'webtorrent' |
24 | ? ObjectStorageCommand.getWebTorrentBaseUrl() | ||
25 | : ObjectStorageCommand.getPlaylistBaseUrl() | ||
19 | 26 | ||
20 | const config = { | 27 | expectStartWith(file.fileUrl, shouldStartWith) |
21 | transcoding: { | 28 | |
22 | enabled: false, | 29 | await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) |
23 | resolutions: { | ||
24 | '240p': true, | ||
25 | '360p': true, | ||
26 | '480p': true, | ||
27 | '720p': true, | ||
28 | '1080p': true, | ||
29 | '1440p': true, | ||
30 | '2160p': true | ||
31 | }, | ||
32 | hls: { | ||
33 | enabled: false | ||
34 | } | ||
35 | } | ||
36 | } | 30 | } |
31 | } | ||
32 | |||
33 | function runTests (objectStorage: boolean) { | ||
34 | let servers: PeerTubeServer[] = [] | ||
35 | const videosUUID: string[] = [] | ||
37 | 36 | ||
38 | before(async function () { | 37 | before(async function () { |
39 | this.timeout(60000) | 38 | this.timeout(60000) |
40 | 39 | ||
40 | const config = objectStorage | ||
41 | ? ObjectStorageCommand.getDefaultConfig() | ||
42 | : {} | ||
43 | |||
41 | // Run server 2 to have transcoding enabled | 44 | // Run server 2 to have transcoding enabled |
42 | servers = await createMultipleServers(2) | 45 | servers = await createMultipleServers(2, config) |
43 | await setAccessTokensToServers(servers) | 46 | await setAccessTokensToServers(servers) |
44 | 47 | ||
45 | await servers[0].config.updateCustomSubConfig({ newConfig: config }) | 48 | await servers[0].config.disableTranscoding() |
46 | 49 | ||
47 | await doubleFollow(servers[0], servers[1]) | 50 | await doubleFollow(servers[0], servers[1]) |
48 | 51 | ||
52 | if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() | ||
53 | |||
49 | for (let i = 1; i <= 5; i++) { | 54 | for (let i = 1; i <= 5; i++) { |
50 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) | 55 | const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) |
51 | videosUUID.push(uuid) | 56 | videosUUID.push(uuid) |
@@ -81,27 +86,29 @@ describe('Test create transcoding jobs', function () { | |||
81 | let infoHashes: { [id: number]: string } | 86 | let infoHashes: { [id: number]: string } |
82 | 87 | ||
83 | for (const video of data) { | 88 | for (const video of data) { |
84 | const videoDetail = await server.videos.get({ id: video.uuid }) | 89 | const videoDetails = await server.videos.get({ id: video.uuid }) |
85 | 90 | ||
86 | if (video.uuid === videosUUID[1]) { | 91 | if (video.uuid === videosUUID[1]) { |
87 | expect(videoDetail.files).to.have.lengthOf(4) | 92 | expect(videoDetails.files).to.have.lengthOf(4) |
88 | expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) | 93 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
94 | |||
95 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
89 | 96 | ||
90 | if (!infoHashes) { | 97 | if (!infoHashes) { |
91 | infoHashes = {} | 98 | infoHashes = {} |
92 | 99 | ||
93 | for (const file of videoDetail.files) { | 100 | for (const file of videoDetails.files) { |
94 | infoHashes[file.resolution.id.toString()] = file.magnetUri | 101 | infoHashes[file.resolution.id.toString()] = file.magnetUri |
95 | } | 102 | } |
96 | } else { | 103 | } else { |
97 | for (const resolution of Object.keys(infoHashes)) { | 104 | for (const resolution of Object.keys(infoHashes)) { |
98 | const file = videoDetail.files.find(f => f.resolution.id.toString() === resolution) | 105 | const file = videoDetails.files.find(f => f.resolution.id.toString() === resolution) |
99 | expect(file.magnetUri).to.equal(infoHashes[resolution]) | 106 | expect(file.magnetUri).to.equal(infoHashes[resolution]) |
100 | } | 107 | } |
101 | } | 108 | } |
102 | } else { | 109 | } else { |
103 | expect(videoDetail.files).to.have.lengthOf(1) | 110 | expect(videoDetails.files).to.have.lengthOf(1) |
104 | expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) | 111 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
105 | } | 112 | } |
106 | } | 113 | } |
107 | } | 114 | } |
@@ -125,6 +132,8 @@ describe('Test create transcoding jobs', function () { | |||
125 | expect(videoDetails.files[1].resolution.id).to.equal(480) | 132 | expect(videoDetails.files[1].resolution.id).to.equal(480) |
126 | 133 | ||
127 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) | 134 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) |
135 | |||
136 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
128 | } | 137 | } |
129 | }) | 138 | }) |
130 | 139 | ||
@@ -139,11 +148,15 @@ describe('Test create transcoding jobs', function () { | |||
139 | const videoDetails = await server.videos.get({ id: videosUUID[2] }) | 148 | const videoDetails = await server.videos.get({ id: videosUUID[2] }) |
140 | 149 | ||
141 | expect(videoDetails.files).to.have.lengthOf(1) | 150 | expect(videoDetails.files).to.have.lengthOf(1) |
151 | if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
152 | |||
142 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | 153 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) |
143 | 154 | ||
144 | const files = videoDetails.streamingPlaylists[0].files | 155 | const files = videoDetails.streamingPlaylists[0].files |
145 | expect(files).to.have.lengthOf(1) | 156 | expect(files).to.have.lengthOf(1) |
146 | expect(files[0].resolution.id).to.equal(480) | 157 | expect(files[0].resolution.id).to.equal(480) |
158 | |||
159 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
147 | } | 160 | } |
148 | }) | 161 | }) |
149 | 162 | ||
@@ -160,6 +173,8 @@ describe('Test create transcoding jobs', function () { | |||
160 | const files = videoDetails.streamingPlaylists[0].files | 173 | const files = videoDetails.streamingPlaylists[0].files |
161 | expect(files).to.have.lengthOf(1) | 174 | expect(files).to.have.lengthOf(1) |
162 | expect(files[0].resolution.id).to.equal(480) | 175 | expect(files[0].resolution.id).to.equal(480) |
176 | |||
177 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
163 | } | 178 | } |
164 | }) | 179 | }) |
165 | 180 | ||
@@ -178,15 +193,15 @@ describe('Test create transcoding jobs', function () { | |||
178 | 193 | ||
179 | const files = videoDetails.streamingPlaylists[0].files | 194 | const files = videoDetails.streamingPlaylists[0].files |
180 | expect(files).to.have.lengthOf(4) | 195 | expect(files).to.have.lengthOf(4) |
196 | |||
197 | if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') | ||
181 | } | 198 | } |
182 | }) | 199 | }) |
183 | 200 | ||
184 | it('Should optimize the video file and generate HLS videos if enabled in config', async function () { | 201 | it('Should optimize the video file and generate HLS videos if enabled in config', async function () { |
185 | this.timeout(120000) | 202 | this.timeout(120000) |
186 | 203 | ||
187 | config.transcoding.hls.enabled = true | 204 | await servers[0].config.enableTranscoding() |
188 | await servers[0].config.updateCustomSubConfig({ newConfig: config }) | ||
189 | |||
190 | await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) | 205 | await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) |
191 | 206 | ||
192 | await waitJobs(servers) | 207 | await waitJobs(servers) |
@@ -197,10 +212,28 @@ describe('Test create transcoding jobs', function () { | |||
197 | expect(videoDetails.files).to.have.lengthOf(4) | 212 | expect(videoDetails.files).to.have.lengthOf(4) |
198 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) | 213 | expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) |
199 | expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) | 214 | expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) |
215 | |||
216 | if (objectStorage) { | ||
217 | await checkFilesInObjectStorage(videoDetails.files, 'webtorrent') | ||
218 | await checkFilesInObjectStorage(videoDetails.streamingPlaylists[0].files, 'playlist') | ||
219 | } | ||
200 | } | 220 | } |
201 | }) | 221 | }) |
202 | 222 | ||
203 | after(async function () { | 223 | after(async function () { |
204 | await cleanupTests(servers) | 224 | await cleanupTests(servers) |
205 | }) | 225 | }) |
226 | } | ||
227 | |||
228 | describe('Test create transcoding jobs', function () { | ||
229 | |||
230 | describe('On filesystem', function () { | ||
231 | runTests(false) | ||
232 | }) | ||
233 | |||
234 | describe('On object storage', function () { | ||
235 | if (areObjectStorageTestsDisabled()) return | ||
236 | |||
237 | runTests(true) | ||
238 | }) | ||
206 | }) | 239 | }) |
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts index 7f7873df3..c9a2eb831 100644 --- a/server/tests/helpers/request.ts +++ b/server/tests/helpers/request.ts | |||
@@ -13,7 +13,7 @@ describe('Request helpers', function () { | |||
13 | 13 | ||
14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { | 14 | it('Should throw an error when the bytes limit is exceeded for request', async function () { |
15 | try { | 15 | try { |
16 | await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 3 }) | 16 | await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 }) |
17 | } catch { | 17 | } catch { |
18 | return | 18 | return |
19 | } | 19 | } |
@@ -23,7 +23,7 @@ describe('Request helpers', function () { | |||
23 | 23 | ||
24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { | 24 | it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { |
25 | try { | 25 | try { |
26 | await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath1, { bodyKBLimit: 3 }) | 26 | await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 }) |
27 | } catch { | 27 | } catch { |
28 | 28 | ||
29 | await wait(500) | 29 | await wait(500) |
@@ -35,8 +35,8 @@ describe('Request helpers', function () { | |||
35 | }) | 35 | }) |
36 | 36 | ||
37 | it('Should succeed if the file is below the limit', async function () { | 37 | it('Should succeed if the file is below the limit', async function () { |
38 | await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 5 }) | 38 | await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 }) |
39 | await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath2, { bodyKBLimit: 5 }) | 39 | await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 }) |
40 | 40 | ||
41 | expect(await pathExists(destPath2)).to.be.true | 41 | expect(await pathExists(destPath2)).to.be.true |
42 | }) | 42 | }) |
diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index 7fc92f804..aa2c8e8fa 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts | |||
@@ -16,6 +16,10 @@ function dateIsValid (dateString: string, interval = 300000) { | |||
16 | return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval | 16 | return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval |
17 | } | 17 | } |
18 | 18 | ||
19 | function expectStartWith (str: string, start: string) { | ||
20 | expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true | ||
21 | } | ||
22 | |||
19 | async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { | 23 | async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { |
20 | const res = await makeGetRequest({ | 24 | const res = await makeGetRequest({ |
21 | url, | 25 | url, |
@@ -42,5 +46,6 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f | |||
42 | export { | 46 | export { |
43 | dateIsValid, | 47 | dateIsValid, |
44 | testImage, | 48 | testImage, |
45 | testFileExistsOrNot | 49 | testFileExistsOrNot, |
50 | expectStartWith | ||
46 | } | 51 | } |
diff --git a/shared/extra-utils/miscs/tests.ts b/shared/extra-utils/miscs/tests.ts index 3dfb2487e..dd86041fe 100644 --- a/shared/extra-utils/miscs/tests.ts +++ b/shared/extra-utils/miscs/tests.ts | |||
@@ -28,7 +28,9 @@ const FIXTURE_URLS = { | |||
28 | 28 | ||
29 | badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', | 29 | badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', |
30 | goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', | 30 | goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', |
31 | video4K: 'https://download.cpy.re/peertube/4k_file.txt' | 31 | goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4', |
32 | |||
33 | file4K: 'https://download.cpy.re/peertube/4k_file.txt' | ||
32 | } | 34 | } |
33 | 35 | ||
34 | function parallelTests () { | 36 | function parallelTests () { |
@@ -42,7 +44,15 @@ function isGithubCI () { | |||
42 | function areHttpImportTestsDisabled () { | 44 | function areHttpImportTestsDisabled () { |
43 | const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' | 45 | const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' |
44 | 46 | ||
45 | if (disabled) console.log('Import tests are disabled') | 47 | if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled') |
48 | |||
49 | return disabled | ||
50 | } | ||
51 | |||
52 | function areObjectStorageTestsDisabled () { | ||
53 | const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' | ||
54 | |||
55 | if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') | ||
46 | 56 | ||
47 | return disabled | 57 | return disabled |
48 | } | 58 | } |
@@ -89,6 +99,7 @@ export { | |||
89 | buildAbsoluteFixturePath, | 99 | buildAbsoluteFixturePath, |
90 | getFileSize, | 100 | getFileSize, |
91 | buildRequestStub, | 101 | buildRequestStub, |
102 | areObjectStorageTestsDisabled, | ||
92 | wait, | 103 | wait, |
93 | root | 104 | root |
94 | } | 105 | } |
diff --git a/shared/extra-utils/mock-servers/index.ts b/shared/extra-utils/mock-servers/index.ts index 0ec07f685..93c00c788 100644 --- a/shared/extra-utils/mock-servers/index.ts +++ b/shared/extra-utils/mock-servers/index.ts | |||
@@ -2,3 +2,4 @@ export * from './mock-email' | |||
2 | export * from './mock-instances-index' | 2 | export * from './mock-instances-index' |
3 | export * from './mock-joinpeertube-versions' | 3 | export * from './mock-joinpeertube-versions' |
4 | export * from './mock-plugin-blocklist' | 4 | export * from './mock-plugin-blocklist' |
5 | export * from './mock-object-storage' | ||
diff --git a/shared/extra-utils/mock-servers/mock-object-storage.ts b/shared/extra-utils/mock-servers/mock-object-storage.ts new file mode 100644 index 000000000..19ea7c87c --- /dev/null +++ b/shared/extra-utils/mock-servers/mock-object-storage.ts | |||
@@ -0,0 +1,42 @@ | |||
1 | import * as express from 'express' | ||
2 | import got, { RequestError } from 'got' | ||
3 | import { Server } from 'http' | ||
4 | import { pipeline } from 'stream' | ||
5 | import { randomInt } from '@shared/core-utils' | ||
6 | import { ObjectStorageCommand } from '../server' | ||
7 | |||
8 | export class MockObjectStorage { | ||
9 | private server: Server | ||
10 | |||
11 | initialize () { | ||
12 | return new Promise<number>(res => { | ||
13 | const app = express() | ||
14 | |||
15 | app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
16 | const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` | ||
17 | |||
18 | if (process.env.DEBUG) { | ||
19 | console.log('Receiving request on mocked server %s.', req.url) | ||
20 | console.log('Proxifying request to %s', url) | ||
21 | } | ||
22 | |||
23 | return pipeline( | ||
24 | got.stream(url, { throwHttpErrors: false }), | ||
25 | res, | ||
26 | (err: RequestError) => { | ||
27 | if (!err) return | ||
28 | |||
29 | console.error('Pipeline failed.', err) | ||
30 | } | ||
31 | ) | ||
32 | }) | ||
33 | |||
34 | const port = 42301 + randomInt(1, 100) | ||
35 | this.server = app.listen(port, () => res(port)) | ||
36 | }) | ||
37 | } | ||
38 | |||
39 | terminate () { | ||
40 | if (this.server) this.server.close() | ||
41 | } | ||
42 | } | ||
diff --git a/shared/extra-utils/requests/requests.ts b/shared/extra-utils/requests/requests.ts index 70f790222..e3ecd1af2 100644 --- a/shared/extra-utils/requests/requests.ts +++ b/shared/extra-utils/requests/requests.ts | |||
@@ -121,6 +121,20 @@ function unwrapText (test: request.Test): Promise<string> { | |||
121 | return test.then(res => res.text) | 121 | return test.then(res => res.text) |
122 | } | 122 | } |
123 | 123 | ||
124 | function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> { | ||
125 | return test.then(res => { | ||
126 | if (res.body instanceof Buffer) { | ||
127 | return JSON.parse(new TextDecoder().decode(res.body)) | ||
128 | } | ||
129 | |||
130 | return res.body | ||
131 | }) | ||
132 | } | ||
133 | |||
134 | function unwrapTextOrDecode (test: request.Test): Promise<string> { | ||
135 | return test.then(res => res.text || new TextDecoder().decode(res.body)) | ||
136 | } | ||
137 | |||
124 | // --------------------------------------------------------------------------- | 138 | // --------------------------------------------------------------------------- |
125 | 139 | ||
126 | export { | 140 | export { |
@@ -134,6 +148,8 @@ export { | |||
134 | makeRawRequest, | 148 | makeRawRequest, |
135 | makeActivityPubGetRequest, | 149 | makeActivityPubGetRequest, |
136 | unwrapBody, | 150 | unwrapBody, |
151 | unwrapTextOrDecode, | ||
152 | unwrapBodyOrDecodeToJSON, | ||
137 | unwrapText | 153 | unwrapText |
138 | } | 154 | } |
139 | 155 | ||
diff --git a/shared/extra-utils/server/config-command.ts b/shared/extra-utils/server/config-command.ts index 11148aa46..51d04fa63 100644 --- a/shared/extra-utils/server/config-command.ts +++ b/shared/extra-utils/server/config-command.ts | |||
@@ -18,6 +18,70 @@ export class ConfigCommand extends AbstractCommand { | |||
18 | } | 18 | } |
19 | } | 19 | } |
20 | 20 | ||
21 | enableImports () { | ||
22 | return this.updateExistingSubConfig({ | ||
23 | newConfig: { | ||
24 | import: { | ||
25 | videos: { | ||
26 | http: { | ||
27 | enabled: true | ||
28 | }, | ||
29 | |||
30 | torrent: { | ||
31 | enabled: true | ||
32 | } | ||
33 | } | ||
34 | } | ||
35 | } | ||
36 | }) | ||
37 | } | ||
38 | |||
39 | enableLive (options: { | ||
40 | allowReplay?: boolean | ||
41 | transcoding?: boolean | ||
42 | } = {}) { | ||
43 | return this.updateExistingSubConfig({ | ||
44 | newConfig: { | ||
45 | live: { | ||
46 | enabled: true, | ||
47 | allowReplay: options.allowReplay ?? true, | ||
48 | transcoding: { | ||
49 | enabled: options.transcoding ?? true, | ||
50 | resolutions: ConfigCommand.getCustomConfigResolutions(true) | ||
51 | } | ||
52 | } | ||
53 | } | ||
54 | }) | ||
55 | } | ||
56 | |||
57 | disableTranscoding () { | ||
58 | return this.updateExistingSubConfig({ | ||
59 | newConfig: { | ||
60 | transcoding: { | ||
61 | enabled: false | ||
62 | } | ||
63 | } | ||
64 | }) | ||
65 | } | ||
66 | |||
67 | enableTranscoding (webtorrent = true, hls = true) { | ||
68 | return this.updateExistingSubConfig({ | ||
69 | newConfig: { | ||
70 | transcoding: { | ||
71 | enabled: true, | ||
72 | resolutions: ConfigCommand.getCustomConfigResolutions(true), | ||
73 | |||
74 | webtorrent: { | ||
75 | enabled: webtorrent | ||
76 | }, | ||
77 | hls: { | ||
78 | enabled: hls | ||
79 | } | ||
80 | } | ||
81 | } | ||
82 | }) | ||
83 | } | ||
84 | |||
21 | getConfig (options: OverrideCommandOptions = {}) { | 85 | getConfig (options: OverrideCommandOptions = {}) { |
22 | const path = '/api/v1/config' | 86 | const path = '/api/v1/config' |
23 | 87 | ||
@@ -81,6 +145,14 @@ export class ConfigCommand extends AbstractCommand { | |||
81 | }) | 145 | }) |
82 | } | 146 | } |
83 | 147 | ||
148 | async updateExistingSubConfig (options: OverrideCommandOptions & { | ||
149 | newConfig: DeepPartial<CustomConfig> | ||
150 | }) { | ||
151 | const existing = await this.getCustomConfig(options) | ||
152 | |||
153 | return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) | ||
154 | } | ||
155 | |||
84 | updateCustomSubConfig (options: OverrideCommandOptions & { | 156 | updateCustomSubConfig (options: OverrideCommandOptions & { |
85 | newConfig: DeepPartial<CustomConfig> | 157 | newConfig: DeepPartial<CustomConfig> |
86 | }) { | 158 | }) { |
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts index 9055dfc57..92ff7a0f9 100644 --- a/shared/extra-utils/server/index.ts +++ b/shared/extra-utils/server/index.ts | |||
@@ -6,6 +6,7 @@ export * from './follows-command' | |||
6 | export * from './follows' | 6 | export * from './follows' |
7 | export * from './jobs' | 7 | export * from './jobs' |
8 | export * from './jobs-command' | 8 | export * from './jobs-command' |
9 | export * from './object-storage-command' | ||
9 | export * from './plugins-command' | 10 | export * from './plugins-command' |
10 | export * from './plugins' | 11 | export * from './plugins' |
11 | export * from './redundancy-command' | 12 | export * from './redundancy-command' |
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts index c4eb12dc2..91771c176 100644 --- a/shared/extra-utils/server/jobs-command.ts +++ b/shared/extra-utils/server/jobs-command.ts | |||
@@ -5,6 +5,16 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared' | |||
5 | 5 | ||
6 | export class JobsCommand extends AbstractCommand { | 6 | export class JobsCommand extends AbstractCommand { |
7 | 7 | ||
8 | async getLatest (options: OverrideCommandOptions & { | ||
9 | jobType: JobType | ||
10 | }) { | ||
11 | const { data } = await this.getJobsList({ ...options, start: 0, count: 1, sort: '-createdAt' }) | ||
12 | |||
13 | if (data.length === 0) return undefined | ||
14 | |||
15 | return data[0] | ||
16 | } | ||
17 | |||
8 | getJobsList (options: OverrideCommandOptions & { | 18 | getJobsList (options: OverrideCommandOptions & { |
9 | state?: JobState | 19 | state?: JobState |
10 | jobType?: JobType | 20 | jobType?: JobType |
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index 64a0353eb..27104bfdf 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts | |||
@@ -3,7 +3,7 @@ import { JobState } from '../../models' | |||
3 | import { wait } from '../miscs' | 3 | import { wait } from '../miscs' |
4 | import { PeerTubeServer } from './server' | 4 | import { PeerTubeServer } from './server' |
5 | 5 | ||
6 | async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { | 6 | async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer, skipDelayed = false) { |
7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT | 7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT |
8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) | 8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) |
9 | : 250 | 9 | : 250 |
@@ -13,7 +13,9 @@ async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { | |||
13 | if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] | 13 | if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] |
14 | else servers = serversArg as PeerTubeServer[] | 14 | else servers = serversArg as PeerTubeServer[] |
15 | 15 | ||
16 | const states: JobState[] = [ 'waiting', 'active', 'delayed' ] | 16 | const states: JobState[] = [ 'waiting', 'active' ] |
17 | if (!skipDelayed) states.push('delayed') | ||
18 | |||
17 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] | 19 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] |
18 | let pendingRequests: boolean | 20 | let pendingRequests: boolean |
19 | 21 | ||
diff --git a/shared/extra-utils/server/object-storage-command.ts b/shared/extra-utils/server/object-storage-command.ts new file mode 100644 index 000000000..b4de8f4cb --- /dev/null +++ b/shared/extra-utils/server/object-storage-command.ts | |||
@@ -0,0 +1,77 @@ | |||
1 | |||
2 | import { HttpStatusCode } from '@shared/models' | ||
3 | import { makePostBodyRequest } from '../requests' | ||
4 | import { AbstractCommand } from '../shared' | ||
5 | |||
6 | export class ObjectStorageCommand extends AbstractCommand { | ||
7 | static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists' | ||
8 | static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos' | ||
9 | |||
10 | static getDefaultConfig () { | ||
11 | return { | ||
12 | object_storage: { | ||
13 | enabled: true, | ||
14 | endpoint: 'http://' + this.getEndpointHost(), | ||
15 | region: this.getRegion(), | ||
16 | |||
17 | credentials: this.getCredentialsConfig(), | ||
18 | |||
19 | streaming_playlists: { | ||
20 | bucket_name: this.DEFAULT_PLAYLIST_BUCKET | ||
21 | }, | ||
22 | |||
23 | videos: { | ||
24 | bucket_name: this.DEFAULT_WEBTORRENT_BUCKET | ||
25 | } | ||
26 | } | ||
27 | } | ||
28 | } | ||
29 | |||
30 | static getCredentialsConfig () { | ||
31 | return { | ||
32 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
33 | secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
34 | } | ||
35 | } | ||
36 | |||
37 | static getEndpointHost () { | ||
38 | return 'localhost:9444' | ||
39 | } | ||
40 | |||
41 | static getRegion () { | ||
42 | return 'us-east-1' | ||
43 | } | ||
44 | |||
45 | static getWebTorrentBaseUrl () { | ||
46 | return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/` | ||
47 | } | ||
48 | |||
49 | static getPlaylistBaseUrl () { | ||
50 | return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/` | ||
51 | } | ||
52 | |||
53 | static async prepareDefaultBuckets () { | ||
54 | await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET) | ||
55 | await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET) | ||
56 | } | ||
57 | |||
58 | static async createBucket (name: string) { | ||
59 | await makePostBodyRequest({ | ||
60 | url: this.getEndpointHost(), | ||
61 | path: '/ui/' + name + '?delete', | ||
62 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
63 | }) | ||
64 | |||
65 | await makePostBodyRequest({ | ||
66 | url: this.getEndpointHost(), | ||
67 | path: '/ui/' + name + '?create', | ||
68 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
69 | }) | ||
70 | |||
71 | await makePostBodyRequest({ | ||
72 | url: this.getEndpointHost(), | ||
73 | path: '/ui/' + name + '?make-public', | ||
74 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
75 | }) | ||
76 | } | ||
77 | } | ||
diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 3c335b8e4..bc5e1cd5f 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts | |||
@@ -38,11 +38,13 @@ import { PluginsCommand } from './plugins-command' | |||
38 | import { RedundancyCommand } from './redundancy-command' | 38 | import { RedundancyCommand } from './redundancy-command' |
39 | import { ServersCommand } from './servers-command' | 39 | import { ServersCommand } from './servers-command' |
40 | import { StatsCommand } from './stats-command' | 40 | import { StatsCommand } from './stats-command' |
41 | import { ObjectStorageCommand } from './object-storage-command' | ||
41 | 42 | ||
42 | export type RunServerOptions = { | 43 | export type RunServerOptions = { |
43 | hideLogs?: boolean | 44 | hideLogs?: boolean |
44 | nodeArgs?: string[] | 45 | nodeArgs?: string[] |
45 | peertubeArgs?: string[] | 46 | peertubeArgs?: string[] |
47 | env?: { [ id: string ]: string } | ||
46 | } | 48 | } |
47 | 49 | ||
48 | export class PeerTubeServer { | 50 | export class PeerTubeServer { |
@@ -121,6 +123,7 @@ export class PeerTubeServer { | |||
121 | servers?: ServersCommand | 123 | servers?: ServersCommand |
122 | login?: LoginCommand | 124 | login?: LoginCommand |
123 | users?: UsersCommand | 125 | users?: UsersCommand |
126 | objectStorage?: ObjectStorageCommand | ||
124 | videos?: VideosCommand | 127 | videos?: VideosCommand |
125 | 128 | ||
126 | constructor (options: { serverNumber: number } | { url: string }) { | 129 | constructor (options: { serverNumber: number } | { url: string }) { |
@@ -202,6 +205,10 @@ export class PeerTubeServer { | |||
202 | env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() | 205 | env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() |
203 | env['NODE_CONFIG'] = JSON.stringify(configOverride) | 206 | env['NODE_CONFIG'] = JSON.stringify(configOverride) |
204 | 207 | ||
208 | if (options.env) { | ||
209 | Object.assign(env, options.env) | ||
210 | } | ||
211 | |||
205 | const forkOptions = { | 212 | const forkOptions = { |
206 | silent: true, | 213 | silent: true, |
207 | env, | 214 | env, |
@@ -209,10 +216,17 @@ export class PeerTubeServer { | |||
209 | execArgv: options.nodeArgs || [] | 216 | execArgv: options.nodeArgs || [] |
210 | } | 217 | } |
211 | 218 | ||
212 | return new Promise<void>(res => { | 219 | return new Promise<void>((res, rej) => { |
213 | const self = this | 220 | const self = this |
214 | 221 | ||
215 | this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) | 222 | this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) |
223 | |||
224 | const onExit = function () { | ||
225 | return rej(new Error('Process exited')) | ||
226 | } | ||
227 | |||
228 | this.app.on('exit', onExit) | ||
229 | |||
216 | this.app.stdout.on('data', function onStdout (data) { | 230 | this.app.stdout.on('data', function onStdout (data) { |
217 | let dontContinue = false | 231 | let dontContinue = false |
218 | 232 | ||
@@ -241,6 +255,7 @@ export class PeerTubeServer { | |||
241 | console.log(data.toString()) | 255 | console.log(data.toString()) |
242 | } else { | 256 | } else { |
243 | self.app.stdout.removeListener('data', onStdout) | 257 | self.app.stdout.removeListener('data', onStdout) |
258 | self.app.removeListener('exit', onExit) | ||
244 | } | 259 | } |
245 | 260 | ||
246 | process.on('exit', () => { | 261 | process.on('exit', () => { |
@@ -365,5 +380,6 @@ export class PeerTubeServer { | |||
365 | this.login = new LoginCommand(this) | 380 | this.login = new LoginCommand(this) |
366 | this.users = new UsersCommand(this) | 381 | this.users = new UsersCommand(this) |
367 | this.videos = new VideosCommand(this) | 382 | this.videos = new VideosCommand(this) |
383 | this.objectStorage = new ObjectStorageCommand(this) | ||
368 | } | 384 | } |
369 | } | 385 | } |
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index f0622feb0..21ab9405b 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts | |||
@@ -10,11 +10,11 @@ async function createSingleServer (serverNumber: number, configOverride?: Object | |||
10 | return server | 10 | return server |
11 | } | 11 | } |
12 | 12 | ||
13 | function createMultipleServers (totalServers: number, configOverride?: Object) { | 13 | function createMultipleServers (totalServers: number, configOverride?: Object, options: RunServerOptions = {}) { |
14 | const serverPromises: Promise<PeerTubeServer>[] = [] | 14 | const serverPromises: Promise<PeerTubeServer>[] = [] |
15 | 15 | ||
16 | for (let i = 1; i <= totalServers; i++) { | 16 | for (let i = 1; i <= totalServers; i++) { |
17 | serverPromises.push(createSingleServer(i, configOverride)) | 17 | serverPromises.push(createSingleServer(i, configOverride, options)) |
18 | } | 18 | } |
19 | 19 | ||
20 | return Promise.all(serverPromises) | 20 | return Promise.all(serverPromises) |
diff --git a/shared/extra-utils/videos/live-command.ts b/shared/extra-utils/videos/live-command.ts index 81ae458e0..74f5d3089 100644 --- a/shared/extra-utils/videos/live-command.ts +++ b/shared/extra-utils/videos/live-command.ts | |||
@@ -126,7 +126,7 @@ export class LiveCommand extends AbstractCommand { | |||
126 | video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) | 126 | video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) |
127 | 127 | ||
128 | await wait(500) | 128 | await wait(500) |
129 | } while (video.isLive === true && video.state.id !== VideoState.PUBLISHED) | 129 | } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED) |
130 | } | 130 | } |
131 | 131 | ||
132 | async countPlaylists (options: OverrideCommandOptions & { | 132 | async countPlaylists (options: OverrideCommandOptions & { |
diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts index 9a6df07a8..29f99ed6d 100644 --- a/shared/extra-utils/videos/live.ts +++ b/shared/extra-utils/videos/live.ts | |||
@@ -89,6 +89,12 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi | |||
89 | } | 89 | } |
90 | } | 90 | } |
91 | 91 | ||
92 | async function waitUntilLiveSavedOnAllServers (servers: PeerTubeServer[], videoId: string) { | ||
93 | for (const server of servers) { | ||
94 | await server.live.waitUntilSaved({ videoId }) | ||
95 | } | ||
96 | } | ||
97 | |||
92 | async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { | 98 | async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { |
93 | const basePath = server.servers.buildDirectory('streaming-playlists') | 99 | const basePath = server.servers.buildDirectory('streaming-playlists') |
94 | const hlsPath = join(basePath, 'hls', videoUUID) | 100 | const hlsPath = join(basePath, 'hls', videoUUID) |
@@ -126,5 +132,6 @@ export { | |||
126 | testFfmpegStreamError, | 132 | testFfmpegStreamError, |
127 | stopFfmpeg, | 133 | stopFfmpeg, |
128 | waitUntilLivePublishedOnAllServers, | 134 | waitUntilLivePublishedOnAllServers, |
135 | waitUntilLiveSavedOnAllServers, | ||
129 | checkLiveCleanupAfterSave | 136 | checkLiveCleanupAfterSave |
130 | } | 137 | } |
diff --git a/shared/extra-utils/videos/streaming-playlists-command.ts b/shared/extra-utils/videos/streaming-playlists-command.ts index 9662685da..5d40d35cb 100644 --- a/shared/extra-utils/videos/streaming-playlists-command.ts +++ b/shared/extra-utils/videos/streaming-playlists-command.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { HttpStatusCode } from '@shared/models' | 1 | import { HttpStatusCode } from '@shared/models' |
2 | import { unwrapBody, unwrapText } from '../requests' | 2 | import { unwrapBody, unwrapTextOrDecode, unwrapBodyOrDecodeToJSON } from '../requests' |
3 | import { AbstractCommand, OverrideCommandOptions } from '../shared' | 3 | import { AbstractCommand, OverrideCommandOptions } from '../shared' |
4 | 4 | ||
5 | export class StreamingPlaylistsCommand extends AbstractCommand { | 5 | export class StreamingPlaylistsCommand extends AbstractCommand { |
@@ -7,7 +7,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand { | |||
7 | get (options: OverrideCommandOptions & { | 7 | get (options: OverrideCommandOptions & { |
8 | url: string | 8 | url: string |
9 | }) { | 9 | }) { |
10 | return unwrapText(this.getRawRequest({ | 10 | return unwrapTextOrDecode(this.getRawRequest({ |
11 | ...options, | 11 | ...options, |
12 | 12 | ||
13 | url: options.url, | 13 | url: options.url, |
@@ -33,7 +33,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand { | |||
33 | getSegmentSha256 (options: OverrideCommandOptions & { | 33 | getSegmentSha256 (options: OverrideCommandOptions & { |
34 | url: string | 34 | url: string |
35 | }) { | 35 | }) { |
36 | return unwrapBody<{ [ id: string ]: string }>(this.getRawRequest({ | 36 | return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ |
37 | ...options, | 37 | ...options, |
38 | 38 | ||
39 | url: options.url, | 39 | url: options.url, |
diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index a224b8f5f..6671e3fa6 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts | |||
@@ -9,17 +9,16 @@ async function checkSegmentHash (options: { | |||
9 | server: PeerTubeServer | 9 | server: PeerTubeServer |
10 | baseUrlPlaylist: string | 10 | baseUrlPlaylist: string |
11 | baseUrlSegment: string | 11 | baseUrlSegment: string |
12 | videoUUID: string | ||
13 | resolution: number | 12 | resolution: number |
14 | hlsPlaylist: VideoStreamingPlaylist | 13 | hlsPlaylist: VideoStreamingPlaylist |
15 | }) { | 14 | }) { |
16 | const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options | 15 | const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist } = options |
17 | const command = server.streamingPlaylists | 16 | const command = server.streamingPlaylists |
18 | 17 | ||
19 | const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) | 18 | const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) |
20 | const videoName = basename(file.fileUrl) | 19 | const videoName = basename(file.fileUrl) |
21 | 20 | ||
22 | const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) | 21 | const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) |
23 | 22 | ||
24 | const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) | 23 | const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) |
25 | 24 | ||
@@ -28,7 +27,7 @@ async function checkSegmentHash (options: { | |||
28 | const range = `${offset}-${offset + length - 1}` | 27 | const range = `${offset}-${offset + length - 1}` |
29 | 28 | ||
30 | const segmentBody = await command.getSegment({ | 29 | const segmentBody = await command.getSegment({ |
31 | url: `${baseUrlSegment}/${videoUUID}/${videoName}`, | 30 | url: `${baseUrlSegment}/${videoName}`, |
32 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, | 31 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, |
33 | range: `bytes=${range}` | 32 | range: `bytes=${range}` |
34 | }) | 33 | }) |
diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 33725bfdc..d35339c8d 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts | |||
@@ -188,6 +188,17 @@ export class VideosCommand extends AbstractCommand { | |||
188 | return id | 188 | return id |
189 | } | 189 | } |
190 | 190 | ||
191 | async listFiles (options: OverrideCommandOptions & { | ||
192 | id: number | string | ||
193 | }) { | ||
194 | const video = await this.get(options) | ||
195 | |||
196 | const files = video.files || [] | ||
197 | const hlsFiles = video.streamingPlaylists[0]?.files || [] | ||
198 | |||
199 | return files.concat(hlsFiles) | ||
200 | } | ||
201 | |||
191 | // --------------------------------------------------------------------------- | 202 | // --------------------------------------------------------------------------- |
192 | 203 | ||
193 | listMyVideos (options: OverrideCommandOptions & { | 204 | listMyVideos (options: OverrideCommandOptions & { |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 4ab249e0b..ff96283a4 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -19,6 +19,7 @@ export type JobType = | |||
19 | | 'video-redundancy' | 19 | | 'video-redundancy' |
20 | | 'video-live-ending' | 20 | | 'video-live-ending' |
21 | | 'actor-keys' | 21 | | 'actor-keys' |
22 | | 'move-to-object-storage' | ||
22 | 23 | ||
23 | export interface Job { | 24 | export interface Job { |
24 | id: number | 25 | id: number |
@@ -136,3 +137,8 @@ export interface VideoLiveEndingPayload { | |||
136 | export interface ActorKeysPayload { | 137 | export interface ActorKeysPayload { |
137 | actorId: number | 138 | actorId: number |
138 | } | 139 | } |
140 | |||
141 | export interface MoveObjectStoragePayload { | ||
142 | videoUUID: string | ||
143 | isNewVideo: boolean | ||
144 | } | ||
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index faa9b9868..733c433a0 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts | |||
@@ -26,6 +26,7 @@ export * from './video-resolution.enum' | |||
26 | export * from './video-schedule-update.model' | 26 | export * from './video-schedule-update.model' |
27 | export * from './video-sort-field.type' | 27 | export * from './video-sort-field.type' |
28 | export * from './video-state.enum' | 28 | export * from './video-state.enum' |
29 | export * from './video-storage.enum' | ||
29 | 30 | ||
30 | export * from './video-streaming-playlist.model' | 31 | export * from './video-streaming-playlist.model' |
31 | export * from './video-streaming-playlist.type' | 32 | export * from './video-streaming-playlist.type' |
diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts index 49d997f24..c6af481e7 100644 --- a/shared/models/videos/video-state.enum.ts +++ b/shared/models/videos/video-state.enum.ts | |||
@@ -3,5 +3,6 @@ export const enum VideoState { | |||
3 | TO_TRANSCODE = 2, | 3 | TO_TRANSCODE = 2, |
4 | TO_IMPORT = 3, | 4 | TO_IMPORT = 3, |
5 | WAITING_FOR_LIVE = 4, | 5 | WAITING_FOR_LIVE = 4, |
6 | LIVE_ENDED = 5 | 6 | LIVE_ENDED = 5, |
7 | TO_MOVE_TO_EXTERNAL_STORAGE = 6 | ||
7 | } | 8 | } |
diff --git a/shared/models/videos/video-storage.enum.ts b/shared/models/videos/video-storage.enum.ts new file mode 100644 index 000000000..7c6690db2 --- /dev/null +++ b/shared/models/videos/video-storage.enum.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export const enum VideoStorage { | ||
2 | FILE_SYSTEM, | ||
3 | OBJECT_STORAGE, | ||
4 | } | ||
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml index ce0f89d7b..1b474582a 100644 --- a/support/docker/production/config/custom-environment-variables.yaml +++ b/support/docker/production/config/custom-environment-variables.yaml | |||
@@ -45,6 +45,29 @@ smtp: | |||
45 | __format: "json" | 45 | __format: "json" |
46 | from_address: "PEERTUBE_SMTP_FROM" | 46 | from_address: "PEERTUBE_SMTP_FROM" |
47 | 47 | ||
48 | object_storage: | ||
49 | enabled: | ||
50 | __name: "PEERTUBE_OBJECT_STORAGE_ENABLED" | ||
51 | __format: "json" | ||
52 | |||
53 | endpoint: "PEERTUBE_OBJECT_STORAGE_ENDPOINT" | ||
54 | |||
55 | region: "PEERTUBE_OBJECT_STORAGE_REGION" | ||
56 | |||
57 | max_upload_part: | ||
58 | __name: "PEERTUBE_OBJECT_STORAGE_MAX_UPLOAD_PART" | ||
59 | __format: "json" | ||
60 | |||
61 | streaming_playlists: | ||
62 | bucket_name: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BUCKET_NAME" | ||
63 | prefix: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_PREFIX" | ||
64 | base_url: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BASE_URL" | ||
65 | |||
66 | videos: | ||
67 | bucket_name: "PEERTUBE_OBJECT_STORAGE_VIDEOS_BUCKET_NAME" | ||
68 | prefix: "PEERTUBE_OBJECT_STORAGE_VIDEOS_PREFIX" | ||
69 | base_url: "PEERTUBE_OBJECT_STORAGE_VIDEOS_BASE_URL" | ||
70 | |||
48 | log: | 71 | log: |
49 | level: "PEERTUBE_LOG_LEVEL" | 72 | level: "PEERTUBE_LOG_LEVEL" |
50 | log_ping_requests: | 73 | log_ping_requests: |
@@ -49,6 +49,770 @@ | |||
49 | resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" | 49 | resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" |
50 | integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== | 50 | integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== |
51 | 51 | ||
52 | "@aws-crypto/crc32@^1.0.0": | ||
53 | version "1.1.0" | ||
54 | resolved "https://registry.yarnpkg.com/@aws-crypto/crc32/-/crc32-1.1.0.tgz#aff048e207798fad0b0e7765b12d474c273779b6" | ||
55 | integrity sha512-ifvfaaJVvT+JUTi3zSkX4wtuGGVJrAcjN7ftg+JiE/frNBP3zNwo4xipzWBsMLZfNuzMZuaesEYyqkZcs5tzCQ== | ||
56 | dependencies: | ||
57 | tslib "^1.11.1" | ||
58 | |||
59 | "@aws-crypto/ie11-detection@^1.0.0": | ||
60 | version "1.0.0" | ||
61 | resolved "https://registry.yarnpkg.com/@aws-crypto/ie11-detection/-/ie11-detection-1.0.0.tgz#d3a6af29ba7f15458f79c41d1cd8cac3925e726a" | ||
62 | integrity sha512-kCKVhCF1oDxFYgQrxXmIrS5oaWulkvRcPz+QBDMsUr2crbF4VGgGT6+uQhSwJFdUAQ2A//Vq+uT83eJrkzFgXA== | ||
63 | dependencies: | ||
64 | tslib "^1.11.1" | ||
65 | |||
66 | "@aws-crypto/sha256-browser@^1.0.0": | ||
67 | version "1.1.1" | ||
68 | resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-browser/-/sha256-browser-1.1.1.tgz#85dddf13e8f8d74c0d6592d993e4bf401da9f420" | ||
69 | integrity sha512-nS4vdan97It6HcweV58WXtjPbPSc0JXd3sAwlw3Ou5Mc3WllSycAS32Tv2LRn8butNQoU9AE3jEQAOgiMdNC1Q== | ||
70 | dependencies: | ||
71 | "@aws-crypto/ie11-detection" "^1.0.0" | ||
72 | "@aws-crypto/sha256-js" "^1.1.0" | ||
73 | "@aws-crypto/supports-web-crypto" "^1.0.0" | ||
74 | "@aws-sdk/types" "^3.1.0" | ||
75 | "@aws-sdk/util-locate-window" "^3.0.0" | ||
76 | "@aws-sdk/util-utf8-browser" "^3.0.0" | ||
77 | tslib "^1.11.1" | ||
78 | |||
79 | "@aws-crypto/sha256-js@^1.0.0", "@aws-crypto/sha256-js@^1.1.0": | ||
80 | version "1.1.0" | ||
81 | resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-js/-/sha256-js-1.1.0.tgz#a58386ad18186e392e0f1d98d18831261d27b071" | ||
82 | integrity sha512-VIhuqbPgXDVr8sZe2yhgQcDRRmzf4CI8fmC1A3bHiRfE6wlz1d8KpeemqbuoEHotz/Dch9yOxlshyQDNjNFeHA== | ||
83 | dependencies: | ||
84 | "@aws-sdk/types" "^3.1.0" | ||
85 | "@aws-sdk/util-utf8-browser" "^3.0.0" | ||
86 | tslib "^1.11.1" | ||
87 | |||
88 | "@aws-crypto/supports-web-crypto@^1.0.0": | ||
89 | version "1.0.0" | ||
90 | resolved "https://registry.yarnpkg.com/@aws-crypto/supports-web-crypto/-/supports-web-crypto-1.0.0.tgz#c40901bc17ac1e875e248df16a2b47ad8bfd9a93" | ||
91 | integrity sha512-IHLfv+WmVH89EW4n6a5eE8/hUlz6qkWGMn/v4r5ZgzcXdTC5nolii2z3k46y01hWRiC2PPhOdeSLzMUCUMco7g== | ||
92 | dependencies: | ||
93 | tslib "^1.11.1" | ||
94 | |||
95 | "@aws-sdk/abort-controller@3.25.0": | ||
96 | version "3.25.0" | ||
97 | resolved "https://registry.yarnpkg.com/@aws-sdk/abort-controller/-/abort-controller-3.25.0.tgz#a9ea250140de378d8beb6d2f427067fa30423e9e" | ||
98 | integrity sha512-uEVKqKkPVz6atbCxCNJY5O7V+ieSK8crUswXo8/WePyEbGEgxJ4t9x/WG4lV8kBjelmvQHDR4GqfJmb5Sh9xSg== | ||
99 | dependencies: | ||
100 | "@aws-sdk/types" "3.25.0" | ||
101 | tslib "^2.3.0" | ||
102 | |||
103 | "@aws-sdk/chunked-blob-reader-native@3.23.0": | ||
104 | version "3.23.0" | ||
105 | resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.23.0.tgz#72d711e3cc904bb380e99cdd60c59deacd1596ac" | ||
106 | integrity sha512-Ya5f8Ntv0EyZw+AHkpV6n6qqHzpCDNlkX50uj/dwFCMmPiHFWsWMvd0Qu04Y7miycJINEatRrJ5V8r/uVvZIDg== | ||
107 | dependencies: | ||
108 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
109 | tslib "^2.3.0" | ||
110 | |||
111 | "@aws-sdk/chunked-blob-reader@3.23.0": | ||
112 | version "3.23.0" | ||
113 | resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader/-/chunked-blob-reader-3.23.0.tgz#83eb6a437172b671e699850378bcb558e15374ec" | ||
114 | integrity sha512-gmJhCuXrKOOumppviE4K30NvsIQIqqxbGDNptrJrMYBO0qXCbK8/BypZ/hS/oT3loDzlSIxG2z5GDL/va9lbFw== | ||
115 | dependencies: | ||
116 | tslib "^2.3.0" | ||
117 | |||
118 | "@aws-sdk/client-s3@^3.23.0": | ||
119 | version "3.25.0" | ||
120 | resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.25.0.tgz#6b8146c18e76606378c5f736554cb41ad4ce229e" | ||
121 | integrity sha512-yVDPo6x7DCt9t833SjqWI+AQBx81/m54gLF/ePQZBeHL5mPMEyMXTF0o22yUP5t8f92U2VAyRCP2NvKtB9WgBg== | ||
122 | dependencies: | ||
123 | "@aws-crypto/sha256-browser" "^1.0.0" | ||
124 | "@aws-crypto/sha256-js" "^1.0.0" | ||
125 | "@aws-sdk/client-sts" "3.25.0" | ||
126 | "@aws-sdk/config-resolver" "3.25.0" | ||
127 | "@aws-sdk/credential-provider-node" "3.25.0" | ||
128 | "@aws-sdk/eventstream-serde-browser" "3.25.0" | ||
129 | "@aws-sdk/eventstream-serde-config-resolver" "3.25.0" | ||
130 | "@aws-sdk/eventstream-serde-node" "3.25.0" | ||
131 | "@aws-sdk/fetch-http-handler" "3.25.0" | ||
132 | "@aws-sdk/hash-blob-browser" "3.25.0" | ||
133 | "@aws-sdk/hash-node" "3.25.0" | ||
134 | "@aws-sdk/hash-stream-node" "3.25.0" | ||
135 | "@aws-sdk/invalid-dependency" "3.25.0" | ||
136 | "@aws-sdk/md5-js" "3.25.0" | ||
137 | "@aws-sdk/middleware-apply-body-checksum" "3.25.0" | ||
138 | "@aws-sdk/middleware-bucket-endpoint" "3.25.0" | ||
139 | "@aws-sdk/middleware-content-length" "3.25.0" | ||
140 | "@aws-sdk/middleware-expect-continue" "3.25.0" | ||
141 | "@aws-sdk/middleware-host-header" "3.25.0" | ||
142 | "@aws-sdk/middleware-location-constraint" "3.25.0" | ||
143 | "@aws-sdk/middleware-logger" "3.25.0" | ||
144 | "@aws-sdk/middleware-retry" "3.25.0" | ||
145 | "@aws-sdk/middleware-sdk-s3" "3.25.0" | ||
146 | "@aws-sdk/middleware-serde" "3.25.0" | ||
147 | "@aws-sdk/middleware-signing" "3.25.0" | ||
148 | "@aws-sdk/middleware-ssec" "3.25.0" | ||
149 | "@aws-sdk/middleware-stack" "3.25.0" | ||
150 | "@aws-sdk/middleware-user-agent" "3.25.0" | ||
151 | "@aws-sdk/node-config-provider" "3.25.0" | ||
152 | "@aws-sdk/node-http-handler" "3.25.0" | ||
153 | "@aws-sdk/protocol-http" "3.25.0" | ||
154 | "@aws-sdk/smithy-client" "3.25.0" | ||
155 | "@aws-sdk/types" "3.25.0" | ||
156 | "@aws-sdk/url-parser" "3.25.0" | ||
157 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
158 | "@aws-sdk/util-base64-node" "3.23.0" | ||
159 | "@aws-sdk/util-body-length-browser" "3.23.0" | ||
160 | "@aws-sdk/util-body-length-node" "3.23.0" | ||
161 | "@aws-sdk/util-user-agent-browser" "3.25.0" | ||
162 | "@aws-sdk/util-user-agent-node" "3.25.0" | ||
163 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
164 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
165 | "@aws-sdk/util-waiter" "3.25.0" | ||
166 | "@aws-sdk/xml-builder" "3.23.0" | ||
167 | entities "2.2.0" | ||
168 | fast-xml-parser "3.19.0" | ||
169 | tslib "^2.3.0" | ||
170 | |||
171 | "@aws-sdk/client-sso@3.25.0": | ||
172 | version "3.25.0" | ||
173 | resolved "https://registry.yarnpkg.com/@aws-sdk/client-sso/-/client-sso-3.25.0.tgz#9756178afb08e399b5aef5d12dfece3825bc2e26" | ||
174 | integrity sha512-b8v4tb7rncnqE5ktBlQEckFdNT+Pk2mBg4e1Uc9C1Z3XmZM+wOWtlbu+KRvgMgDWSx2FzLIjAKe3mLaM4o1Xhg== | ||
175 | dependencies: | ||
176 | "@aws-crypto/sha256-browser" "^1.0.0" | ||
177 | "@aws-crypto/sha256-js" "^1.0.0" | ||
178 | "@aws-sdk/config-resolver" "3.25.0" | ||
179 | "@aws-sdk/fetch-http-handler" "3.25.0" | ||
180 | "@aws-sdk/hash-node" "3.25.0" | ||
181 | "@aws-sdk/invalid-dependency" "3.25.0" | ||
182 | "@aws-sdk/middleware-content-length" "3.25.0" | ||
183 | "@aws-sdk/middleware-host-header" "3.25.0" | ||
184 | "@aws-sdk/middleware-logger" "3.25.0" | ||
185 | "@aws-sdk/middleware-retry" "3.25.0" | ||
186 | "@aws-sdk/middleware-serde" "3.25.0" | ||
187 | "@aws-sdk/middleware-stack" "3.25.0" | ||
188 | "@aws-sdk/middleware-user-agent" "3.25.0" | ||
189 | "@aws-sdk/node-config-provider" "3.25.0" | ||
190 | "@aws-sdk/node-http-handler" "3.25.0" | ||
191 | "@aws-sdk/protocol-http" "3.25.0" | ||
192 | "@aws-sdk/smithy-client" "3.25.0" | ||
193 | "@aws-sdk/types" "3.25.0" | ||
194 | "@aws-sdk/url-parser" "3.25.0" | ||
195 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
196 | "@aws-sdk/util-base64-node" "3.23.0" | ||
197 | "@aws-sdk/util-body-length-browser" "3.23.0" | ||
198 | "@aws-sdk/util-body-length-node" "3.23.0" | ||
199 | "@aws-sdk/util-user-agent-browser" "3.25.0" | ||
200 | "@aws-sdk/util-user-agent-node" "3.25.0" | ||
201 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
202 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
203 | tslib "^2.3.0" | ||
204 | |||
205 | "@aws-sdk/client-sts@3.25.0": | ||
206 | version "3.25.0" | ||
207 | resolved "https://registry.yarnpkg.com/@aws-sdk/client-sts/-/client-sts-3.25.0.tgz#e189c46d560daaa56b872330a5e7d125d00d5a1f" | ||
208 | integrity sha512-VQoG4GX+Pf5U/WtUgVgXLF2xC1jK6o4YmOxz09GhPfKT0y26x8hh42jY3zRCys7ldA3VKkfTLCeqMm3UKqXJZg== | ||
209 | dependencies: | ||
210 | "@aws-crypto/sha256-browser" "^1.0.0" | ||
211 | "@aws-crypto/sha256-js" "^1.0.0" | ||
212 | "@aws-sdk/config-resolver" "3.25.0" | ||
213 | "@aws-sdk/credential-provider-node" "3.25.0" | ||
214 | "@aws-sdk/fetch-http-handler" "3.25.0" | ||
215 | "@aws-sdk/hash-node" "3.25.0" | ||
216 | "@aws-sdk/invalid-dependency" "3.25.0" | ||
217 | "@aws-sdk/middleware-content-length" "3.25.0" | ||
218 | "@aws-sdk/middleware-host-header" "3.25.0" | ||
219 | "@aws-sdk/middleware-logger" "3.25.0" | ||
220 | "@aws-sdk/middleware-retry" "3.25.0" | ||
221 | "@aws-sdk/middleware-sdk-sts" "3.25.0" | ||
222 | "@aws-sdk/middleware-serde" "3.25.0" | ||
223 | "@aws-sdk/middleware-signing" "3.25.0" | ||
224 | "@aws-sdk/middleware-stack" "3.25.0" | ||
225 | "@aws-sdk/middleware-user-agent" "3.25.0" | ||
226 | "@aws-sdk/node-config-provider" "3.25.0" | ||
227 | "@aws-sdk/node-http-handler" "3.25.0" | ||
228 | "@aws-sdk/protocol-http" "3.25.0" | ||
229 | "@aws-sdk/smithy-client" "3.25.0" | ||
230 | "@aws-sdk/types" "3.25.0" | ||
231 | "@aws-sdk/url-parser" "3.25.0" | ||
232 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
233 | "@aws-sdk/util-base64-node" "3.23.0" | ||
234 | "@aws-sdk/util-body-length-browser" "3.23.0" | ||
235 | "@aws-sdk/util-body-length-node" "3.23.0" | ||
236 | "@aws-sdk/util-user-agent-browser" "3.25.0" | ||
237 | "@aws-sdk/util-user-agent-node" "3.25.0" | ||
238 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
239 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
240 | entities "2.2.0" | ||
241 | fast-xml-parser "3.19.0" | ||
242 | tslib "^2.3.0" | ||
243 | |||
244 | "@aws-sdk/config-resolver@3.25.0": | ||
245 | version "3.25.0" | ||
246 | resolved "https://registry.yarnpkg.com/@aws-sdk/config-resolver/-/config-resolver-3.25.0.tgz#d7caba201a00aeb9d60aeddb8901b7e58f7f5a2b" | ||
247 | integrity sha512-t5CE90jYkxQyGGxG22atf8040lHuL17wptGp1kN8nSxaG6PudKhxQuHPAGYt6FHgrqqeyFccp/P3jiDSjqUaVw== | ||
248 | dependencies: | ||
249 | "@aws-sdk/signature-v4" "3.25.0" | ||
250 | "@aws-sdk/types" "3.25.0" | ||
251 | tslib "^2.3.0" | ||
252 | |||
253 | "@aws-sdk/credential-provider-env@3.25.0": | ||
254 | version "3.25.0" | ||
255 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.25.0.tgz#9899ff627f40f09223126d6d2f1153b3ade2e804" | ||
256 | integrity sha512-I65/PNGQG+ktt1QSHCWwQ8v7QRK1eRdLkQl3zB5rwBuANbQ3Yu+vA+lAwU+IbpGCOEpHJO3lDN330It5B4Rtvg== | ||
257 | dependencies: | ||
258 | "@aws-sdk/property-provider" "3.25.0" | ||
259 | "@aws-sdk/types" "3.25.0" | ||
260 | tslib "^2.3.0" | ||
261 | |||
262 | "@aws-sdk/credential-provider-imds@3.25.0": | ||
263 | version "3.25.0" | ||
264 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.25.0.tgz#c40b76bb6a4561fb4c5fd94ce437aac938aaa23f" | ||
265 | integrity sha512-BhPM89tjeXsa0KXxz2UTLeAY798Qg1cddFXPZXaJyHQ6eWsrDSoKbSOaeP+rznp037NNLnLX6PB8MOtfu3MAzw== | ||
266 | dependencies: | ||
267 | "@aws-sdk/property-provider" "3.25.0" | ||
268 | "@aws-sdk/types" "3.25.0" | ||
269 | tslib "^2.3.0" | ||
270 | |||
271 | "@aws-sdk/credential-provider-ini@3.25.0": | ||
272 | version "3.25.0" | ||
273 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.25.0.tgz#32652f30247f84dd49e4c96ecec91577f972f2e3" | ||
274 | integrity sha512-p6yvqcZMN+eNZbJXnrFQgLpA06pVA2XagGJdkdDb3q9J4HYoWQduocWUfr3dy0HJdjDZ01BVT/ldBanUyhznQQ== | ||
275 | dependencies: | ||
276 | "@aws-sdk/credential-provider-env" "3.25.0" | ||
277 | "@aws-sdk/credential-provider-imds" "3.25.0" | ||
278 | "@aws-sdk/credential-provider-sso" "3.25.0" | ||
279 | "@aws-sdk/credential-provider-web-identity" "3.25.0" | ||
280 | "@aws-sdk/property-provider" "3.25.0" | ||
281 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
282 | "@aws-sdk/types" "3.25.0" | ||
283 | "@aws-sdk/util-credentials" "3.23.0" | ||
284 | tslib "^2.3.0" | ||
285 | |||
286 | "@aws-sdk/credential-provider-node@3.25.0": | ||
287 | version "3.25.0" | ||
288 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.25.0.tgz#f8f4c9b8ae51a89f44c11fbbf999e1363424f39e" | ||
289 | integrity sha512-GZedy79oSpnDr2I54su3EE1fwpTRFBw/Sn4RBE4VWCM8AWq7ZNk7IKAmbnBrmt+gpFpr9k2PifUIJ7fAcbNvJQ== | ||
290 | dependencies: | ||
291 | "@aws-sdk/credential-provider-env" "3.25.0" | ||
292 | "@aws-sdk/credential-provider-imds" "3.25.0" | ||
293 | "@aws-sdk/credential-provider-ini" "3.25.0" | ||
294 | "@aws-sdk/credential-provider-process" "3.25.0" | ||
295 | "@aws-sdk/credential-provider-sso" "3.25.0" | ||
296 | "@aws-sdk/credential-provider-web-identity" "3.25.0" | ||
297 | "@aws-sdk/property-provider" "3.25.0" | ||
298 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
299 | "@aws-sdk/types" "3.25.0" | ||
300 | "@aws-sdk/util-credentials" "3.23.0" | ||
301 | tslib "^2.3.0" | ||
302 | |||
303 | "@aws-sdk/credential-provider-process@3.25.0": | ||
304 | version "3.25.0" | ||
305 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.25.0.tgz#472938d6582152252fb69247531125ed24017d4e" | ||
306 | integrity sha512-qMldWWDvvy6Q+HMcTAVWUJP7MLjLXqf0P08Vb5oGYOlyh4TCJDorccRVVsQvutjQggpBaIMTQdzjdamqtZ1y+w== | ||
307 | dependencies: | ||
308 | "@aws-sdk/property-provider" "3.25.0" | ||
309 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
310 | "@aws-sdk/types" "3.25.0" | ||
311 | "@aws-sdk/util-credentials" "3.23.0" | ||
312 | tslib "^2.3.0" | ||
313 | |||
314 | "@aws-sdk/credential-provider-sso@3.25.0": | ||
315 | version "3.25.0" | ||
316 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.25.0.tgz#e2065ee6aec63a647acc816732ffcd270eb3c669" | ||
317 | integrity sha512-cGP1Zcw2fZHn4CYGgq4soody4x5TrsWk0Pf9F8yCjRMSSZqs3rj0+PrXy4xqkiLCvTSrse6p4e4wMMpaFAm7Tg== | ||
318 | dependencies: | ||
319 | "@aws-sdk/client-sso" "3.25.0" | ||
320 | "@aws-sdk/property-provider" "3.25.0" | ||
321 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
322 | "@aws-sdk/types" "3.25.0" | ||
323 | "@aws-sdk/util-credentials" "3.23.0" | ||
324 | tslib "^2.3.0" | ||
325 | |||
326 | "@aws-sdk/credential-provider-web-identity@3.25.0": | ||
327 | version "3.25.0" | ||
328 | resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.25.0.tgz#9c330322eea3a5f1f0166556c1f18ecc0992b0bf" | ||
329 | integrity sha512-6NvOaynsXGuNYbrGzT5h+kkGMaKtAI6zKgPqS/20NKlO5PJc9Eo56Hdbq0gBohXSBzRJE5Jx/1OOrTdvRlwniw== | ||
330 | dependencies: | ||
331 | "@aws-sdk/property-provider" "3.25.0" | ||
332 | "@aws-sdk/types" "3.25.0" | ||
333 | tslib "^2.3.0" | ||
334 | |||
335 | "@aws-sdk/eventstream-marshaller@3.25.0": | ||
336 | version "3.25.0" | ||
337 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-marshaller/-/eventstream-marshaller-3.25.0.tgz#8db1f633a638f50d8e37441f01d739238d374549" | ||
338 | integrity sha512-gUZIIxupgCIGyspiIV6bEplSRWnhAR9MkyrCJbHhbs4GjWIYlFqp7W0+Y7HY1tIeeXCUf0O8KE3paUMszKPXtg== | ||
339 | dependencies: | ||
340 | "@aws-crypto/crc32" "^1.0.0" | ||
341 | "@aws-sdk/types" "3.25.0" | ||
342 | "@aws-sdk/util-hex-encoding" "3.23.0" | ||
343 | tslib "^2.3.0" | ||
344 | |||
345 | "@aws-sdk/eventstream-serde-browser@3.25.0": | ||
346 | version "3.25.0" | ||
347 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-browser/-/eventstream-serde-browser-3.25.0.tgz#55481e23acb454d876948fd3b6e609a79977fa7d" | ||
348 | integrity sha512-QJF08OIZiufoBPPoVcRwBPvZIpKMSZpISZfpCHcY1GaTpMIzz35N7Nkd10JGpfzpUO9oFcgcmm2q3XHo1XJyyw== | ||
349 | dependencies: | ||
350 | "@aws-sdk/eventstream-marshaller" "3.25.0" | ||
351 | "@aws-sdk/eventstream-serde-universal" "3.25.0" | ||
352 | "@aws-sdk/types" "3.25.0" | ||
353 | tslib "^2.3.0" | ||
354 | |||
355 | "@aws-sdk/eventstream-serde-config-resolver@3.25.0": | ||
356 | version "3.25.0" | ||
357 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.25.0.tgz#5b8f4ef24fb1bf6c9f0353fb219a68206bad5eb4" | ||
358 | integrity sha512-Fb4VS3waKNzc6pK6tQBmWM+JmCNQJYNG/QBfb8y4AoJOZ+I7yX0Qgo90drh8IiUcIKDeprUFjSi/cGIa/KHIsg== | ||
359 | dependencies: | ||
360 | "@aws-sdk/types" "3.25.0" | ||
361 | tslib "^2.3.0" | ||
362 | |||
363 | "@aws-sdk/eventstream-serde-node@3.25.0": | ||
364 | version "3.25.0" | ||
365 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-node/-/eventstream-serde-node-3.25.0.tgz#7ae7fcb8db1e554638f8f1c0fea514cfb07e2524" | ||
366 | integrity sha512-gPs+6w0zXf+p0PuOxxmpAlCvP/7E7+8oAar8Ys27exnLXNgqJJK1k5hMBSrfR9GLVti3EhJ1M9x5Seg1SN0/SA== | ||
367 | dependencies: | ||
368 | "@aws-sdk/eventstream-marshaller" "3.25.0" | ||
369 | "@aws-sdk/eventstream-serde-universal" "3.25.0" | ||
370 | "@aws-sdk/types" "3.25.0" | ||
371 | tslib "^2.3.0" | ||
372 | |||
373 | "@aws-sdk/eventstream-serde-universal@3.25.0": | ||
374 | version "3.25.0" | ||
375 | resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-universal/-/eventstream-serde-universal-3.25.0.tgz#bf84056fcad894c14f7239272248ea5b3ff39d47" | ||
376 | integrity sha512-NgsQk5dXg7NlRDEKGRUdiAx7WESQGD1jEhXitklL3/PHRZ7Y9BJugEFlBvKpU7tiHZBcomTbl/gE2o6i2op/jA== | ||
377 | dependencies: | ||
378 | "@aws-sdk/eventstream-marshaller" "3.25.0" | ||
379 | "@aws-sdk/types" "3.25.0" | ||
380 | tslib "^2.3.0" | ||
381 | |||
382 | "@aws-sdk/fetch-http-handler@3.25.0": | ||
383 | version "3.25.0" | ||
384 | resolved "https://registry.yarnpkg.com/@aws-sdk/fetch-http-handler/-/fetch-http-handler-3.25.0.tgz#0ba013ced267b8ead120be1fcba5bdbbc379b82f" | ||
385 | integrity sha512-792kkbfSRBdiFb7Q2cDJts9MKxzAwuQSwUIwRKAOMazU8HkKbKnXXAFSsK3T7VasOFOh7O7YEGN0q9UgEw1q+g== | ||
386 | dependencies: | ||
387 | "@aws-sdk/protocol-http" "3.25.0" | ||
388 | "@aws-sdk/querystring-builder" "3.25.0" | ||
389 | "@aws-sdk/types" "3.25.0" | ||
390 | "@aws-sdk/util-base64-browser" "3.23.0" | ||
391 | tslib "^2.3.0" | ||
392 | |||
393 | "@aws-sdk/hash-blob-browser@3.25.0": | ||
394 | version "3.25.0" | ||
395 | resolved "https://registry.yarnpkg.com/@aws-sdk/hash-blob-browser/-/hash-blob-browser-3.25.0.tgz#2708daf0f2b53c6670a94276c1048a9a34706108" | ||
396 | integrity sha512-dsvV/nkW8v9wIotd3xJn3TQ8AxVLl56H82WkGkHcfw61csRxj3eSUNv0apUBopCcQPK8OK4l2nHAg08r0+LWXg== | ||
397 | dependencies: | ||
398 | "@aws-sdk/chunked-blob-reader" "3.23.0" | ||
399 | "@aws-sdk/chunked-blob-reader-native" "3.23.0" | ||
400 | "@aws-sdk/types" "3.25.0" | ||
401 | tslib "^2.3.0" | ||
402 | |||
403 | "@aws-sdk/hash-node@3.25.0": | ||
404 | version "3.25.0" | ||
405 | resolved "https://registry.yarnpkg.com/@aws-sdk/hash-node/-/hash-node-3.25.0.tgz#b149ddf170f4038c7cc3afe8f12e21b0f63e0771" | ||
406 | integrity sha512-qRn6iqG9VLt8D29SBABcbauDLn92ssMjtpyVApiOhDYyFm2VA2avomOHD6y2PRBMwM5FMQAygZbpA2HIN2F96w== | ||
407 | dependencies: | ||
408 | "@aws-sdk/types" "3.25.0" | ||
409 | "@aws-sdk/util-buffer-from" "3.23.0" | ||
410 | tslib "^2.3.0" | ||
411 | |||
412 | "@aws-sdk/hash-stream-node@3.25.0": | ||
413 | version "3.25.0" | ||
414 | resolved "https://registry.yarnpkg.com/@aws-sdk/hash-stream-node/-/hash-stream-node-3.25.0.tgz#6fa38cc349a9037367f20ce2601ff0510035dfa2" | ||
415 | integrity sha512-pzScUO9pPEEHQ5YQk1sl1bPlU2tt0OCblxUwboZJ9mRgNnWwkMWxe7Mec5IfyMWVUcbIznUHn7qRYEvJQ9JXmw== | ||
416 | dependencies: | ||
417 | "@aws-sdk/types" "3.25.0" | ||
418 | tslib "^2.3.0" | ||
419 | |||
420 | "@aws-sdk/invalid-dependency@3.25.0": | ||
421 | version "3.25.0" | ||
422 | resolved "https://registry.yarnpkg.com/@aws-sdk/invalid-dependency/-/invalid-dependency-3.25.0.tgz#a75dfb7e86a0e1eb6083b61397dc49a1db041434" | ||
423 | integrity sha512-ZBXjBAF2JSiO/wGBa1oaXsd1q5YG3diS8TfIUMXeQoe9O66R5LGoGOQeAbB/JjlwFot6DZfAcfocvl6CtWwqkw== | ||
424 | dependencies: | ||
425 | "@aws-sdk/types" "3.25.0" | ||
426 | tslib "^2.3.0" | ||
427 | |||
428 | "@aws-sdk/is-array-buffer@3.23.0": | ||
429 | version "3.23.0" | ||
430 | resolved "https://registry.yarnpkg.com/@aws-sdk/is-array-buffer/-/is-array-buffer-3.23.0.tgz#3a5d601b0102ea3a4d832bde647509c8405b2ec9" | ||
431 | integrity sha512-XN20/scFthok0lCbjtinW77CoIBoar8cbOzmu+HkYTnBBpJrF6Ai5g9sgglO8r+X+OLn4PrDrTP+BxdpNuIh9g== | ||
432 | dependencies: | ||
433 | tslib "^2.3.0" | ||
434 | |||
435 | "@aws-sdk/md5-js@3.25.0": | ||
436 | version "3.25.0" | ||
437 | resolved "https://registry.yarnpkg.com/@aws-sdk/md5-js/-/md5-js-3.25.0.tgz#32cefc43a8c0ee1d85586b95eba0be4912cde534" | ||
438 | integrity sha512-97MtL1VF3JCkyJJnwi8LcXpqItnH1VtgoqtVqmaASYp5GXnlsnA1WDnB0754ufPHlssS1aBj/gkLzMZ0Htw/Rg== | ||
439 | dependencies: | ||
440 | "@aws-sdk/types" "3.25.0" | ||
441 | "@aws-sdk/util-utf8-browser" "3.23.0" | ||
442 | "@aws-sdk/util-utf8-node" "3.23.0" | ||
443 | tslib "^2.3.0" | ||
444 | |||
445 | "@aws-sdk/middleware-apply-body-checksum@3.25.0": | ||
446 | version "3.25.0" | ||
447 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-apply-body-checksum/-/middleware-apply-body-checksum-3.25.0.tgz#4263ea8c8e1808e5a4a278fb704ebe7aa891f698" | ||
448 | integrity sha512-162qFG7eap4vDKuKrpXWQYE4tbIETNrpTQX6jrPgqostOy1O0Nc5Bn1COIoOMgeMVnkOAZV7qV1J/XAYGz32Yw== | ||
449 | dependencies: | ||
450 | "@aws-sdk/is-array-buffer" "3.23.0" | ||
451 | "@aws-sdk/protocol-http" "3.25.0" | ||
452 | "@aws-sdk/types" "3.25.0" | ||
453 | tslib "^2.3.0" | ||
454 | |||
455 | "@aws-sdk/middleware-bucket-endpoint@3.25.0": | ||
456 | version "3.25.0" | ||
457 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.25.0.tgz#d698230ae552533a1b8ded2c3e6885b4a8374795" | ||
458 | integrity sha512-r/6ECFiw/TNjzhAuZzUx3M/1mAtezHTp3e8twB4dDbRRQqABrEZ/dynXi1VxrT2kKW0ZgZNXqEer/NfPOtWB8g== | ||
459 | dependencies: | ||
460 | "@aws-sdk/protocol-http" "3.25.0" | ||
461 | "@aws-sdk/types" "3.25.0" | ||
462 | "@aws-sdk/util-arn-parser" "3.23.0" | ||
463 | tslib "^2.3.0" | ||
464 | |||
465 | "@aws-sdk/middleware-content-length@3.25.0": | ||
466 | version "3.25.0" | ||
467 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-content-length/-/middleware-content-length-3.25.0.tgz#71031d326e52f788396e0ed8216410840059ac53" | ||
468 | integrity sha512-uOXus0MmZi/mucRIr5yfwM1vDhYG66CujNfnhyEaq5f4kcDA1Q5qPWSn9dkQPV9JWTZK3WTuYiOPSgtmlAYTAg== | ||
469 | dependencies: | ||
470 | "@aws-sdk/protocol-http" "3.25.0" | ||
471 | "@aws-sdk/types" "3.25.0" | ||
472 | tslib "^2.3.0" | ||
473 | |||
474 | "@aws-sdk/middleware-expect-continue@3.25.0": | ||
475 | version "3.25.0" | ||
476 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.25.0.tgz#bb41ea9d716c6ce04c4d8fb2cc2dd5fd37f6ccd9" | ||
477 | integrity sha512-o3euv8NIO0zlHML81krtfs4TrF5gZwoxBYtY+6tRHXlgutsHe1yfg1wrhWnJNbJg1QhPwXxbMNfYX7MM83D8Ng== | ||
478 | dependencies: | ||
479 | "@aws-sdk/middleware-header-default" "3.25.0" | ||
480 | "@aws-sdk/protocol-http" "3.25.0" | ||
481 | "@aws-sdk/types" "3.25.0" | ||
482 | tslib "^2.3.0" | ||
483 | |||
484 | "@aws-sdk/middleware-header-default@3.25.0": | ||
485 | version "3.25.0" | ||
486 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-header-default/-/middleware-header-default-3.25.0.tgz#17fec9b1941e81059a1374eba58b52230da35a2b" | ||
487 | integrity sha512-xkFfZcctPL0VTxmEKITf6/MSDv/8rY+8uA9OMt/YZqfbg0RfeqR2+R1xlDNDxeHeK/v+g5gTNIYTQLM8L2unNA== | ||
488 | dependencies: | ||
489 | "@aws-sdk/protocol-http" "3.25.0" | ||
490 | "@aws-sdk/types" "3.25.0" | ||
491 | tslib "^2.3.0" | ||
492 | |||
493 | "@aws-sdk/middleware-host-header@3.25.0": | ||
494 | version "3.25.0" | ||
495 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.25.0.tgz#f08dd8c45362cf5cb152c478027092e3d1f4aa58" | ||
496 | integrity sha512-xKD/CfsUS3ul2VaQ3IgIUXgA7jU2/Guo/DUhYKrLZTOxm0nuvsIFw0RqSCtRBCLptE5Qi+unkc1LcFDbfqrRbg== | ||
497 | dependencies: | ||
498 | "@aws-sdk/protocol-http" "3.25.0" | ||
499 | "@aws-sdk/types" "3.25.0" | ||
500 | tslib "^2.3.0" | ||
501 | |||
502 | "@aws-sdk/middleware-location-constraint@3.25.0": | ||
503 | version "3.25.0" | ||
504 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.25.0.tgz#7ba5798aa46cd08c90823f649fcdae0ce5227095" | ||
505 | integrity sha512-diwmJ+MRQrq3H9VH+8CNAT4dImf2j3CLewlMrUEY+HsJN9xl2mtU6GQaluQg60iw6FjurLUKKGTTZCul4PGkIQ== | ||
506 | dependencies: | ||
507 | "@aws-sdk/types" "3.25.0" | ||
508 | tslib "^2.3.0" | ||
509 | |||
510 | "@aws-sdk/middleware-logger@3.25.0": | ||
511 | version "3.25.0" | ||
512 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.25.0.tgz#03294611be7a2f4aba06e9d80e04318c0991d769" | ||
513 | integrity sha512-M1F7BlAsDKoEM8hBaU2pHlLSM40rzzgtZ6jFNhfmTwGcjxe1N7JXCH5QPa7aI8wnJq2RoIRHVfVsUH4GwvOZnA== | ||
514 | dependencies: | ||
515 | "@aws-sdk/types" "3.25.0" | ||
516 | tslib "^2.3.0" | ||
517 | |||
518 | "@aws-sdk/middleware-retry@3.25.0": | ||
519 | version "3.25.0" | ||
520 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-retry/-/middleware-retry-3.25.0.tgz#e9f1b011494142aa27ece3ef881e8a3d4866797c" | ||
521 | integrity sha512-SzdWPo4ESUR6AXvIf4eC8s5sko2G9Hou6cUIr+BWI4h7whA32j/aWUmvcMHxWT/eaSuPeruXrnvKyLvuM0RjJg== | ||
522 | dependencies: | ||
523 | "@aws-sdk/protocol-http" "3.25.0" | ||
524 | "@aws-sdk/service-error-classification" "3.25.0" | ||
525 | "@aws-sdk/types" "3.25.0" | ||
526 | tslib "^2.3.0" | ||
527 | uuid "^8.3.2" | ||
528 | |||
529 | "@aws-sdk/middleware-sdk-s3@3.25.0": | ||
530 | version "3.25.0" | ||
531 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.25.0.tgz#64278bbc97c3a2c26411f155642cc35e8de38887" | ||
532 | integrity sha512-Y1P6JnpAdj7p5Q43aSLSuYBCc3hKpZ/mrqFSGN8VFXl7Tzo7tYfjpd9SVRxNGJK7O7tDAUsPNmuGqBrdA2tj8w== | ||
533 | dependencies: | ||
534 | "@aws-sdk/protocol-http" "3.25.0" | ||
535 | "@aws-sdk/types" "3.25.0" | ||
536 | "@aws-sdk/util-arn-parser" "3.23.0" | ||
537 | tslib "^2.3.0" | ||
538 | |||
539 | "@aws-sdk/middleware-sdk-sts@3.25.0": | ||
540 | version "3.25.0" | ||
541 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.25.0.tgz#15d4836958f70187cbb6819a0c0742b751fb44ed" | ||
542 | integrity sha512-1SoZZTVejo+32eH0WqXaFvt/NIkVEYWquh3OJpkghMi2oOnMfeIRI0uSoqshL6949f4iSfUvvtuzDpyA7XNCQA== | ||
543 | dependencies: | ||
544 | "@aws-sdk/middleware-signing" "3.25.0" | ||
545 | "@aws-sdk/property-provider" "3.25.0" | ||
546 | "@aws-sdk/protocol-http" "3.25.0" | ||
547 | "@aws-sdk/signature-v4" "3.25.0" | ||
548 | "@aws-sdk/types" "3.25.0" | ||
549 | tslib "^2.3.0" | ||
550 | |||
551 | "@aws-sdk/middleware-serde@3.25.0": | ||
552 | version "3.25.0" | ||
553 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-serde/-/middleware-serde-3.25.0.tgz#e1284ed4af64b4444cfeb7b5275f489418fa2f58" | ||
554 | integrity sha512-065Kugo8yXzBkcVAxctxFCHKlHcINnaQRsJ8ifvgc+UOEgvTG9+LfGWDwfdgarW9CkF7RkCoZOyaqFsO+HJWsg== | ||
555 | dependencies: | ||
556 | "@aws-sdk/types" "3.25.0" | ||
557 | tslib "^2.3.0" | ||
558 | |||
559 | "@aws-sdk/middleware-signing@3.25.0": | ||
560 | version "3.25.0" | ||
561 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-signing/-/middleware-signing-3.25.0.tgz#de19f5b27c34161081553a87285f1b5690e2cb9a" | ||
562 | integrity sha512-FkhxGMV3UY5HIAwUcarfxdq/CF/tYukdg+bkbTNluMpkcJczqn6shpEIQAGa5FFQP3Lya+STL1NuNXfOP7bG9w== | ||
563 | dependencies: | ||
564 | "@aws-sdk/property-provider" "3.25.0" | ||
565 | "@aws-sdk/protocol-http" "3.25.0" | ||
566 | "@aws-sdk/signature-v4" "3.25.0" | ||
567 | "@aws-sdk/types" "3.25.0" | ||
568 | tslib "^2.3.0" | ||
569 | |||
570 | "@aws-sdk/middleware-ssec@3.25.0": | ||
571 | version "3.25.0" | ||
572 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.25.0.tgz#f8cf5bb6fe48d842b1df77f35ccb0f77f1a07b71" | ||
573 | integrity sha512-bnrHb8oddW+vDexbNzZtpfshshKru+skcmq3dyXlL8LB/NlJsMiQJE8xoGbq5odTLiflIgaDBt527m5q58i+fg== | ||
574 | dependencies: | ||
575 | "@aws-sdk/types" "3.25.0" | ||
576 | tslib "^2.3.0" | ||
577 | |||
578 | "@aws-sdk/middleware-stack@3.25.0": | ||
579 | version "3.25.0" | ||
580 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-stack/-/middleware-stack-3.25.0.tgz#8fc022c90b030c80308bf2930c4a7040052234b4" | ||
581 | integrity sha512-s2VgdsasOVKHY3/SIGsw9AeZMMsdcIbBGWim9n5IO3j8C8y54EdRLVCEja8ePvMDZKIzuummwatYPHaUrnqPtQ== | ||
582 | dependencies: | ||
583 | tslib "^2.3.0" | ||
584 | |||
585 | "@aws-sdk/middleware-user-agent@3.25.0": | ||
586 | version "3.25.0" | ||
587 | resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.25.0.tgz#2033da6cdcfbf4641b991e3ee3c60ba9809898e7" | ||
588 | integrity sha512-HXd/Qknq8Cp7fzJYU7jDDpN7ReJ3arUrnt+dAPNaDDrhmrBbCZp+24UXN6X6DAj0JICRoRuF/l7KxjwdF5FShw== | ||
589 | dependencies: | ||
590 | "@aws-sdk/protocol-http" "3.25.0" | ||
591 | "@aws-sdk/types" "3.25.0" | ||
592 | tslib "^2.3.0" | ||
593 | |||
594 | "@aws-sdk/node-config-provider@3.25.0": | ||
595 | version "3.25.0" | ||
596 | resolved "https://registry.yarnpkg.com/@aws-sdk/node-config-provider/-/node-config-provider-3.25.0.tgz#6ec3e9031b7ff0c51d6e0b33aeff3547ea5619b3" | ||
597 | integrity sha512-95FiUDuh1YGo0Giti0Xz9l2TV0Wzw75M1xx0TduFcm1dpLKl+znxTgYh+4G+MOSMHNGy+6K91yxurv4PGYgCWw== | ||
598 | dependencies: | ||
599 | "@aws-sdk/property-provider" "3.25.0" | ||
600 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
601 | "@aws-sdk/types" "3.25.0" | ||
602 | tslib "^2.3.0" | ||
603 | |||
604 | "@aws-sdk/node-http-handler@3.25.0": | ||
605 | version "3.25.0" | ||
606 | resolved "https://registry.yarnpkg.com/@aws-sdk/node-http-handler/-/node-http-handler-3.25.0.tgz#b636ea2c39b4a47cf9bffd4cdb6a41c603b99bff" | ||
607 | integrity sha512-zVeAM/bXewZiuMtcUZI/xGDID6knkzOv73ueVkzUbP0Ki8bfao7diR3hMbIt5Fy/r8cAVjJce9v6zFqo4sr1WA== | ||
608 | dependencies: | ||
609 | "@aws-sdk/abort-controller" "3.25.0" | ||
610 | "@aws-sdk/protocol-http" "3.25.0" | ||
611 | "@aws-sdk/querystring-builder" "3.25.0" | ||
612 | "@aws-sdk/types" "3.25.0" | ||
613 | tslib "^2.3.0" | ||
614 | |||
615 | "@aws-sdk/property-provider@3.25.0": | ||
616 | version "3.25.0" | ||
617 | resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.25.0.tgz#2fd7246917b9b6ff448a599163a479bc417a1421" | ||
618 | integrity sha512-jUnPDguLWsyGLPfdxGdeaXe3j/CjS3kxBmctvI+soZg57rA2hntP9rm7SUZ2+5rj4mmJaI3bzchiaY3kE3JmpA== | ||
619 | dependencies: | ||
620 | "@aws-sdk/types" "3.25.0" | ||
621 | tslib "^2.3.0" | ||
622 | |||
623 | "@aws-sdk/protocol-http@3.25.0": | ||
624 | version "3.25.0" | ||
625 | resolved "https://registry.yarnpkg.com/@aws-sdk/protocol-http/-/protocol-http-3.25.0.tgz#4b638cb90672fc2d6cb6d15bebc8bb1fb297da2e" | ||
626 | integrity sha512-4Jebt5G8uIFa+HZO7KOgOtA66E/CXysQekiV5dfAsU8ca+rX5PB6qhpWZ2unX/l6He+oDQ0zMoW70JkNiP4/4w== | ||
627 | dependencies: | ||
628 | "@aws-sdk/types" "3.25.0" | ||
629 | tslib "^2.3.0" | ||
630 | |||
631 | "@aws-sdk/querystring-builder@3.25.0": | ||
632 | version "3.25.0" | ||
633 | resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.25.0.tgz#9e6f5eaa5d6805fbf45ae4a47ccbaf823584a4a2" | ||
634 | integrity sha512-o/R3/viOxjWckI+kepkxJSL7fIdg1hHYOW/rOpo9HbXS0CJrHVnB8vlBb+Xwl1IFyY2gg+5YZTjiufcgpgRBkw== | ||
635 | dependencies: | ||
636 | "@aws-sdk/types" "3.25.0" | ||
637 | "@aws-sdk/util-uri-escape" "3.23.0" | ||
638 | tslib "^2.3.0" | ||
639 | |||
640 | "@aws-sdk/querystring-parser@3.25.0": | ||
641 | version "3.25.0" | ||
642 | resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.25.0.tgz#7fe0a3ddf95a4e5475f53be056fce435fb24b774" | ||
643 | integrity sha512-FCNyaOLFLVS5j43MhVA7/VJUDX0t/9RyNTNulHgzFjj6ffsgqcY0uwUq1RO3QCL4asl56zOrLVJgK+Z7wMbvFg== | ||
644 | dependencies: | ||
645 | "@aws-sdk/types" "3.25.0" | ||
646 | tslib "^2.3.0" | ||
647 | |||
648 | "@aws-sdk/service-error-classification@3.25.0": | ||
649 | version "3.25.0" | ||
650 | resolved "https://registry.yarnpkg.com/@aws-sdk/service-error-classification/-/service-error-classification-3.25.0.tgz#1f24fe74f0a89f00d4f6f2ad1d7bb6b0e2f871e7" | ||
651 | integrity sha512-66FfIab87LnnHtOLrGrVOht9Pw6lE8appyOpBdtoeoU5DP7ARSWuDdsYmKdGdRCWvn/RaVFbSYua9k0M1WsGqg== | ||
652 | |||
653 | "@aws-sdk/shared-ini-file-loader@3.23.0": | ||
654 | version "3.23.0" | ||
655 | resolved "https://registry.yarnpkg.com/@aws-sdk/shared-ini-file-loader/-/shared-ini-file-loader-3.23.0.tgz#574901a31e65e425632a9cae6a64f6382a2b76e8" | ||
656 | integrity sha512-YUp46l6E3dLKHp1cKMkZI4slTjsVc/Lm7nPCTVc3oQvZ1MvC99N/jMCmZ7X5YYofuAUSdc9eJ8sYiF2BnUww9g== | ||
657 | dependencies: | ||
658 | tslib "^2.3.0" | ||
659 | |||
660 | "@aws-sdk/signature-v4@3.25.0": | ||
661 | version "3.25.0" | ||
662 | resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4/-/signature-v4-3.25.0.tgz#c7fb8184a09593ef6dc62029ca45e252b51247b2" | ||
663 | integrity sha512-6KDRRz9XVrj9RxrBLC6dzfnb2TDl3CjIzcNpLdRuKFgzEEdwV+5D+EZuAQU3MuHG5pWTIwG72k/dmCbJ2MDPUQ== | ||
664 | dependencies: | ||
665 | "@aws-sdk/is-array-buffer" "3.23.0" | ||
666 | "@aws-sdk/types" "3.25.0" | ||
667 | "@aws-sdk/util-hex-encoding" "3.23.0" | ||
668 | "@aws-sdk/util-uri-escape" "3.23.0" | ||
669 | tslib "^2.3.0" | ||
670 | |||
671 | "@aws-sdk/smithy-client@3.25.0": | ||
672 | version "3.25.0" | ||
673 | resolved "https://registry.yarnpkg.com/@aws-sdk/smithy-client/-/smithy-client-3.25.0.tgz#bfdf77f1fa82b26bb7893f16056e8e60e49a140a" | ||
674 | integrity sha512-+/iMCNziL5/muaY/gl3xkRsSZyeoVCUSjSbbZjDIXbqDbB9SOz4o3UAIgWHoCgYNfsF25GQR6rThLi61FrSyoQ== | ||
675 | dependencies: | ||
676 | "@aws-sdk/middleware-stack" "3.25.0" | ||
677 | "@aws-sdk/types" "3.25.0" | ||
678 | tslib "^2.3.0" | ||
679 | |||
680 | "@aws-sdk/types@3.25.0", "@aws-sdk/types@^3.1.0": | ||
681 | version "3.25.0" | ||
682 | resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.25.0.tgz#981210272dae2d259130f6dca8429522d9a564bb" | ||
683 | integrity sha512-vS0+cTKwj6CujlR07HmeEBxzWPWSrdmZMYnxn/QC9KW9dFu0lsyCGSCqWsFluI6GI0flsnYYWNkP5y4bfD9tqg== | ||
684 | |||
685 | "@aws-sdk/url-parser@3.25.0": | ||
686 | version "3.25.0" | ||
687 | resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.25.0.tgz#668c7d9d4bc21854c10bfb8bdf762a9206776fae" | ||
688 | integrity sha512-qZ3Vq0NjHsE7Qq6R5NVRswIAsiyYjCDnAV+/Vt4jU/K0V3mGumiasiJyRyblW4Da8R6kfcJk0mHSMFRJfoHh8Q== | ||
689 | dependencies: | ||
690 | "@aws-sdk/querystring-parser" "3.25.0" | ||
691 | "@aws-sdk/types" "3.25.0" | ||
692 | tslib "^2.3.0" | ||
693 | |||
694 | "@aws-sdk/util-arn-parser@3.23.0": | ||
695 | version "3.23.0" | ||
696 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.23.0.tgz#7372460ba98a6826f97d9622759764bcf09add79" | ||
697 | integrity sha512-J3+/wnC21kbb3UAHo7x31aCZxzIa7GBijt6Q7nad/j2aF38EZtE3SI0aZpD8250Vi+9zsZ4672QDUeSZ5BR5kg== | ||
698 | dependencies: | ||
699 | tslib "^2.3.0" | ||
700 | |||
701 | "@aws-sdk/util-base64-browser@3.23.0": | ||
702 | version "3.23.0" | ||
703 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-base64-browser/-/util-base64-browser-3.23.0.tgz#61594ac9529756361c81ece287548ab5b8c5a768" | ||
704 | integrity sha512-xlI/qw+uhLJWa3k0mRtRHQ42v5QzsMFEUXScredQMfJ/34qzXyocsG6OHPOTV1I8WSANrxnHR5m1Ae3iU6JuVw== | ||
705 | dependencies: | ||
706 | tslib "^2.3.0" | ||
707 | |||
708 | "@aws-sdk/util-base64-node@3.23.0": | ||
709 | version "3.23.0" | ||
710 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-base64-node/-/util-base64-node-3.23.0.tgz#d0da9ed6b8aaa7513ba4b36a20b4794c72c074ce" | ||
711 | integrity sha512-Kf8JIAUtjrPcD5CJzrig2B5CtegWswUNpW4zBarww/UJhHlp8WzKlCxxA+yNS1ghT0ZMjrRvxPabKDGpkyUfmQ== | ||
712 | dependencies: | ||
713 | "@aws-sdk/util-buffer-from" "3.23.0" | ||
714 | tslib "^2.3.0" | ||
715 | |||
716 | "@aws-sdk/util-body-length-browser@3.23.0": | ||
717 | version "3.23.0" | ||
718 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-body-length-browser/-/util-body-length-browser-3.23.0.tgz#1a5c5e7ea5e15d93bd178021c54d2ea41faeb1cd" | ||
719 | integrity sha512-Bi6u/5omQbOBSB5BxqVvaPgVplLRjhhSuqK3XAukbeBPh7lcibIBdy7YvbhQyl4i8Hb2QjFnqqfzA0lNBe5eiw== | ||
720 | dependencies: | ||
721 | tslib "^2.3.0" | ||
722 | |||
723 | "@aws-sdk/util-body-length-node@3.23.0": | ||
724 | version "3.23.0" | ||
725 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-body-length-node/-/util-body-length-node-3.23.0.tgz#2a7890b4fa6de78a042db9537a67f90ccb2a3034" | ||
726 | integrity sha512-8kSczloA78mikPaJ742SU9Wpwfcz3HOruoXiP/pOy69UZEsMe4P7zTZI1bo8BAp7j6IFUPCXth9E3UAtkbz+CQ== | ||
727 | dependencies: | ||
728 | tslib "^2.3.0" | ||
729 | |||
730 | "@aws-sdk/util-buffer-from@3.23.0": | ||
731 | version "3.23.0" | ||
732 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-buffer-from/-/util-buffer-from-3.23.0.tgz#3bc02f50c6e8a5c2b9db61faeb3bebc9de701c3b" | ||
733 | integrity sha512-axXy1FvEOM1uECgMPmyHF1S3Hd7JI+BerhhcAlGig0bbqUsZVQUNL9yhOsWreA+nf1v08Ucj8P2SHPCT9Hvpgg== | ||
734 | dependencies: | ||
735 | "@aws-sdk/is-array-buffer" "3.23.0" | ||
736 | tslib "^2.3.0" | ||
737 | |||
738 | "@aws-sdk/util-credentials@3.23.0": | ||
739 | version "3.23.0" | ||
740 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-credentials/-/util-credentials-3.23.0.tgz#6b3138c3853c72adc93c3f57e8fb28f58ffdc364" | ||
741 | integrity sha512-6TDGZnFa0kZr+vSsWXXMfWt347jbMGKtzGnBxbrmiQgZMijz9s/wLYxsjglZ+CyqI/QrSMOTtqy6mEgJxdnGWQ== | ||
742 | dependencies: | ||
743 | "@aws-sdk/shared-ini-file-loader" "3.23.0" | ||
744 | tslib "^2.3.0" | ||
745 | |||
746 | "@aws-sdk/util-hex-encoding@3.23.0": | ||
747 | version "3.23.0" | ||
748 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.23.0.tgz#a8de34faf9e51dd4be379be0e9d3bdc093ae6bf4" | ||
749 | integrity sha512-RFDCwNrJMmmPSMVRadxRNePqTXGwtL9s4844x44D0bbGg1TdC42rrg0PRKYkxFL7wd1FbibVQOzciZAvzF+Z+w== | ||
750 | dependencies: | ||
751 | tslib "^2.3.0" | ||
752 | |||
753 | "@aws-sdk/util-locate-window@^3.0.0": | ||
754 | version "3.23.0" | ||
755 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-locate-window/-/util-locate-window-3.23.0.tgz#e9bf2a023dce2ea1d13ec2e8c7c92abb333a1442" | ||
756 | integrity sha512-mM8kWW7SWIxCshkNllpYqCQi5SzwJ+sv5nURhtquOB5/H3qGqZm0V5lUE3qpE1AYmqKwk6qbGUy1woFn1T5nrw== | ||
757 | dependencies: | ||
758 | tslib "^2.3.0" | ||
759 | |||
760 | "@aws-sdk/util-uri-escape@3.23.0": | ||
761 | version "3.23.0" | ||
762 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-uri-escape/-/util-uri-escape-3.23.0.tgz#52539674966eb456d65408d9028ed114e94dfd49" | ||
763 | integrity sha512-SvQx2E/FDlI5vLT67wwn/k1j2R/G58tYj4Te6GNgEwPGL43X2+7c0+d/WTgndMaRvxSBHZMUTxBYh1HOeU7loA== | ||
764 | dependencies: | ||
765 | tslib "^2.3.0" | ||
766 | |||
767 | "@aws-sdk/util-user-agent-browser@3.25.0": | ||
768 | version "3.25.0" | ||
769 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.25.0.tgz#a0f480f1a5b10350370643445b09413102187935" | ||
770 | integrity sha512-qGqiWfs49NRmQVXPsBXgMRVkjDZocicU0V2wak98e0t7TOI+KmP8hnwsTkE6c4KwhsFOOUhAzjn5zk3kOwi6tQ== | ||
771 | dependencies: | ||
772 | "@aws-sdk/types" "3.25.0" | ||
773 | bowser "^2.11.0" | ||
774 | tslib "^2.3.0" | ||
775 | |||
776 | "@aws-sdk/util-user-agent-node@3.25.0": | ||
777 | version "3.25.0" | ||
778 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.25.0.tgz#db22cb64893c4635adf17086c5cb4a5070c4ac16" | ||
779 | integrity sha512-4AWyCNP3n/qxv36OS+WH3l4ooRvwyfdbYWFXNXeGcxMcLANDG0upJQRT1g7H8+/afMaJ6v/BQM/H6tdocJSKjQ== | ||
780 | dependencies: | ||
781 | "@aws-sdk/node-config-provider" "3.25.0" | ||
782 | "@aws-sdk/types" "3.25.0" | ||
783 | tslib "^2.3.0" | ||
784 | |||
785 | "@aws-sdk/util-utf8-browser@3.23.0", "@aws-sdk/util-utf8-browser@^3.0.0": | ||
786 | version "3.23.0" | ||
787 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.23.0.tgz#dff7e891c67936de677b7d7a6c796e5c2e1b1510" | ||
788 | integrity sha512-fSB95AKnvCnAbCd7o0xLbErfAgD9wnLCaEu23AgfGAiaG3nFF8Z2+wtjebU/9Z4RI9d/x83Ho/yguRnJdkMsPA== | ||
789 | dependencies: | ||
790 | tslib "^2.3.0" | ||
791 | |||
792 | "@aws-sdk/util-utf8-node@3.23.0": | ||
793 | version "3.23.0" | ||
794 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-node/-/util-utf8-node-3.23.0.tgz#9f9fe76745c79c8a148f15d78e9a5c03d2bf0441" | ||
795 | integrity sha512-yao8+8okyfCxRvxZe3GBdO7lJnQEBf3P6rDgleOQD/0DZmMjOQGXCvDd42oagE2TegXhkUnJfVOZU2GqdoR0hg== | ||
796 | dependencies: | ||
797 | "@aws-sdk/util-buffer-from" "3.23.0" | ||
798 | tslib "^2.3.0" | ||
799 | |||
800 | "@aws-sdk/util-waiter@3.25.0": | ||
801 | version "3.25.0" | ||
802 | resolved "https://registry.yarnpkg.com/@aws-sdk/util-waiter/-/util-waiter-3.25.0.tgz#cd2252c99f335e461134f55c3b7eb89ef6893dca" | ||
803 | integrity sha512-rhJ7Q2fcPD8y4H0qNEpaspkSUya0OaNcVrca9wCZKs7jWnropPzrQ+e2MH7fWJ/8jgcBV890+Txr4fWkD4J01g== | ||
804 | dependencies: | ||
805 | "@aws-sdk/abort-controller" "3.25.0" | ||
806 | "@aws-sdk/types" "3.25.0" | ||
807 | tslib "^2.3.0" | ||
808 | |||
809 | "@aws-sdk/xml-builder@3.23.0": | ||
810 | version "3.23.0" | ||
811 | resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.23.0.tgz#e318f539b68fa9c0a36da49e85a96cdca13a8113" | ||
812 | integrity sha512-5LEGdhQIJtGTwg4dIYyNtpz5QvPcQoxsqJygmj+VB8KLd+mWorH1IOpiL74z0infeK9N+ZFUUPKIzPJa9xLPqw== | ||
813 | dependencies: | ||
814 | tslib "^2.3.0" | ||
815 | |||
52 | "@babel/code-frame@7.12.11": | 816 | "@babel/code-frame@7.12.11": |
53 | version "7.12.11" | 817 | version "7.12.11" |
54 | resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" | 818 | resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" |
@@ -1652,6 +2416,11 @@ boolean@3.0.4: | |||
1652 | resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.4.tgz#aa1df8749af41d7211b66b4eee584722ff428c27" | 2416 | resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.4.tgz#aa1df8749af41d7211b66b4eee584722ff428c27" |
1653 | integrity sha512-5pyOr+w2LNN72F2mAq6J0ckHUfJYSgRKma7e/wlcMMhgOLV9OI0ERhERYXxUqo+dPyVxcbXKy9n+wg13+LpNnA== | 2417 | integrity sha512-5pyOr+w2LNN72F2mAq6J0ckHUfJYSgRKma7e/wlcMMhgOLV9OI0ERhERYXxUqo+dPyVxcbXKy9n+wg13+LpNnA== |
1654 | 2418 | ||
2419 | bowser@^2.11.0: | ||
2420 | version "2.11.0" | ||
2421 | resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f" | ||
2422 | integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA== | ||
2423 | |||
1655 | boxen@^4.2.0: | 2424 | boxen@^4.2.0: |
1656 | version "4.2.0" | 2425 | version "4.2.0" |
1657 | resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" | 2426 | resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" |
@@ -2909,7 +3678,7 @@ enquirer@^2.3.5: | |||
2909 | dependencies: | 3678 | dependencies: |
2910 | ansi-colors "^4.1.1" | 3679 | ansi-colors "^4.1.1" |
2911 | 3680 | ||
2912 | entities@^2.0.0: | 3681 | entities@2.2.0, entities@^2.0.0: |
2913 | version "2.2.0" | 3682 | version "2.2.0" |
2914 | resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" | 3683 | resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" |
2915 | integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== | 3684 | integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== |
@@ -3403,7 +4172,7 @@ fast-safe-stringify@^2.0.4, fast-safe-stringify@^2.0.7: | |||
3403 | resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f" | 4172 | resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f" |
3404 | integrity sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag== | 4173 | integrity sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag== |
3405 | 4174 | ||
3406 | fast-xml-parser@^3.19.0: | 4175 | fast-xml-parser@3.19.0, fast-xml-parser@^3.19.0: |
3407 | version "3.19.0" | 4176 | version "3.19.0" |
3408 | resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" | 4177 | resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" |
3409 | integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== | 4178 | integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== |
@@ -7947,7 +8716,7 @@ tsconfig-paths@^3.9.0: | |||
7947 | minimist "^1.2.0" | 8716 | minimist "^1.2.0" |
7948 | strip-bom "^3.0.0" | 8717 | strip-bom "^3.0.0" |
7949 | 8718 | ||
7950 | tslib@^1.8.1, tslib@^1.9.0: | 8719 | tslib@^1.11.1, tslib@^1.8.1, tslib@^1.9.0: |
7951 | version "1.14.1" | 8720 | version "1.14.1" |
7952 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" | 8721 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" |
7953 | integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== | 8722 | integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== |
@@ -7957,6 +8726,11 @@ tslib@^2.0.0, tslib@^2.2.0: | |||
7957 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" | 8726 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" |
7958 | integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== | 8727 | integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== |
7959 | 8728 | ||
8729 | tslib@^2.3.0: | ||
8730 | version "2.3.1" | ||
8731 | resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" | ||
8732 | integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== | ||
8733 | |||
7960 | tsutils@^3.21.0: | 8734 | tsutils@^3.21.0: |
7961 | version "3.21.0" | 8735 | version "3.21.0" |
7962 | resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" | 8736 | resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" |