aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--.github/workflows/test.yml6
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.ts3
-rw-r--r--client/src/app/+videos/+video-watch/shared/information/video-alert.component.html4
-rw-r--r--client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts4
-rw-r--r--config/default.yaml33
-rw-r--r--config/production.yaml.example33
-rw-r--r--package.json1
-rwxr-xr-xscripts/ci.sh3
-rwxr-xr-xscripts/create-transcoding-job.ts13
-rw-r--r--scripts/optimize-old-videos.ts91
-rw-r--r--server/controllers/api/videos/upload.ts34
-rw-r--r--server/controllers/download.ts25
-rw-r--r--server/helpers/webtorrent.ts31
-rw-r--r--server/initializers/checker-after-init.ts23
-rw-r--r--server/initializers/config.ts20
-rw-r--r--server/initializers/constants.ts14
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/migrations/0065-video-file-size.ts28
-rw-r--r--server/initializers/migrations/0660-object-storage.ts58
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts2
-rw-r--r--server/lib/hls.ts82
-rw-r--r--server/lib/job-queue/handlers/move-to-object-storage.ts114
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts27
-rw-r--r--server/lib/job-queue/handlers/video-import.ts16
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts18
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts128
-rw-r--r--server/lib/job-queue/job-queue.ts13
-rw-r--r--server/lib/live/live-manager.ts2
-rw-r--r--server/lib/live/live-utils.ts4
-rw-r--r--server/lib/live/shared/muxing-session.ts4
-rw-r--r--server/lib/object-storage/index.ts3
-rw-r--r--server/lib/object-storage/keys.ts20
-rw-r--r--server/lib/object-storage/shared/client.ts56
-rw-r--r--server/lib/object-storage/shared/index.ts3
-rw-r--r--server/lib/object-storage/shared/logger.ts7
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts229
-rw-r--r--server/lib/object-storage/urls.ts40
-rw-r--r--server/lib/object-storage/videos.ts72
-rw-r--r--server/lib/paths.ts (renamed from server/lib/video-paths.ts)66
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts2
-rw-r--r--server/lib/thumbnail.ts30
-rw-r--r--server/lib/transcoding/video-transcoding.ts228
-rw-r--r--server/lib/video-path-manager.ts139
-rw-r--r--server/lib/video-state.ts99
-rw-r--r--server/lib/video-urls.ts31
-rw-r--r--server/lib/video.ts47
-rw-r--r--server/models/video/formatter/video-format-utils.ts2
-rw-r--r--server/models/video/sql/shared/video-tables.ts9
-rw-r--r--server/models/video/video-file.ts26
-rw-r--r--server/models/video/video-job-info.ts100
-rw-r--r--server/models/video/video-streaming-playlist.ts30
-rw-r--r--server/models/video/video.ts51
-rw-r--r--server/tests/api/index.ts1
-rw-r--r--server/tests/api/live/live-save-replay.ts30
-rw-r--r--server/tests/api/object-storage/index.ts3
-rw-r--r--server/tests/api/object-storage/live.ts136
-rw-r--r--server/tests/api/object-storage/video-imports.ts112
-rw-r--r--server/tests/api/object-storage/videos.ts391
-rw-r--r--server/tests/api/redundancy/redundancy.ts6
-rw-r--r--server/tests/api/videos/video-hls.ts73
-rw-r--r--server/tests/cli/create-import-video-file-job.ts56
-rw-r--r--server/tests/cli/create-transcoding-job.ts95
-rw-r--r--server/tests/helpers/request.ts8
-rw-r--r--shared/extra-utils/miscs/checks.ts7
-rw-r--r--shared/extra-utils/miscs/tests.ts15
-rw-r--r--shared/extra-utils/mock-servers/index.ts1
-rw-r--r--shared/extra-utils/mock-servers/mock-object-storage.ts42
-rw-r--r--shared/extra-utils/requests/requests.ts16
-rw-r--r--shared/extra-utils/server/config-command.ts72
-rw-r--r--shared/extra-utils/server/index.ts1
-rw-r--r--shared/extra-utils/server/jobs-command.ts10
-rw-r--r--shared/extra-utils/server/jobs.ts6
-rw-r--r--shared/extra-utils/server/object-storage-command.ts77
-rw-r--r--shared/extra-utils/server/server.ts18
-rw-r--r--shared/extra-utils/server/servers.ts4
-rw-r--r--shared/extra-utils/videos/live-command.ts2
-rw-r--r--shared/extra-utils/videos/live.ts7
-rw-r--r--shared/extra-utils/videos/streaming-playlists-command.ts6
-rw-r--r--shared/extra-utils/videos/streaming-playlists.ts7
-rw-r--r--shared/extra-utils/videos/videos-command.ts11
-rw-r--r--shared/models/server/job.model.ts6
-rw-r--r--shared/models/videos/index.ts1
-rw-r--r--shared/models/videos/video-state.enum.ts3
-rw-r--r--shared/models/videos/video-storage.enum.ts4
-rw-r--r--support/docker/production/config/custom-environment-variables.yaml23
-rw-r--r--yarn.lock780
86 files changed, 3585 insertions, 573 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c5bbd9e2c..35c91bf85 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -31,6 +31,11 @@ jobs:
31 ports: 31 ports:
32 - 10389:10389 32 - 10389:10389
33 33
34 s3ninja:
35 image: scireum/s3-ninja
36 ports:
37 - 9444:9000
38
34 strategy: 39 strategy:
35 fail-fast: false 40 fail-fast: false
36 matrix: 41 matrix:
@@ -40,6 +45,7 @@ jobs:
40 PGUSER: peertube 45 PGUSER: peertube
41 PGHOST: localhost 46 PGHOST: localhost
42 NODE_PENDING_JOB_WAIT: 250 47 NODE_PENDING_JOB_WAIT: 250
48 ENABLE_OBJECT_STORAGE_TESTS: true
43 49
44 steps: 50 steps:
45 - uses: actions/checkout@v2 51 - uses: actions/checkout@v2
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index 29ba95c5c..4b02e1bc1 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -36,7 +36,8 @@ export class JobsComponent extends RestTable implements OnInit {
36 'video-live-ending', 36 'video-live-ending',
37 'video-redundancy', 37 'video-redundancy',
38 'video-transcoding', 38 'video-transcoding',
39 'videos-views' 39 'videos-views',
40 'move-to-object-storage'
40 ] 41 ]
41 42
42 jobs: Job[] = [] 43 jobs: Job[] = []
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
index 3480d3656..e2dd44bf7 100644
--- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
+++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.html
@@ -6,6 +6,10 @@
6 The video is being transcoded, it may not work properly. 6 The video is being transcoded, it may not work properly.
7</div> 7</div>
8 8
9<div i18n class="alert alert-warning" *ngIf="isVideoToMoveToExternalStorage()">
10 The video is being moved to an external server, it may not work properly.
11</div>
12
9<div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()"> 13<div i18n class="alert alert-info" *ngIf="hasVideoScheduledPublication()">
10 This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}. 14 This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}.
11</div> 15</div>
diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
index 8a46ba0d5..0072492ac 100644
--- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
+++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts
@@ -18,6 +18,10 @@ export class VideoAlertComponent {
18 return this.video && this.video.state.id === VideoState.TO_IMPORT 18 return this.video && this.video.state.id === VideoState.TO_IMPORT
19 } 19 }
20 20
21 isVideoToMoveToExternalStorage () {
22 return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
23 }
24
21 hasVideoScheduledPublication () { 25 hasVideoScheduledPublication () {
22 return this.video && this.video.scheduledUpdate !== undefined 26 return this.video && this.video.scheduledUpdate !== undefined
23 } 27 }
diff --git a/config/default.yaml b/config/default.yaml
index 3d0ae6e87..3865ab5cf 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -95,6 +95,39 @@ storage:
95 # If not, peertube will fallback to the default fil 95 # If not, peertube will fallback to the default fil
96 client_overrides: 'storage/client-overrides/' 96 client_overrides: 'storage/client-overrides/'
97 97
98object_storage:
99 enabled: false
100
101 # Without protocol, will default to HTTPS
102 endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example
103
104 region: 'us-east-1'
105
106 credentials:
107 # You can also use AWS_ACCESS_KEY_ID env variable
108 access_key_id: ''
109 # You can also use AWS_SECRET_ACCESS_KEY env variable
110 secret_access_key: ''
111
112 # Maximum amount to upload in one request to object storage
113 max_upload_part: 2GB
114
115 streaming_playlists:
116 bucket_name: 'streaming-playlists'
117
118 # Allows setting all buckets to the same value but with a different prefix
119 prefix: '' # Example: 'streaming-playlists:'
120
121 # Base url for object URL generation, scheme and host will be replaced by this URL
122 # Useful when you want to use a CDN/external proxy
123 base_url: '' # Example: 'https://mirror.example.com'
124
125 # Same settings but for webtorrent videos
126 videos:
127 bucket_name: 'videos'
128 prefix: ''
129 base_url: ''
130
98log: 131log:
99 level: 'info' # 'debug' | 'info' | 'warn' | 'error' 132 level: 'info' # 'debug' | 'info' | 'warn' | 'error'
100 rotation: 133 rotation:
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 514ab99a4..94238fad0 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -93,6 +93,39 @@ storage:
93 # If not, peertube will fallback to the default file 93 # If not, peertube will fallback to the default file
94 client_overrides: '/var/www/peertube/storage/client-overrides/' 94 client_overrides: '/var/www/peertube/storage/client-overrides/'
95 95
96object_storage:
97 enabled: false
98
99 # Without protocol, will default to HTTPS
100 endpoint: '' # 's3.amazonaws.com' or 's3.fr-par.scw.cloud' for example
101
102 region: 'us-east-1'
103
104 credentials:
105 # You can also use AWS_ACCESS_KEY_ID env variable
106 access_key_id: ''
107 # You can also use AWS_SECRET_ACCESS_KEY env variable
108 secret_access_key: ''
109
110 # Maximum amount to upload in one request to object storage
111 max_upload_part: 2GB
112
113 streaming_playlists:
114 bucket_name: 'streaming-playlists'
115
116 # Allows setting all buckets to the same value but with a different prefix
117 prefix: '' # Example: 'streaming-playlists:'
118
119 # Base url for object URL generation, scheme and host will be replaced by this URL
120 # Useful when you want to use a CDN/external proxy
121 base_url: '' # Example: 'https://mirror.example.com'
122
123 # Same settings but for webtorrent videos
124 videos:
125 bucket_name: 'videos'
126 prefix: ''
127 base_url: ''
128
96log: 129log:
97 level: 'info' # 'debug' | 'info' | 'warn' | 'error' 130 level: 'info' # 'debug' | 'info' | 'warn' | 'error'
98 rotation: 131 rotation:
diff --git a/package.json b/package.json
index 807a90742..d2b1e1245 100644
--- a/package.json
+++ b/package.json
@@ -73,6 +73,7 @@
73 "swagger-cli": "swagger-cli" 73 "swagger-cli": "swagger-cli"
74 }, 74 },
75 "dependencies": { 75 "dependencies": {
76 "@aws-sdk/client-s3": "^3.23.0",
76 "@uploadx/core": "^4.4.0", 77 "@uploadx/core": "^4.4.0",
77 "async": "^3.0.1", 78 "async": "^3.0.1",
78 "async-lru": "^1.1.1", 79 "async-lru": "^1.1.1",
diff --git a/scripts/ci.sh b/scripts/ci.sh
index 71b1be53b..f49dbe6ad 100755
--- a/scripts/ci.sh
+++ b/scripts/ci.sh
@@ -89,9 +89,10 @@ elif [ "$1" = "api-4" ]; then
89 89
90 moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation) 90 moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation)
91 redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy) 91 redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy)
92 objectStorageFiles=$(findTestFiles ./dist/server/tests/api/object-storage)
92 activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub) 93 activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub)
93 94
94 MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles 95 MOCHA_PARALLEL=true TS_NODE_FILES=true runTest "$1" 2 $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles
95elif [ "$1" = "external-plugins" ]; then 96elif [ "$1" = "external-plugins" ]; then
96 npm run build:server 97 npm run build:server
97 98
diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts
index 3a552c19a..0bb9bfeab 100755
--- a/scripts/create-transcoding-job.ts
+++ b/scripts/create-transcoding-job.ts
@@ -6,9 +6,10 @@ import { VideoModel } from '../server/models/video/video'
6import { initDatabaseModels } from '../server/initializers/database' 6import { initDatabaseModels } from '../server/initializers/database'
7import { JobQueue } from '../server/lib/job-queue' 7import { JobQueue } from '../server/lib/job-queue'
8import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils' 8import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
9import { VideoTranscodingPayload } from '@shared/models' 9import { VideoState, VideoTranscodingPayload } from '@shared/models'
10import { CONFIG } from '@server/initializers/config' 10import { CONFIG } from '@server/initializers/config'
11import { isUUIDValid } from '@server/helpers/custom-validators/misc' 11import { isUUIDValid } from '@server/helpers/custom-validators/misc'
12import { addTranscodingJob } from '@server/lib/video'
12 13
13program 14program
14 .option('-v, --video [videoUUID]', 'Video UUID') 15 .option('-v, --video [videoUUID]', 'Video UUID')
@@ -47,7 +48,7 @@ async function run () {
47 if (!video) throw new Error('Video not found.') 48 if (!video) throw new Error('Video not found.')
48 49
49 const dataInput: VideoTranscodingPayload[] = [] 50 const dataInput: VideoTranscodingPayload[] = []
50 const { resolution } = await video.getMaxQualityResolution() 51 const resolution = video.getMaxQualityFile().resolution
51 52
52 // Generate HLS files 53 // Generate HLS files
53 if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { 54 if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
@@ -62,6 +63,7 @@ async function run () {
62 resolution, 63 resolution,
63 isPortraitMode: false, 64 isPortraitMode: false,
64 copyCodecs: false, 65 copyCodecs: false,
66 isNewVideo: false,
65 isMaxQuality: false 67 isMaxQuality: false
66 }) 68 })
67 } 69 }
@@ -87,10 +89,13 @@ async function run () {
87 } 89 }
88 } 90 }
89 91
90 await JobQueue.Instance.init() 92 JobQueue.Instance.init()
93
94 video.state = VideoState.TO_TRANSCODE
95 await video.save()
91 96
92 for (const d of dataInput) { 97 for (const d of dataInput) {
93 await JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: d }) 98 await addTranscodingJob(d, {})
94 console.log('Transcoding job for video %s created.', video.uuid) 99 console.log('Transcoding job for video %s created.', video.uuid)
95 } 100 }
96} 101}
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts
index 9e66105dd..245e4cf28 100644
--- a/scripts/optimize-old-videos.ts
+++ b/scripts/optimize-old-videos.ts
@@ -1,15 +1,18 @@
1import { registerTSPaths } from '../server/helpers/register-ts-paths' 1import { registerTSPaths } from '../server/helpers/register-ts-paths'
2registerTSPaths() 2registerTSPaths()
3 3
4import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
5import { VideoModel } from '../server/models/video/video'
6import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
7import { initDatabaseModels } from '../server/initializers/database'
8import { basename, dirname } from 'path'
9import { copy, move, remove } from 'fs-extra' 4import { copy, move, remove } from 'fs-extra'
5import { basename, dirname } from 'path'
10import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
11import { getVideoFilePath } from '@server/lib/video-paths' 7import { CONFIG } from '@server/initializers/config'
8import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage'
9import { VideoPathManager } from '@server/lib/video-path-manager'
12import { getMaxBitrate } from '@shared/core-utils' 10import { getMaxBitrate } from '@shared/core-utils'
11import { MoveObjectStoragePayload } from '@shared/models'
12import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils'
13import { initDatabaseModels } from '../server/initializers/database'
14import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding'
15import { VideoModel } from '../server/models/video/video'
13 16
14run() 17run()
15 .then(() => process.exit(0)) 18 .then(() => process.exit(0))
@@ -39,43 +42,49 @@ async function run () {
39 currentVideoId = video.id 42 currentVideoId = video.id
40 43
41 for (const file of video.VideoFiles) { 44 for (const file of video.VideoFiles) {
42 currentFilePath = getVideoFilePath(video, file) 45 await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => {
43 46 currentFilePath = path
44 const [ videoBitrate, fps, dataResolution ] = await Promise.all([ 47
45 getVideoFileBitrate(currentFilePath), 48 const [ videoBitrate, fps, dataResolution ] = await Promise.all([
46 getVideoFileFPS(currentFilePath), 49 getVideoFileBitrate(currentFilePath),
47 getVideoFileResolution(currentFilePath) 50 getVideoFileFPS(currentFilePath),
48 ]) 51 getVideoFileResolution(currentFilePath)
49 52 ])
50 const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) 53
51 const isMaxBitrateExceeded = videoBitrate > maxBitrate 54 const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
52 if (isMaxBitrateExceeded) { 55 const isMaxBitrateExceeded = videoBitrate > maxBitrate
53 console.log( 56 if (isMaxBitrateExceeded) {
54 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', 57 console.log(
55 basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 58 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
56 ) 59 basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
57 60 )
58 const backupFile = `${currentFilePath}_backup` 61
59 await copy(currentFilePath, backupFile) 62 const backupFile = `${currentFilePath}_backup`
60 63 await copy(currentFilePath, backupFile)
61 await optimizeOriginalVideofile(video, file) 64
62 // Update file path, the video filename changed 65 await optimizeOriginalVideofile(video, file)
63 currentFilePath = getVideoFilePath(video, file) 66 // Update file path, the video filename changed
64 67 currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
65 const originalDuration = await getDurationFromVideoFile(backupFile) 68
66 const newDuration = await getDurationFromVideoFile(currentFilePath) 69 const originalDuration = await getDurationFromVideoFile(backupFile)
67 70 const newDuration = await getDurationFromVideoFile(currentFilePath)
68 if (originalDuration === newDuration) { 71
69 console.log('Finished optimizing %s', basename(currentFilePath)) 72 if (originalDuration === newDuration) {
70 await remove(backupFile) 73 console.log('Finished optimizing %s', basename(currentFilePath))
71 continue 74 await remove(backupFile)
75 return
76 }
77
78 console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
79 await move(backupFile, currentFilePath, { overwrite: true })
80 await createTorrentAndSetInfoHash(video, file)
81 await file.save()
72 } 82 }
83 })
84 }
73 85
74 console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) 86 if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
75 await move(backupFile, currentFilePath, { overwrite: true }) 87 await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any)
76 await createTorrentAndSetInfoHash(video, file)
77 await file.save()
78 }
79 } 88 }
80 } 89 }
81 90
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 89f50714d..5c740c041 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -1,12 +1,21 @@
1import * as express from 'express' 1import * as express from 'express'
2import { move } from 'fs-extra' 2import { move } from 'fs-extra'
3import { basename } from 'path'
3import { getLowercaseExtension } from '@server/helpers/core-utils' 4import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' 5import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
5import { uuidToShort } from '@server/helpers/uuid' 6import { uuidToShort } from '@server/helpers/uuid'
6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 7import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
7import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 8import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
8import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' 9import { generateWebTorrentVideoFilename } from '@server/lib/paths'
9import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 10import {
11 addMoveToObjectStorageJob,
12 addOptimizeOrMergeAudioJob,
13 buildLocalVideoFromReq,
14 buildVideoThumbnailsFromReq,
15 setVideoTags
16} from '@server/lib/video'
17import { VideoPathManager } from '@server/lib/video-path-manager'
18import { buildNextVideoState } from '@server/lib/video-state'
10import { openapiOperationDoc } from '@server/middlewares/doc' 19import { openapiOperationDoc } from '@server/middlewares/doc'
11import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 20import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
12import { uploadx } from '@uploadx/core' 21import { uploadx } from '@uploadx/core'
@@ -139,23 +148,20 @@ async function addVideo (options: {
139 148
140 const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) 149 const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
141 150
142 videoData.state = CONFIG.TRANSCODING.ENABLED 151 videoData.state = buildNextVideoState()
143 ? VideoState.TO_TRANSCODE
144 : VideoState.PUBLISHED
145
146 videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware 152 videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
147 153
148 const video = new VideoModel(videoData) as MVideoFullLight 154 const video = new VideoModel(videoData) as MVideoFullLight
149 video.VideoChannel = videoChannel 155 video.VideoChannel = videoChannel
150 video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object 156 video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
151 157
152 const videoFile = await buildNewFile(video, videoPhysicalFile) 158 const videoFile = await buildNewFile(videoPhysicalFile)
153 159
154 // Move physical file 160 // Move physical file
155 const destination = getVideoFilePath(video, videoFile) 161 const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
156 await move(videoPhysicalFile.path, destination) 162 await move(videoPhysicalFile.path, destination)
157 // This is important in case if there is another attempt in the retry process 163 // This is important in case if there is another attempt in the retry process
158 videoPhysicalFile.filename = getVideoFilePath(video, videoFile) 164 videoPhysicalFile.filename = basename(destination)
159 videoPhysicalFile.path = destination 165 videoPhysicalFile.path = destination
160 166
161 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ 167 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
@@ -210,9 +216,13 @@ async function addVideo (options: {
210 216
211 createTorrentFederate(video, videoFile) 217 createTorrentFederate(video, videoFile)
212 .then(() => { 218 .then(() => {
213 if (video.state !== VideoState.TO_TRANSCODE) return 219 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
220 return addMoveToObjectStorageJob(video)
221 }
214 222
215 return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) 223 if (video.state === VideoState.TO_TRANSCODE) {
224 return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
225 }
216 }) 226 })
217 .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) 227 .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
218 228
@@ -227,7 +237,7 @@ async function addVideo (options: {
227 }) 237 })
228} 238}
229 239
230async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { 240async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) {
231 const videoFile = new VideoFileModel({ 241 const videoFile = new VideoFileModel({
232 extname: getLowercaseExtension(videoPhysicalFile.filename), 242 extname: getLowercaseExtension(videoPhysicalFile.filename),
233 size: videoPhysicalFile.size, 243 size: videoPhysicalFile.size,
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index ddacc1b68..ffe40d57e 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -3,9 +3,9 @@ import * as express from 'express'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' 4import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
5import { Hooks } from '@server/lib/plugins/hooks' 5import { Hooks } from '@server/lib/plugins/hooks'
6import { getVideoFilePath } from '@server/lib/video-paths' 6import { VideoPathManager } from '@server/lib/video-path-manager'
7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
8import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' 8import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' 9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
10import { asyncMiddleware, videosDownloadValidator } from '../middlewares' 10import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
11 11
@@ -81,7 +81,15 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
81 81
82 if (!checkAllowResult(res, allowParameters, allowedResult)) return 82 if (!checkAllowResult(res, allowParameters, allowedResult)) return
83 83
84 return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) 84 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
85 return res.redirect(videoFile.getObjectStorageUrl())
86 }
87
88 await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => {
89 const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}`
90
91 return res.download(path, filename)
92 })
85} 93}
86 94
87async function downloadHLSVideoFile (req: express.Request, res: express.Response) { 95async function downloadHLSVideoFile (req: express.Request, res: express.Response) {
@@ -107,8 +115,15 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
107 115
108 if (!checkAllowResult(res, allowParameters, allowedResult)) return 116 if (!checkAllowResult(res, allowParameters, allowedResult)) return
109 117
110 const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` 118 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
111 return res.download(getVideoFilePath(streamingPlaylist, videoFile), filename) 119 return res.redirect(videoFile.getObjectStorageUrl())
120 }
121
122 await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => {
123 const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
124
125 return res.download(path, filename)
126 })
112} 127}
113 128
114function getVideoFile (req: express.Request, files: MVideoFile[]) { 129function getVideoFile (req: express.Request, files: MVideoFile[]) {
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index ecf63e93e..c84376304 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -6,7 +6,8 @@ import { dirname, join } from 'path'
6import * as WebTorrent from 'webtorrent' 6import * as WebTorrent from 'webtorrent'
7import { isArray } from '@server/helpers/custom-validators/misc' 7import { isArray } from '@server/helpers/custom-validators/misc'
8import { WEBSERVER } from '@server/initializers/constants' 8import { WEBSERVER } from '@server/initializers/constants'
9import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' 9import { generateTorrentFileName } from '@server/lib/paths'
10import { VideoPathManager } from '@server/lib/video-path-manager'
10import { MVideo } from '@server/types/models/video/video' 11import { MVideo } from '@server/types/models/video/video'
11import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' 12import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
12import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' 13import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
@@ -78,7 +79,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
78 }) 79 })
79} 80}
80 81
81async function createTorrentAndSetInfoHash ( 82function createTorrentAndSetInfoHash (
82 videoOrPlaylist: MVideo | MStreamingPlaylistVideo, 83 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
83 videoFile: MVideoFile 84 videoFile: MVideoFile
84) { 85) {
@@ -95,22 +96,24 @@ async function createTorrentAndSetInfoHash (
95 urlList: [ videoFile.getFileUrl(video) ] 96 urlList: [ videoFile.getFileUrl(video) ]
96 } 97 }
97 98
98 const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) 99 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => {
100 const torrent = await createTorrentPromise(videoPath, options)
99 101
100 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) 102 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
101 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) 103 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
102 logger.info('Creating torrent %s.', torrentPath) 104 logger.info('Creating torrent %s.', torrentPath)
103 105
104 await writeFile(torrentPath, torrent) 106 await writeFile(torrentPath, torrent)
105 107
106 // Remove old torrent file if it existed 108 // Remove old torrent file if it existed
107 if (videoFile.hasTorrent()) { 109 if (videoFile.hasTorrent()) {
108 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) 110 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
109 } 111 }
110 112
111 const parsedTorrent = parseTorrent(torrent) 113 const parsedTorrent = parseTorrent(torrent)
112 videoFile.infoHash = parsedTorrent.infoHash 114 videoFile.infoHash = parsedTorrent.infoHash
113 videoFile.torrentFilename = torrentFilename 115 videoFile.torrentFilename = torrentFilename
116 })
114} 117}
115 118
116function generateMagnetUri ( 119function generateMagnetUri (
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index 911734fa0..09f587274 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -153,6 +153,29 @@ function checkConfig () {
153 } 153 }
154 } 154 }
155 155
156 // Object storage
157 if (CONFIG.OBJECT_STORAGE.ENABLED === true) {
158
159 if (!CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME) {
160 return 'videos_bucket should be set when object storage support is enabled.'
161 }
162
163 if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) {
164 return 'streaming_playlists_bucket should be set when object storage support is enabled.'
165 }
166
167 if (
168 CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME &&
169 CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX
170 ) {
171 if (CONFIG.OBJECT_STORAGE.VIDEOS.PREFIX === '') {
172 return 'Object storage bucket prefixes should be set when the same bucket is used for both types of video.'
173 } else {
174 return 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
175 }
176 }
177 }
178
156 return null 179 return null
157} 180}
158 181
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 30a9823b9..0e684eef8 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -73,6 +73,26 @@ const CONFIG = {
73 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')), 73 PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
74 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')) 74 CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides'))
75 }, 75 },
76 OBJECT_STORAGE: {
77 ENABLED: config.get<boolean>('object_storage.enabled'),
78 MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
79 ENDPOINT: config.get<string>('object_storage.endpoint'),
80 REGION: config.get<string>('object_storage.region'),
81 CREDENTIALS: {
82 ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
83 SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
84 },
85 VIDEOS: {
86 BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
87 PREFIX: config.get<string>('object_storage.videos.prefix'),
88 BASE_URL: config.get<string>('object_storage.videos.base_url')
89 },
90 STREAMING_PLAYLISTS: {
91 BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'),
92 PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'),
93 BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url')
94 }
95 },
76 WEBSERVER: { 96 WEBSERVER: {
77 SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http', 97 SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
78 WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws', 98 WS: config.get<boolean>('webserver.https') === true ? 'wss' : 'ws',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5f121d9a4..8a1526ae8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
24 24
25// --------------------------------------------------------------------------- 25// ---------------------------------------------------------------------------
26 26
27const LAST_MIGRATION_VERSION = 655 27const LAST_MIGRATION_VERSION = 660
28 28
29// --------------------------------------------------------------------------- 29// ---------------------------------------------------------------------------
30 30
@@ -147,7 +147,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
147 'videos-views': 1, 147 'videos-views': 1,
148 'activitypub-refresher': 1, 148 'activitypub-refresher': 1,
149 'video-redundancy': 1, 149 'video-redundancy': 1,
150 'video-live-ending': 1 150 'video-live-ending': 1,
151 'move-to-object-storage': 3
151} 152}
152// Excluded keys are jobs that can be configured by admins 153// Excluded keys are jobs that can be configured by admins
153const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { 154const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
@@ -162,7 +163,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
162 'videos-views': 1, 163 'videos-views': 1,
163 'activitypub-refresher': 1, 164 'activitypub-refresher': 1,
164 'video-redundancy': 1, 165 'video-redundancy': 1,
165 'video-live-ending': 10 166 'video-live-ending': 10,
167 'move-to-object-storage': 1
166} 168}
167const JOB_TTL: { [id in JobType]: number } = { 169const JOB_TTL: { [id in JobType]: number } = {
168 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 170 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
@@ -178,7 +180,8 @@ const JOB_TTL: { [id in JobType]: number } = {
178 'videos-views': undefined, // Unlimited 180 'videos-views': undefined, // Unlimited
179 'activitypub-refresher': 60000 * 10, // 10 minutes 181 'activitypub-refresher': 60000 * 10, // 10 minutes
180 'video-redundancy': 1000 * 3600 * 3, // 3 hours 182 'video-redundancy': 1000 * 3600 * 3, // 3 hours
181 'video-live-ending': 1000 * 60 * 10 // 10 minutes 183 'video-live-ending': 1000 * 60 * 10, // 10 minutes
184 'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
182} 185}
183const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 186const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
184 'videos-views': { 187 'videos-views': {
@@ -412,7 +415,8 @@ const VIDEO_STATES: { [ id in VideoState ]: string } = {
412 [VideoState.TO_TRANSCODE]: 'To transcode', 415 [VideoState.TO_TRANSCODE]: 'To transcode',
413 [VideoState.TO_IMPORT]: 'To import', 416 [VideoState.TO_IMPORT]: 'To import',
414 [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', 417 [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
415 [VideoState.LIVE_ENDED]: 'Livestream ended' 418 [VideoState.LIVE_ENDED]: 'Livestream ended',
419 [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage'
416} 420}
417 421
418const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { 422const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 38e7a76d0..0e690f6ae 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -45,6 +45,7 @@ import { VideoTagModel } from '../models/video/video-tag'
45import { VideoViewModel } from '../models/video/video-view' 45import { VideoViewModel } from '../models/video/video-view'
46import { CONFIG } from './config' 46import { CONFIG } from './config'
47import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' 47import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
48import { VideoJobInfoModel } from '@server/models/video/video-job-info'
48 49
49require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 50require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
50 51
@@ -143,7 +144,8 @@ async function initDatabaseModels (silent: boolean) {
143 TrackerModel, 144 TrackerModel,
144 VideoTrackerModel, 145 VideoTrackerModel,
145 PluginModel, 146 PluginModel,
146 ActorCustomPageModel 147 ActorCustomPageModel,
148 VideoJobInfoModel
147 ]) 149 ])
148 150
149 // Check extensions exist in the database 151 // Check extensions exist in the database
diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts
index 1aeb27f2d..ac952a98c 100644
--- a/server/initializers/migrations/0065-video-file-size.ts
+++ b/server/initializers/migrations/0065-video-file-size.ts
@@ -1,7 +1,4 @@
1import * as Sequelize from 'sequelize' 1import * as Sequelize from 'sequelize'
2import { stat } from 'fs-extra'
3import { VideoModel } from '../../models/video/video'
4import { getVideoFilePath } from '@server/lib/video-paths'
5 2
6function up (utils: { 3function up (utils: {
7 transaction: Sequelize.Transaction 4 transaction: Sequelize.Transaction
@@ -9,30 +6,7 @@ function up (utils: {
9 sequelize: Sequelize.Sequelize 6 sequelize: Sequelize.Sequelize
10 db: any 7 db: any
11}): Promise<void> { 8}): Promise<void> {
12 return utils.db.Video.listOwnedAndPopulateAuthorAndTags() 9 throw new Error('Removed, please upgrade from a previous version first.')
13 .then((videos: VideoModel[]) => {
14 const tasks: Promise<any>[] = []
15
16 videos.forEach(video => {
17 video.VideoFiles.forEach(videoFile => {
18 const p = new Promise((res, rej) => {
19 stat(getVideoFilePath(video, videoFile), (err, stats) => {
20 if (err) return rej(err)
21
22 videoFile.size = stats.size
23 videoFile.save().then(res).catch(rej)
24 })
25 })
26
27 tasks.push(p)
28 })
29 })
30
31 return tasks
32 })
33 .then((tasks: Promise<any>[]) => {
34 return Promise.all(tasks)
35 })
36} 10}
37 11
38function down (options) { 12function down (options) {
diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts
new file mode 100644
index 000000000..c815c71c6
--- /dev/null
+++ b/server/initializers/migrations/0660-object-storage.ts
@@ -0,0 +1,58 @@
1import * as Sequelize from 'sequelize'
2import { VideoStorage } from '@shared/models'
3
4async function up (utils: {
5 transaction: Sequelize.Transaction
6 queryInterface: Sequelize.QueryInterface
7 sequelize: Sequelize.Sequelize
8 db: any
9}): Promise<void> {
10 {
11 const query = `
12 CREATE TABLE IF NOT EXISTS "videoJobInfo" (
13 "id" serial,
14 "pendingMove" INTEGER NOT NULL,
15 "pendingTranscode" INTEGER NOT NULL,
16 "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
17 "createdAt" timestamp WITH time zone NOT NULL,
18 "updatedAt" timestamp WITH time zone NOT NULL,
19 PRIMARY KEY ("id")
20 );
21 `
22
23 await utils.sequelize.query(query)
24 }
25
26 {
27 await utils.queryInterface.addColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: true })
28 }
29 {
30 await utils.sequelize.query(
31 `UPDATE "videoFile" SET "storage" = ${VideoStorage.FILE_SYSTEM}`
32 )
33 }
34 {
35 await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false })
36 }
37
38 {
39 await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: true })
40 }
41 {
42 await utils.sequelize.query(
43 `UPDATE "videoStreamingPlaylist" SET "storage" = ${VideoStorage.FILE_SYSTEM}`
44 )
45 }
46 {
47 await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { type: Sequelize.INTEGER, allowNull: false })
48 }
49}
50
51function down (options) {
52 throw new Error('Not implemented.')
53}
54
55export {
56 up,
57 down
58}
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 1fa16295d..bd9ed45a9 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -6,7 +6,7 @@ import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/vide
6import { logger } from '@server/helpers/logger' 6import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video' 7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' 8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths' 9import { generateTorrentFileName } from '@server/lib/paths'
10import { VideoCaptionModel } from '@server/models/video/video-caption' 10import { VideoCaptionModel } from '@server/models/video/video-caption'
11import { VideoFileModel } from '@server/models/video/video-file' 11import { VideoFileModel } from '@server/models/video/video-file'
12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 0e77ab9fa..0828a2d0f 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,4 +1,4 @@
1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, stat, writeFile } from 'fs-extra' 1import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
@@ -8,11 +8,12 @@ import { logger } from '../helpers/logger'
8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' 8import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
9import { generateRandomString } from '../helpers/utils' 9import { generateRandomString } from '../helpers/utils'
10import { CONFIG } from '../initializers/config' 10import { CONFIG } from '../initializers/config'
11import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' 11import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
12import { sequelizeTypescript } from '../initializers/database' 12import { sequelizeTypescript } from '../initializers/database'
13import { VideoFileModel } from '../models/video/video-file' 13import { VideoFileModel } from '../models/video/video-file'
14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
15import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' 15import { getHlsResolutionPlaylistFilename } from './paths'
16import { VideoPathManager } from './video-path-manager'
16 17
17async function updateStreamingPlaylistsInfohashesIfNeeded () { 18async function updateStreamingPlaylistsInfohashesIfNeeded () {
18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() 19 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@@ -31,75 +32,66 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
31} 32}
32 33
33async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 34async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
34 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
35
36 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] 35 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
37 36
38 const masterPlaylistPath = join(directory, playlist.playlistFilename)
39
40 for (const file of playlist.VideoFiles) { 37 for (const file of playlist.VideoFiles) {
41 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) 38 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
42 39
43 // If we did not generated a playlist for this resolution, skip 40 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => {
44 const filePlaylistPath = join(directory, playlistFilename) 41 const size = await getVideoStreamSize(videoFilePath)
45 if (await pathExists(filePlaylistPath) === false) continue
46
47 const videoFilePath = getVideoFilePath(playlist, file)
48 42
49 const size = await getVideoStreamSize(videoFilePath) 43 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
44 const resolution = `RESOLUTION=${size.width}x${size.height}`
50 45
51 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) 46 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
52 const resolution = `RESOLUTION=${size.width}x${size.height}` 47 if (file.fps) line += ',FRAME-RATE=' + file.fps
53 48
54 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` 49 const codecs = await Promise.all([
55 if (file.fps) line += ',FRAME-RATE=' + file.fps 50 getVideoStreamCodec(videoFilePath),
51 getAudioStreamCodec(videoFilePath)
52 ])
56 53
57 const codecs = await Promise.all([ 54 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
58 getVideoStreamCodec(videoFilePath),
59 getAudioStreamCodec(videoFilePath)
60 ])
61 55
62 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` 56 masterPlaylists.push(line)
63 57 masterPlaylists.push(playlistFilename)
64 masterPlaylists.push(line) 58 })
65 masterPlaylists.push(playlistFilename)
66 } 59 }
67 60
68 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 61 await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
62 return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
63 })
69} 64}
70 65
71async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { 66async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
72 const json: { [filename: string]: { [range: string]: string } } = {} 67 const json: { [filename: string]: { [range: string]: string } } = {}
73 68
74 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
75
76 // For all the resolutions available for this video 69 // For all the resolutions available for this video
77 for (const file of playlist.VideoFiles) { 70 for (const file of playlist.VideoFiles) {
78 const rangeHashes: { [range: string]: string } = {} 71 const rangeHashes: { [range: string]: string } = {}
79 72
80 const videoPath = getVideoFilePath(playlist, file) 73 await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => {
81 const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
82
83 // Maybe the playlist is not generated for this resolution yet
84 if (!await pathExists(resolutionPlaylistPath)) continue
85 74
86 const playlistContent = await readFile(resolutionPlaylistPath) 75 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => {
87 const ranges = getRangesFromPlaylist(playlistContent.toString()) 76 const playlistContent = await readFile(resolutionPlaylistPath)
77 const ranges = getRangesFromPlaylist(playlistContent.toString())
88 78
89 const fd = await open(videoPath, 'r') 79 const fd = await open(videoPath, 'r')
90 for (const range of ranges) { 80 for (const range of ranges) {
91 const buf = Buffer.alloc(range.length) 81 const buf = Buffer.alloc(range.length)
92 await read(fd, buf, 0, range.length, range.offset) 82 await read(fd, buf, 0, range.length, range.offset)
93 83
94 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) 84 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
95 } 85 }
96 await close(fd) 86 await close(fd)
97 87
98 const videoFilename = file.filename 88 const videoFilename = file.filename
99 json[videoFilename] = rangeHashes 89 json[videoFilename] = rangeHashes
90 })
91 })
100 } 92 }
101 93
102 const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) 94 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
103 await outputJSON(outputPath, json) 95 await outputJSON(outputPath, json)
104} 96}
105 97
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts
new file mode 100644
index 000000000..a0c58d211
--- /dev/null
+++ b/server/lib/job-queue/handlers/move-to-object-storage.ts
@@ -0,0 +1,114 @@
1import * as Bull from 'bull'
2import { remove } from 'fs-extra'
3import { join } from 'path'
4import { logger } from '@server/helpers/logger'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { CONFIG } from '@server/initializers/config'
7import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
8import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
9import { moveToNextState } from '@server/lib/video-state'
10import { VideoModel } from '@server/models/video/video'
11import { VideoJobInfoModel } from '@server/models/video/video-job-info'
12import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
13import { MoveObjectStoragePayload, VideoStorage } from '../../../../shared'
14
15export async function processMoveToObjectStorage (job: Bull.Job) {
16 const payload = job.data as MoveObjectStoragePayload
17 logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
18
19 const video = await VideoModel.loadWithFiles(payload.videoUUID)
20 // No video, maybe deleted?
21 if (!video) {
22 logger.info('Can\'t process job %d, video does not exist.', job.id)
23 return undefined
24 }
25
26 if (video.VideoFiles) {
27 await moveWebTorrentFiles(video)
28 }
29
30 if (video.VideoStreamingPlaylists) {
31 await moveHLSFiles(video)
32 }
33
34 const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
35 if (pendingMove === 0) {
36 logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
37 await doAfterLastJob(video, payload.isNewVideo)
38 }
39
40 return payload.videoUUID
41}
42
43// ---------------------------------------------------------------------------
44
45async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
46 for (const file of video.VideoFiles) {
47 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
48
49 const fileUrl = await storeWebTorrentFile(file.filename)
50
51 const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
52 await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
53 }
54}
55
56async function moveHLSFiles (video: MVideoWithAllFiles) {
57 for (const playlist of video.VideoStreamingPlaylists) {
58
59 for (const file of playlist.VideoFiles) {
60 if (file.storage !== VideoStorage.FILE_SYSTEM) continue
61
62 // Resolution playlist
63 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
64 await storeHLSFile(playlist, video, playlistFilename)
65
66 // Resolution fragmented file
67 const fileUrl = await storeHLSFile(playlist, video, file.filename)
68
69 const oldPath = join(getHLSDirectory(video), file.filename)
70
71 await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
72 }
73 }
74}
75
76async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
77 for (const playlist of video.VideoStreamingPlaylists) {
78 if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
79
80 // Master playlist
81 playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename)
82 // Sha256 segments file
83 playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename)
84
85 playlist.storage = VideoStorage.OBJECT_STORAGE
86
87 await playlist.save()
88 }
89
90 // Remove empty hls video directory
91 if (video.VideoStreamingPlaylists) {
92 await remove(getHLSDirectory(video))
93 }
94
95 await moveToNextState(video, isNewVideo)
96}
97
98async function onFileMoved (options: {
99 videoOrPlaylist: MVideo | MStreamingPlaylistVideo
100 file: MVideoFile
101 fileUrl: string
102 oldPath: string
103}) {
104 const { videoOrPlaylist, file, fileUrl, oldPath } = options
105
106 file.fileUrl = fileUrl
107 file.storage = VideoStorage.OBJECT_STORAGE
108
109 await createTorrentAndSetInfoHash(videoOrPlaylist, file)
110 await file.save()
111
112 logger.debug('Removing %s because it\'s now on object storage', oldPath)
113 await remove(oldPath)
114}
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 2f4abf730..e8ee1f759 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -2,15 +2,19 @@ import * as Bull from 'bull'
2import { copy, stat } from 'fs-extra' 2import { copy, stat } from 'fs-extra'
3import { getLowercaseExtension } from '@server/helpers/core-utils' 3import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
5import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 5import { CONFIG } from '@server/initializers/config'
6import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
7import { generateWebTorrentVideoFilename } from '@server/lib/paths'
8import { addMoveToObjectStorageJob } from '@server/lib/video'
9import { VideoPathManager } from '@server/lib/video-path-manager'
6import { UserModel } from '@server/models/user/user' 10import { UserModel } from '@server/models/user/user'
7import { MVideoFullLight } from '@server/types/models' 11import { MVideoFullLight } from '@server/types/models'
8import { VideoFileImportPayload } from '@shared/models' 12import { VideoFileImportPayload, VideoStorage } from '@shared/models'
9import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' 13import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
10import { logger } from '../../../helpers/logger' 14import { logger } from '../../../helpers/logger'
11import { VideoModel } from '../../../models/video/video' 15import { VideoModel } from '../../../models/video/video'
12import { VideoFileModel } from '../../../models/video/video-file' 16import { VideoFileModel } from '../../../models/video/video-file'
13import { onNewWebTorrentFileResolution } from './video-transcoding' 17import { createHlsJobIfEnabled } from './video-transcoding'
14 18
15async function processVideoFileImport (job: Bull.Job) { 19async function processVideoFileImport (job: Bull.Job) {
16 const payload = job.data as VideoFileImportPayload 20 const payload = job.data as VideoFileImportPayload
@@ -29,15 +33,19 @@ async function processVideoFileImport (job: Bull.Job) {
29 33
30 const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) 34 const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId)
31 35
32 const newResolutionPayload = { 36 await createHlsJobIfEnabled(user, {
33 type: 'new-resolution-to-webtorrent' as 'new-resolution-to-webtorrent',
34 videoUUID: video.uuid, 37 videoUUID: video.uuid,
35 resolution: data.resolution, 38 resolution: data.resolution,
36 isPortraitMode: data.isPortraitMode, 39 isPortraitMode: data.isPortraitMode,
37 copyCodecs: false, 40 copyCodecs: true,
38 isNewVideo: false 41 isMaxQuality: false
42 })
43
44 if (CONFIG.OBJECT_STORAGE.ENABLED) {
45 await addMoveToObjectStorageJob(video)
46 } else {
47 await federateVideoIfNeeded(video, false)
39 } 48 }
40 await onNewWebTorrentFileResolution(video, user, newResolutionPayload)
41 49
42 return video 50 return video
43} 51}
@@ -72,12 +80,13 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
72 resolution, 80 resolution,
73 extname: fileExt, 81 extname: fileExt,
74 filename: generateWebTorrentVideoFilename(resolution, fileExt), 82 filename: generateWebTorrentVideoFilename(resolution, fileExt),
83 storage: VideoStorage.FILE_SYSTEM,
75 size, 84 size,
76 fps, 85 fps,
77 videoId: video.id 86 videoId: video.id
78 }) 87 })
79 88
80 const outputPath = getVideoFilePath(video, newVideoFile) 89 const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
81 await copy(inputFilePath, outputPath) 90 await copy(inputFilePath, outputPath)
82 91
83 video.VideoFiles.push(newVideoFile) 92 video.VideoFiles.push(newVideoFile)
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index fec553f2b..a5fa204f5 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -4,11 +4,13 @@ import { getLowercaseExtension } from '@server/helpers/core-utils'
4import { retryTransactionWrapper } from '@server/helpers/database-utils' 4import { retryTransactionWrapper } from '@server/helpers/database-utils'
5import { YoutubeDL } from '@server/helpers/youtube-dl' 5import { YoutubeDL } from '@server/helpers/youtube-dl'
6import { isPostImportVideoAccepted } from '@server/lib/moderation' 6import { isPostImportVideoAccepted } from '@server/lib/moderation'
7import { generateWebTorrentVideoFilename } from '@server/lib/paths'
7import { Hooks } from '@server/lib/plugins/hooks' 8import { Hooks } from '@server/lib/plugins/hooks'
8import { ServerConfigManager } from '@server/lib/server-config-manager' 9import { ServerConfigManager } from '@server/lib/server-config-manager'
9import { isAbleToUploadVideo } from '@server/lib/user' 10import { isAbleToUploadVideo } from '@server/lib/user'
10import { addOptimizeOrMergeAudioJob } from '@server/lib/video' 11import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob } from '@server/lib/video'
11import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { buildNextVideoState } from '@server/lib/video-state'
12import { ThumbnailModel } from '@server/models/video/thumbnail' 14import { ThumbnailModel } from '@server/models/video/thumbnail'
13import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' 15import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
14import { 16import {
@@ -25,7 +27,6 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
25import { logger } from '../../../helpers/logger' 27import { logger } from '../../../helpers/logger'
26import { getSecureTorrentName } from '../../../helpers/utils' 28import { getSecureTorrentName } from '../../../helpers/utils'
27import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' 29import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
28import { CONFIG } from '../../../initializers/config'
29import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 30import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
30import { sequelizeTypescript } from '../../../initializers/database' 31import { sequelizeTypescript } from '../../../initializers/database'
31import { VideoModel } from '../../../models/video/video' 32import { VideoModel } from '../../../models/video/video'
@@ -100,7 +101,6 @@ type ProcessFileOptions = {
100} 101}
101async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) { 102async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
102 let tempVideoPath: string 103 let tempVideoPath: string
103 let videoDestFile: string
104 let videoFile: VideoFileModel 104 let videoFile: VideoFileModel
105 105
106 try { 106 try {
@@ -159,7 +159,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
159 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) 159 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
160 160
161 // Move file 161 // Move file
162 videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile) 162 const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
163 await move(tempVideoPath, videoDestFile) 163 await move(tempVideoPath, videoDestFile)
164 tempVideoPath = null // This path is not used anymore 164 tempVideoPath = null // This path is not used anymore
165 165
@@ -204,7 +204,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
204 204
205 // Update video DB object 205 // Update video DB object
206 video.duration = duration 206 video.duration = duration
207 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 207 video.state = buildNextVideoState(video.state)
208 await video.save({ transaction: t }) 208 await video.save({ transaction: t })
209 209
210 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) 210 if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
@@ -245,6 +245,10 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
245 Notifier.Instance.notifyOnNewVideoIfNeeded(video) 245 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
246 } 246 }
247 247
248 if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
249 return addMoveToObjectStorageJob(videoImportUpdated.Video)
250 }
251
248 // Create transcoding jobs? 252 // Create transcoding jobs?
249 if (video.state === VideoState.TO_TRANSCODE) { 253 if (video.state === VideoState.TO_TRANSCODE) {
250 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User) 254 await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User)
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index aa5bd573a..9ccf724c2 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -4,10 +4,11 @@ import { join } from 'path'
4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { ffprobePromise, getAudioStream, getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { VIDEO_LIVE } from '@server/initializers/constants' 5import { VIDEO_LIVE } from '@server/initializers/constants'
6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live' 6import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
7import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveDirectory } from '@server/lib/paths'
7import { generateVideoMiniature } from '@server/lib/thumbnail' 8import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' 9import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
9import { publishAndFederateIfNeeded } from '@server/lib/video' 10import { VideoPathManager } from '@server/lib/video-path-manager'
10import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' 11import { moveToNextState } from '@server/lib/video-state'
11import { VideoModel } from '@server/models/video/video' 12import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 13import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 14import { VideoLiveModel } from '@server/models/video/video-live'
@@ -55,16 +56,15 @@ export {
55// --------------------------------------------------------------------------- 56// ---------------------------------------------------------------------------
56 57
57async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { 58async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
58 const hlsDirectory = getHLSDirectory(video, false) 59 const replayDirectory = VideoPathManager.Instance.getFSHLSOutputPath(video, VIDEO_LIVE.REPLAY_DIRECTORY)
59 const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
60 60
61 const rootFiles = await readdir(hlsDirectory) 61 const rootFiles = await readdir(getLiveDirectory(video))
62 62
63 const playlistFiles = rootFiles.filter(file => { 63 const playlistFiles = rootFiles.filter(file => {
64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename 64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
65 }) 65 })
66 66
67 await cleanupLiveFiles(hlsDirectory) 67 await cleanupTMPLiveFiles(getLiveDirectory(video))
68 68
69 await live.destroy() 69 await live.destroy()
70 70
@@ -98,7 +98,7 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt
98 98
99 const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe) 99 const { resolution, isPortraitMode } = await getVideoFileResolution(concatenatedTsFilePath, probe)
100 100
101 const outputPath = await generateHlsPlaylistResolutionFromTS({ 101 const { resolutionPlaylistPath: outputPath } = await generateHlsPlaylistResolutionFromTS({
102 video: videoWithFiles, 102 video: videoWithFiles,
103 concatenatedTsFilePath, 103 concatenatedTsFilePath,
104 resolution, 104 resolution,
@@ -133,10 +133,10 @@ async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MSt
133 }) 133 })
134 } 134 }
135 135
136 await publishAndFederateIfNeeded(videoWithFiles, true) 136 await moveToNextState(videoWithFiles, false)
137} 137}
138 138
139async function cleanupLiveFiles (hlsDirectory: string) { 139async function cleanupTMPLiveFiles (hlsDirectory: string) {
140 if (!await pathExists(hlsDirectory)) return 140 if (!await pathExists(hlsDirectory)) return
141 141
142 const files = await readdir(hlsDirectory) 142 const files = await readdir(hlsDirectory)
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 876d1460c..b3149dde8 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,9 +1,11 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils' 2import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils'
3import { getTranscodingJobPriority, publishAndFederateIfNeeded } from '@server/lib/video' 3import { addTranscodingJob, getTranscodingJobPriority } from '@server/lib/video'
4import { getVideoFilePath } from '@server/lib/video-paths' 4import { VideoPathManager } from '@server/lib/video-path-manager'
5import { moveToNextState } from '@server/lib/video-state'
5import { UserModel } from '@server/models/user/user' 6import { UserModel } from '@server/models/user/user'
6import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models' 7import { VideoJobInfoModel } from '@server/models/video/video-job-info'
8import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models'
7import { 9import {
8 HLSTranscodingPayload, 10 HLSTranscodingPayload,
9 MergeAudioTranscodingPayload, 11 MergeAudioTranscodingPayload,
@@ -16,17 +18,14 @@ import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
16import { logger } from '../../../helpers/logger' 18import { logger } from '../../../helpers/logger'
17import { CONFIG } from '../../../initializers/config' 19import { CONFIG } from '../../../initializers/config'
18import { VideoModel } from '../../../models/video/video' 20import { VideoModel } from '../../../models/video/video'
19import { federateVideoIfNeeded } from '../../activitypub/videos'
20import { Notifier } from '../../notifier'
21import { 21import {
22 generateHlsPlaylistResolution, 22 generateHlsPlaylistResolution,
23 mergeAudioVideofile, 23 mergeAudioVideofile,
24 optimizeOriginalVideofile, 24 optimizeOriginalVideofile,
25 transcodeNewWebTorrentResolution 25 transcodeNewWebTorrentResolution
26} from '../../transcoding/video-transcoding' 26} from '../../transcoding/video-transcoding'
27import { JobQueue } from '../job-queue'
28 27
29type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any> 28type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
30 29
31const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { 30const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = {
32 'new-resolution-to-hls': handleHLSJob, 31 'new-resolution-to-hls': handleHLSJob,
@@ -69,15 +68,16 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide
69 : video.getMaxQualityFile() 68 : video.getMaxQualityFile()
70 69
71 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() 70 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
72 const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput)
73 71
74 await generateHlsPlaylistResolution({ 72 await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => {
75 video, 73 return generateHlsPlaylistResolution({
76 videoInputPath, 74 video,
77 resolution: payload.resolution, 75 videoInputPath,
78 copyCodecs: payload.copyCodecs, 76 resolution: payload.resolution,
79 isPortraitMode: payload.isPortraitMode || false, 77 copyCodecs: payload.copyCodecs,
80 job 78 isPortraitMode: payload.isPortraitMode || false,
79 job
80 })
81 }) 81 })
82 82
83 await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload) 83 await retryTransactionWrapper(onHlsPlaylistGeneration, video, user, payload)
@@ -101,7 +101,7 @@ async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudio
101} 101}
102 102
103async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { 103async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
104 const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job) 104 const { transcodeType } = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job)
105 105
106 await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user) 106 await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user)
107} 107}
@@ -121,10 +121,18 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
121 video.VideoFiles = [] 121 video.VideoFiles = []
122 122
123 // Create HLS new resolution jobs 123 // Create HLS new resolution jobs
124 await createLowerResolutionsJobs(video, user, payload.resolution, payload.isPortraitMode, 'hls') 124 await createLowerResolutionsJobs({
125 video,
126 user,
127 videoFileResolution: payload.resolution,
128 isPortraitMode: payload.isPortraitMode,
129 isNewVideo: payload.isNewVideo ?? true,
130 type: 'hls'
131 })
125 } 132 }
126 133
127 return publishAndFederateIfNeeded(video) 134 await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
135 await moveToNextState(video, payload.isNewVideo)
128} 136}
129 137
130async function onVideoFileOptimizer ( 138async function onVideoFileOptimizer (
@@ -143,58 +151,54 @@ async function onVideoFileOptimizer (
143 // Video does not exist anymore 151 // Video does not exist anymore
144 if (!videoDatabase) return undefined 152 if (!videoDatabase) return undefined
145 153
146 let videoPublished = false
147
148 // Generate HLS version of the original file 154 // Generate HLS version of the original file
149 const originalFileHLSPayload = Object.assign({}, payload, { 155 const originalFileHLSPayload = {
156 ...payload,
157
150 isPortraitMode, 158 isPortraitMode,
151 resolution: videoDatabase.getMaxQualityFile().resolution, 159 resolution: videoDatabase.getMaxQualityFile().resolution,
152 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues 160 // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
153 copyCodecs: transcodeType !== 'quick-transcode', 161 copyCodecs: transcodeType !== 'quick-transcode',
154 isMaxQuality: true 162 isMaxQuality: true
155 }) 163 }
156 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) 164 const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
165 const hasNewResolutions = await createLowerResolutionsJobs({
166 video: videoDatabase,
167 user,
168 videoFileResolution: resolution,
169 isPortraitMode,
170 type: 'webtorrent',
171 isNewVideo: payload.isNewVideo ?? true
172 })
157 173
158 const hasNewResolutions = await createLowerResolutionsJobs(videoDatabase, user, resolution, isPortraitMode, 'webtorrent') 174 await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
159 175
176 // Move to next state if there are no other resolutions to generate
160 if (!hasHls && !hasNewResolutions) { 177 if (!hasHls && !hasNewResolutions) {
161 // No transcoding to do, it's now published 178 await moveToNextState(videoDatabase, payload.isNewVideo)
162 videoPublished = await videoDatabase.publishIfNeededAndSave(undefined)
163 } 179 }
164
165 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo)
166
167 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
168 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
169} 180}
170 181
171async function onNewWebTorrentFileResolution ( 182async function onNewWebTorrentFileResolution (
172 video: MVideoUUID, 183 video: MVideo,
173 user: MUserId, 184 user: MUserId,
174 payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload 185 payload: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload
175) { 186) {
176 await publishAndFederateIfNeeded(video) 187 await createHlsJobIfEnabled(user, { ...payload, copyCodecs: true, isMaxQuality: false })
188 await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
177 189
178 await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true, isMaxQuality: false })) 190 await moveToNextState(video, payload.isNewVideo)
179} 191}
180 192
181// ---------------------------------------------------------------------------
182
183export {
184 processVideoTranscoding,
185 onNewWebTorrentFileResolution
186}
187
188// ---------------------------------------------------------------------------
189
190async function createHlsJobIfEnabled (user: MUserId, payload: { 193async function createHlsJobIfEnabled (user: MUserId, payload: {
191 videoUUID: string 194 videoUUID: string
192 resolution: number 195 resolution: number
193 isPortraitMode?: boolean 196 isPortraitMode?: boolean
194 copyCodecs: boolean 197 copyCodecs: boolean
195 isMaxQuality: boolean 198 isMaxQuality: boolean
199 isNewVideo?: boolean
196}) { 200}) {
197 if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false 201 if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false
198 202
199 const jobOptions = { 203 const jobOptions = {
200 priority: await getTranscodingJobPriority(user) 204 priority: await getTranscodingJobPriority(user)
@@ -206,21 +210,35 @@ async function createHlsJobIfEnabled (user: MUserId, payload: {
206 resolution: payload.resolution, 210 resolution: payload.resolution,
207 isPortraitMode: payload.isPortraitMode, 211 isPortraitMode: payload.isPortraitMode,
208 copyCodecs: payload.copyCodecs, 212 copyCodecs: payload.copyCodecs,
209 isMaxQuality: payload.isMaxQuality 213 isMaxQuality: payload.isMaxQuality,
214 isNewVideo: payload.isNewVideo
210 } 215 }
211 216
212 JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions) 217 await addTranscodingJob(hlsTranscodingPayload, jobOptions)
213 218
214 return true 219 return true
215} 220}
216 221
217async function createLowerResolutionsJobs ( 222// ---------------------------------------------------------------------------
218 video: MVideoFullLight, 223
219 user: MUserId, 224export {
220 videoFileResolution: number, 225 processVideoTranscoding,
221 isPortraitMode: boolean, 226 createHlsJobIfEnabled,
227 onNewWebTorrentFileResolution
228}
229
230// ---------------------------------------------------------------------------
231
232async function createLowerResolutionsJobs (options: {
233 video: MVideoFullLight
234 user: MUserId
235 videoFileResolution: number
236 isPortraitMode: boolean
237 isNewVideo: boolean
222 type: 'hls' | 'webtorrent' 238 type: 'hls' | 'webtorrent'
223) { 239}) {
240 const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options
241
224 // Create transcoding jobs if there are enabled resolutions 242 // Create transcoding jobs if there are enabled resolutions
225 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') 243 const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
226 const resolutionCreated: number[] = [] 244 const resolutionCreated: number[] = []
@@ -234,7 +252,8 @@ async function createLowerResolutionsJobs (
234 type: 'new-resolution-to-webtorrent', 252 type: 'new-resolution-to-webtorrent',
235 videoUUID: video.uuid, 253 videoUUID: video.uuid,
236 resolution, 254 resolution,
237 isPortraitMode 255 isPortraitMode,
256 isNewVideo
238 } 257 }
239 } 258 }
240 259
@@ -245,7 +264,8 @@ async function createLowerResolutionsJobs (
245 resolution, 264 resolution,
246 isPortraitMode, 265 isPortraitMode,
247 copyCodecs: false, 266 copyCodecs: false,
248 isMaxQuality: false 267 isMaxQuality: false,
268 isNewVideo
249 } 269 }
250 } 270 }
251 271
@@ -257,7 +277,7 @@ async function createLowerResolutionsJobs (
257 priority: await getTranscodingJobPriority(user) 277 priority: await getTranscodingJobPriority(user)
258 } 278 }
259 279
260 JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions) 280 await addTranscodingJob(dataInput, jobOptions)
261 } 281 }
262 282
263 if (resolutionCreated.length === 0) { 283 if (resolutionCreated.length === 0) {
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 42e8347b1..7a3a1bf82 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -11,6 +11,7 @@ import {
11 EmailPayload, 11 EmailPayload,
12 JobState, 12 JobState,
13 JobType, 13 JobType,
14 MoveObjectStoragePayload,
14 RefreshPayload, 15 RefreshPayload,
15 VideoFileImportPayload, 16 VideoFileImportPayload,
16 VideoImportPayload, 17 VideoImportPayload,
@@ -34,6 +35,7 @@ import { processVideoImport } from './handlers/video-import'
34import { processVideoLiveEnding } from './handlers/video-live-ending' 35import { processVideoLiveEnding } from './handlers/video-live-ending'
35import { processVideoTranscoding } from './handlers/video-transcoding' 36import { processVideoTranscoding } from './handlers/video-transcoding'
36import { processVideosViews } from './handlers/video-views' 37import { processVideosViews } from './handlers/video-views'
38import { processMoveToObjectStorage } from './handlers/move-to-object-storage'
37 39
38type CreateJobArgument = 40type CreateJobArgument =
39 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 41 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -49,9 +51,10 @@ type CreateJobArgument =
49 { type: 'videos-views', payload: {} } | 51 { type: 'videos-views', payload: {} } |
50 { type: 'video-live-ending', payload: VideoLiveEndingPayload } | 52 { type: 'video-live-ending', payload: VideoLiveEndingPayload } |
51 { type: 'actor-keys', payload: ActorKeysPayload } | 53 { type: 'actor-keys', payload: ActorKeysPayload } |
52 { type: 'video-redundancy', payload: VideoRedundancyPayload } 54 { type: 'video-redundancy', payload: VideoRedundancyPayload } |
55 { type: 'move-to-object-storage', payload: MoveObjectStoragePayload }
53 56
54type CreateJobOptions = { 57export type CreateJobOptions = {
55 delay?: number 58 delay?: number
56 priority?: number 59 priority?: number
57} 60}
@@ -70,7 +73,8 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
70 'activitypub-refresher': refreshAPObject, 73 'activitypub-refresher': refreshAPObject,
71 'video-live-ending': processVideoLiveEnding, 74 'video-live-ending': processVideoLiveEnding,
72 'actor-keys': processActorKeys, 75 'actor-keys': processActorKeys,
73 'video-redundancy': processVideoRedundancy 76 'video-redundancy': processVideoRedundancy,
77 'move-to-object-storage': processMoveToObjectStorage
74} 78}
75 79
76const jobTypes: JobType[] = [ 80const jobTypes: JobType[] = [
@@ -87,7 +91,8 @@ const jobTypes: JobType[] = [
87 'activitypub-refresher', 91 'activitypub-refresher',
88 'video-redundancy', 92 'video-redundancy',
89 'actor-keys', 93 'actor-keys',
90 'video-live-ending' 94 'video-live-ending',
95 'move-to-object-storage'
91] 96]
92 97
93class JobQueue { 98class JobQueue {
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 2a429fb33..d7dc841d9 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -20,7 +20,7 @@ import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
20import { federateVideoIfNeeded } from '../activitypub/videos' 20import { federateVideoIfNeeded } from '../activitypub/videos'
21import { JobQueue } from '../job-queue' 21import { JobQueue } from '../job-queue'
22import { PeerTubeSocket } from '../peertube-socket' 22import { PeerTubeSocket } from '../peertube-socket'
23import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' 23import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../paths'
24import { LiveQuotaStore } from './live-quota-store' 24import { LiveQuotaStore } from './live-quota-store'
25import { LiveSegmentShaStore } from './live-segment-sha-store' 25import { LiveSegmentShaStore } from './live-segment-sha-store'
26import { cleanupLive } from './live-utils' 26import { cleanupLive } from './live-utils'
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts
index e4526c7a5..3bf723b98 100644
--- a/server/lib/live/live-utils.ts
+++ b/server/lib/live/live-utils.ts
@@ -1,7 +1,7 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import { basename } from 'path' 2import { basename } from 'path'
3import { MStreamingPlaylist, MVideo } from '@server/types/models' 3import { MStreamingPlaylist, MVideo } from '@server/types/models'
4import { getHLSDirectory } from '../video-paths' 4import { getLiveDirectory } from '../paths'
5 5
6function buildConcatenatedName (segmentOrPlaylistPath: string) { 6function buildConcatenatedName (segmentOrPlaylistPath: string) {
7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) 7 const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/)
@@ -10,7 +10,7 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
10} 10}
11 11
12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { 12async function cleanupLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
13 const hlsDirectory = getHLSDirectory(video) 13 const hlsDirectory = getLiveDirectory(video)
14 14
15 await remove(hlsDirectory) 15 await remove(hlsDirectory)
16 16
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index a80abc843..9b5b6c4fc 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -11,9 +11,9 @@ import { CONFIG } from '@server/initializers/config'
11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants' 11import { MEMOIZE_TTL, VIDEO_LIVE } from '@server/initializers/constants'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' 13import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models'
14import { getLiveDirectory } from '../../paths'
14import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles' 15import { VideoTranscodingProfilesManager } from '../../transcoding/video-transcoding-profiles'
15import { isAbleToUploadVideo } from '../../user' 16import { isAbleToUploadVideo } from '../../user'
16import { getHLSDirectory } from '../../video-paths'
17import { LiveQuotaStore } from '../live-quota-store' 17import { LiveQuotaStore } from '../live-quota-store'
18import { LiveSegmentShaStore } from '../live-segment-sha-store' 18import { LiveSegmentShaStore } from '../live-segment-sha-store'
19import { buildConcatenatedName } from '../live-utils' 19import { buildConcatenatedName } from '../live-utils'
@@ -282,7 +282,7 @@ class MuxingSession extends EventEmitter {
282 } 282 }
283 283
284 private async prepareDirectories () { 284 private async prepareDirectories () {
285 const outPath = getHLSDirectory(this.videoLive.Video) 285 const outPath = getLiveDirectory(this.videoLive.Video)
286 await ensureDir(outPath) 286 await ensureDir(outPath)
287 287
288 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY) 288 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts
new file mode 100644
index 000000000..8b413a40e
--- /dev/null
+++ b/server/lib/object-storage/index.ts
@@ -0,0 +1,3 @@
1export * from './keys'
2export * from './urls'
3export * from './videos'
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts
new file mode 100644
index 000000000..519474775
--- /dev/null
+++ b/server/lib/object-storage/keys.ts
@@ -0,0 +1,20 @@
1import { join } from 'path'
2import { MStreamingPlaylist, MVideoUUID } from '@server/types/models'
3
4function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
5 return join(generateHLSObjectBaseStorageKey(playlist, video), filename)
6}
7
8function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) {
9 return playlist.getStringType() + '_' + video.uuid
10}
11
12function generateWebTorrentObjectStorageKey (filename: string) {
13 return filename
14}
15
16export {
17 generateHLSObjectStorageKey,
18 generateHLSObjectBaseStorageKey,
19 generateWebTorrentObjectStorageKey
20}
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
new file mode 100644
index 000000000..c9a614593
--- /dev/null
+++ b/server/lib/object-storage/shared/client.ts
@@ -0,0 +1,56 @@
1import { S3Client } from '@aws-sdk/client-s3'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { lTags } from './logger'
5
6let endpointParsed: URL
7function getEndpointParsed () {
8 if (endpointParsed) return endpointParsed
9
10 endpointParsed = new URL(getEndpoint())
11
12 return endpointParsed
13}
14
15let s3Client: S3Client
16function getClient () {
17 if (s3Client) return s3Client
18
19 const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE
20
21 s3Client = new S3Client({
22 endpoint: getEndpoint(),
23 region: OBJECT_STORAGE.REGION,
24 credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID
25 ? {
26 accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
27 secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
28 }
29 : undefined
30 })
31
32 logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
33
34 return s3Client
35}
36
37// ---------------------------------------------------------------------------
38
39export {
40 getEndpointParsed,
41 getClient
42}
43
44// ---------------------------------------------------------------------------
45
46let endpoint: string
47function getEndpoint () {
48 if (endpoint) return endpoint
49
50 const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT
51 endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://')
52 ? CONFIG.OBJECT_STORAGE.ENDPOINT
53 : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT
54
55 return endpoint
56}
diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts
new file mode 100644
index 000000000..11e10aa9f
--- /dev/null
+++ b/server/lib/object-storage/shared/index.ts
@@ -0,0 +1,3 @@
1export * from './client'
2export * from './logger'
3export * from './object-storage-helpers'
diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts
new file mode 100644
index 000000000..8ab7cbd71
--- /dev/null
+++ b/server/lib/object-storage/shared/logger.ts
@@ -0,0 +1,7 @@
1import { loggerTagsFactory } from '@server/helpers/logger'
2
3const lTags = loggerTagsFactory('object-storage')
4
5export {
6 lTags
7}
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
new file mode 100644
index 000000000..e23216907
--- /dev/null
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -0,0 +1,229 @@
1import { close, createReadStream, createWriteStream, ensureDir, open, ReadStream, stat } from 'fs-extra'
2import { min } from 'lodash'
3import { dirname } from 'path'
4import { Readable } from 'stream'
5import {
6 CompletedPart,
7 CompleteMultipartUploadCommand,
8 CreateMultipartUploadCommand,
9 DeleteObjectCommand,
10 GetObjectCommand,
11 ListObjectsV2Command,
12 PutObjectCommand,
13 UploadPartCommand
14} from '@aws-sdk/client-s3'
15import { pipelinePromise } from '@server/helpers/core-utils'
16import { isArray } from '@server/helpers/custom-validators/misc'
17import { logger } from '@server/helpers/logger'
18import { CONFIG } from '@server/initializers/config'
19import { getPrivateUrl } from '../urls'
20import { getClient } from './client'
21import { lTags } from './logger'
22
23type BucketInfo = {
24 BUCKET_NAME: string
25 PREFIX?: string
26}
27
28async function storeObject (options: {
29 inputPath: string
30 objectStorageKey: string
31 bucketInfo: BucketInfo
32}): Promise<string> {
33 const { inputPath, objectStorageKey, bucketInfo } = options
34
35 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
36
37 const stats = await stat(inputPath)
38
39 // If bigger than max allowed size we do a multipart upload
40 if (stats.size > CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART) {
41 return multiPartUpload({ inputPath, objectStorageKey, bucketInfo })
42 }
43
44 const fileStream = createReadStream(inputPath)
45 return objectStoragePut({ objectStorageKey, content: fileStream, bucketInfo })
46}
47
48async function removeObject (filename: string, bucketInfo: BucketInfo) {
49 const command = new DeleteObjectCommand({
50 Bucket: bucketInfo.BUCKET_NAME,
51 Key: buildKey(filename, bucketInfo)
52 })
53
54 return getClient().send(command)
55}
56
57async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
58 const s3Client = getClient()
59
60 const commandPrefix = bucketInfo.PREFIX + prefix
61 const listCommand = new ListObjectsV2Command({
62 Bucket: bucketInfo.BUCKET_NAME,
63 Prefix: commandPrefix
64 })
65
66 const listedObjects = await s3Client.send(listCommand)
67
68 // FIXME: use bulk delete when s3ninja will support this operation
69 // const deleteParams = {
70 // Bucket: bucketInfo.BUCKET_NAME,
71 // Delete: { Objects: [] }
72 // }
73
74 if (isArray(listedObjects.Contents) !== true) {
75 const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
76
77 logger.error(message, { response: listedObjects, ...lTags() })
78 throw new Error(message)
79 }
80
81 for (const object of listedObjects.Contents) {
82 const command = new DeleteObjectCommand({
83 Bucket: bucketInfo.BUCKET_NAME,
84 Key: object.Key
85 })
86
87 await s3Client.send(command)
88
89 // FIXME: use bulk delete when s3ninja will support this operation
90 // deleteParams.Delete.Objects.push({ Key: object.Key })
91 }
92
93 // FIXME: use bulk delete when s3ninja will support this operation
94 // const deleteCommand = new DeleteObjectsCommand(deleteParams)
95 // await s3Client.send(deleteCommand)
96
97 // Repeat if not all objects could be listed at once (limit of 1000?)
98 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
99}
100
101async function makeAvailable (options: {
102 key: string
103 destination: string
104 bucketInfo: BucketInfo
105}) {
106 const { key, destination, bucketInfo } = options
107
108 await ensureDir(dirname(options.destination))
109
110 const command = new GetObjectCommand({
111 Bucket: bucketInfo.BUCKET_NAME,
112 Key: buildKey(key, bucketInfo)
113 })
114 const response = await getClient().send(command)
115
116 const file = createWriteStream(destination)
117 await pipelinePromise(response.Body as Readable, file)
118
119 file.close()
120}
121
122function buildKey (key: string, bucketInfo: BucketInfo) {
123 return bucketInfo.PREFIX + key
124}
125
126// ---------------------------------------------------------------------------
127
128export {
129 BucketInfo,
130 buildKey,
131 storeObject,
132 removeObject,
133 removePrefix,
134 makeAvailable
135}
136
137// ---------------------------------------------------------------------------
138
139async function objectStoragePut (options: {
140 objectStorageKey: string
141 content: ReadStream
142 bucketInfo: BucketInfo
143}) {
144 const { objectStorageKey, content, bucketInfo } = options
145
146 const command = new PutObjectCommand({
147 Bucket: bucketInfo.BUCKET_NAME,
148 Key: buildKey(objectStorageKey, bucketInfo),
149 Body: content
150 })
151
152 await getClient().send(command)
153
154 return getPrivateUrl(bucketInfo, objectStorageKey)
155}
156
157async function multiPartUpload (options: {
158 inputPath: string
159 objectStorageKey: string
160 bucketInfo: BucketInfo
161}) {
162 const { objectStorageKey, inputPath, bucketInfo } = options
163
164 const key = buildKey(objectStorageKey, bucketInfo)
165 const s3Client = getClient()
166
167 const statResult = await stat(inputPath)
168
169 const createMultipartCommand = new CreateMultipartUploadCommand({
170 Bucket: bucketInfo.BUCKET_NAME,
171 Key: key
172 })
173 const createResponse = await s3Client.send(createMultipartCommand)
174
175 const fd = await open(inputPath, 'r')
176 let partNumber = 1
177 const parts: CompletedPart[] = []
178 const partSize = CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART
179
180 for (let start = 0; start < statResult.size; start += partSize) {
181 logger.debug(
182 'Uploading part %d of file to %s%s in bucket %s',
183 partNumber, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
184 )
185
186 // FIXME: Remove when https://github.com/aws/aws-sdk-js-v3/pull/2637 is released
187 // The s3 sdk needs to know the length of the http body beforehand, but doesn't support
188 // streams with start and end set, so it just tries to stat the file in stream.path.
189 // This fails for us because we only want to send part of the file. The stream type
190 // is modified so we can set the byteLength here, which s3 detects because array buffers
191 // have this field set
192 const stream: ReadStream & { byteLength: number } =
193 createReadStream(
194 inputPath,
195 { fd, autoClose: false, start, end: (start + partSize) - 1 }
196 ) as ReadStream & { byteLength: number }
197
198 // Calculate if the part size is more than what's left over, and in that case use left over bytes for byteLength
199 stream.byteLength = min([ statResult.size - start, partSize ])
200
201 const uploadPartCommand = new UploadPartCommand({
202 Bucket: bucketInfo.BUCKET_NAME,
203 Key: key,
204 UploadId: createResponse.UploadId,
205 PartNumber: partNumber,
206 Body: stream
207 })
208 const uploadResponse = await s3Client.send(uploadPartCommand)
209
210 parts.push({ ETag: uploadResponse.ETag, PartNumber: partNumber })
211 partNumber += 1
212 }
213 await close(fd)
214
215 const completeUploadCommand = new CompleteMultipartUploadCommand({
216 Bucket: bucketInfo.BUCKET_NAME,
217 Key: objectStorageKey,
218 UploadId: createResponse.UploadId,
219 MultipartUpload: { Parts: parts }
220 })
221 await s3Client.send(completeUploadCommand)
222
223 logger.debug(
224 'Completed %s%s in bucket %s in %d parts',
225 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, partNumber - 1, lTags()
226 )
227
228 return getPrivateUrl(bucketInfo, objectStorageKey)
229}
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts
new file mode 100644
index 000000000..2a889190b
--- /dev/null
+++ b/server/lib/object-storage/urls.ts
@@ -0,0 +1,40 @@
1import { CONFIG } from '@server/initializers/config'
2import { BucketInfo, buildKey, getEndpointParsed } from './shared'
3
4function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) {
5 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
6}
7
8function getWebTorrentPublicFileUrl (fileUrl: string) {
9 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
10 if (!baseUrl) return fileUrl
11
12 return replaceByBaseUrl(fileUrl, baseUrl)
13}
14
15function getHLSPublicFileUrl (fileUrl: string) {
16 const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
17 if (!baseUrl) return fileUrl
18
19 return replaceByBaseUrl(fileUrl, baseUrl)
20}
21
22export {
23 getPrivateUrl,
24 getWebTorrentPublicFileUrl,
25 replaceByBaseUrl,
26 getHLSPublicFileUrl
27}
28
29// ---------------------------------------------------------------------------
30
31function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {
32 if (baseUrl) return baseUrl
33
34 return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`
35}
36
37const regex = new RegExp('https?://[^/]+')
38function replaceByBaseUrl (fileUrl: string, baseUrl: string) {
39 return fileUrl.replace(regex, baseUrl)
40}
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
new file mode 100644
index 000000000..15b8f58d5
--- /dev/null
+++ b/server/lib/object-storage/videos.ts
@@ -0,0 +1,72 @@
1import { join } from 'path'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models'
5import { getHLSDirectory } from '../paths'
6import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
7import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
8
9function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
10 const baseHlsDirectory = getHLSDirectory(video)
11
12 return storeObject({
13 inputPath: join(baseHlsDirectory, filename),
14 objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename),
15 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
16 })
17}
18
19function storeWebTorrentFile (filename: string) {
20 return storeObject({
21 inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
22 objectStorageKey: generateWebTorrentObjectStorageKey(filename),
23 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
24 })
25}
26
27function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) {
28 return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
29}
30
31function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
32 return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
33}
34
35async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) {
36 const key = generateHLSObjectStorageKey(playlist, video, filename)
37
38 logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
39
40 await makeAvailable({
41 key,
42 destination,
43 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
44 })
45
46 return destination
47}
48
49async function makeWebTorrentFileAvailable (filename: string, destination: string) {
50 const key = generateWebTorrentObjectStorageKey(filename)
51
52 logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags())
53
54 await makeAvailable({
55 key,
56 destination,
57 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
58 })
59
60 return destination
61}
62
63export {
64 storeWebTorrentFile,
65 storeHLSFile,
66
67 removeHLSObjectStorage,
68 removeWebTorrentObjectStorage,
69
70 makeWebTorrentFileAvailable,
71 makeHLSFileAvailable
72}
diff --git a/server/lib/video-paths.ts b/server/lib/paths.ts
index 1e4382108..434e637c6 100644
--- a/server/lib/video-paths.ts
+++ b/server/lib/paths.ts
@@ -1,9 +1,8 @@
1import { join } from 'path' 1import { join } from 'path'
2import { extractVideo } from '@server/helpers/video'
3import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
6import { buildUUID } from '@server/helpers/uuid' 2import { buildUUID } from '@server/helpers/uuid'
3import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
7import { removeFragmentedMP4Ext } from '@shared/core-utils' 6import { removeFragmentedMP4Ext } from '@shared/core-utils'
8 7
9// ################## Video file name ################## 8// ################## Video file name ##################
@@ -16,39 +15,18 @@ function generateHLSVideoFilename (resolution: number) {
16 return `${buildUUID()}-${resolution}-fragmented.mp4` 15 return `${buildUUID()}-${resolution}-fragmented.mp4`
17} 16}
18 17
19function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { 18// ################## Streaming playlist ##################
20 if (videoFile.isHLS()) {
21 const video = extractVideo(videoOrPlaylist)
22
23 return join(getHLSDirectory(video), videoFile.filename)
24 }
25
26 const baseDir = isRedundancy
27 ? CONFIG.STORAGE.REDUNDANCY_DIR
28 : CONFIG.STORAGE.VIDEOS_DIR
29
30 return join(baseDir, videoFile.filename)
31}
32
33// ################## Redundancy ##################
34 19
35function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { 20function getLiveDirectory (video: MVideoUUID) {
36 // Base URL used by our HLS player 21 return getHLSDirectory(video)
37 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid
38} 22}
39 23
40function generateWebTorrentRedundancyUrl (file: MVideoFile) { 24function getHLSDirectory (video: MVideoUUID) {
41 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename 25 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
42} 26}
43 27
44// ################## Streaming playlist ################## 28function getHLSRedundancyDirectory (video: MVideoUUID) {
45 29 return join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
46function getHLSDirectory (video: MVideoUUID, isRedundancy = false) {
47 const baseDir = isRedundancy
48 ? HLS_REDUNDANCY_DIRECTORY
49 : HLS_STREAMING_PLAYLIST_DIRECTORY
50
51 return join(baseDir, video.uuid)
52} 30}
53 31
54function getHlsResolutionPlaylistFilename (videoFilename: string) { 32function getHlsResolutionPlaylistFilename (videoFilename: string) {
@@ -81,36 +59,24 @@ function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVi
81 return uuid + '-' + resolution + extension 59 return uuid + '-' + resolution + extension
82} 60}
83 61
84function getTorrentFilePath (videoFile: MVideoFile) { 62function getFSTorrentFilePath (videoFile: MVideoFile) {
85 return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) 63 return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
86} 64}
87 65
88// ################## Meta data ##################
89
90function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) {
91 const path = '/api/v1/videos/'
92
93 return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id
94}
95
96// --------------------------------------------------------------------------- 66// ---------------------------------------------------------------------------
97 67
98export { 68export {
99 generateHLSVideoFilename, 69 generateHLSVideoFilename,
100 generateWebTorrentVideoFilename, 70 generateWebTorrentVideoFilename,
101 71
102 getVideoFilePath,
103
104 generateTorrentFileName, 72 generateTorrentFileName,
105 getTorrentFilePath, 73 getFSTorrentFilePath,
106 74
107 getHLSDirectory, 75 getHLSDirectory,
76 getLiveDirectory,
77 getHLSRedundancyDirectory,
78
108 generateHLSMasterPlaylistFilename, 79 generateHLSMasterPlaylistFilename,
109 generateHlsSha256SegmentsFilename, 80 generateHlsSha256SegmentsFilename,
110 getHlsResolutionPlaylistFilename, 81 getHlsResolutionPlaylistFilename
111
112 getLocalVideoFileMetadataUrl,
113
114 generateWebTorrentRedundancyUrl,
115 generateHLSRedundancyUrl
116} 82}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 137ae53a0..ebfd015b5 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -24,7 +24,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli
24import { getOrCreateAPVideo } from '../activitypub/videos' 24import { getOrCreateAPVideo } from '../activitypub/videos'
25import { downloadPlaylistSegments } from '../hls' 25import { downloadPlaylistSegments } from '../hls'
26import { removeVideoRedundancy } from '../redundancy' 26import { removeVideoRedundancy } from '../redundancy'
27import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' 27import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls'
28import { AbstractScheduler } from './abstract-scheduler' 28import { AbstractScheduler } from './abstract-scheduler'
29 29
30type CandidateToDuplicate = { 30type CandidateToDuplicate = {
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index c08523988..d2384f53c 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -1,5 +1,4 @@
1import { join } from 'path' 1import { join } from 'path'
2
3import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' 2import { ThumbnailType } from '../../shared/models/videos/thumbnail.type'
4import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' 3import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils'
5import { generateImageFilename, processImage } from '../helpers/image-utils' 4import { generateImageFilename, processImage } from '../helpers/image-utils'
@@ -10,7 +9,7 @@ import { ThumbnailModel } from '../models/video/thumbnail'
10import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' 9import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
11import { MThumbnail } from '../types/models/video/thumbnail' 10import { MThumbnail } from '../types/models/video/thumbnail'
12import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' 11import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
13import { getVideoFilePath } from './video-paths' 12import { VideoPathManager } from './video-path-manager'
14 13
15type ImageSize = { height?: number, width?: number } 14type ImageSize = { height?: number, width?: number }
16 15
@@ -116,21 +115,22 @@ function generateVideoMiniature (options: {
116}) { 115}) {
117 const { video, videoFile, type } = options 116 const { video, videoFile, type } = options
118 117
119 const input = getVideoFilePath(video, videoFile) 118 return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => {
119 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
120 120
121 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) 121 const thumbnailCreator = videoFile.isAudio()
122 const thumbnailCreator = videoFile.isAudio() 122 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true)
123 ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) 123 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
124 : () => generateImageFromVideoFile(input, basePath, filename, { height, width })
125 124
126 return updateThumbnailFromFunction({ 125 return updateThumbnailFromFunction({
127 thumbnailCreator, 126 thumbnailCreator,
128 filename, 127 filename,
129 height, 128 height,
130 width, 129 width,
131 type, 130 type,
132 automaticallyGenerated: true, 131 automaticallyGenerated: true,
133 existingThumbnail 132 existingThumbnail
133 })
134 }) 134 })
135} 135}
136 136
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index d2a556360..ee228c011 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -4,13 +4,13 @@ import { basename, extname as extnameUtil, join } from 'path'
4import { toEven } from '@server/helpers/core-utils' 4import { toEven } from '@server/helpers/core-utils'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 6import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
7import { VideoResolution } from '../../../shared/models/videos' 7import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils' 9import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers/ffmpeg-utils'
10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' 10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { CONFIG } from '../../initializers/config' 12import { CONFIG } from '../../initializers/config'
13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' 13import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
14import { VideoFileModel } from '../../models/video/video-file' 14import { VideoFileModel } from '../../models/video/video-file'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' 16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
@@ -19,9 +19,9 @@ import {
19 generateHlsSha256SegmentsFilename, 19 generateHlsSha256SegmentsFilename,
20 generateHLSVideoFilename, 20 generateHLSVideoFilename,
21 generateWebTorrentVideoFilename, 21 generateWebTorrentVideoFilename,
22 getHlsResolutionPlaylistFilename, 22 getHlsResolutionPlaylistFilename
23 getVideoFilePath 23} from '../paths'
24} from '../video-paths' 24import { VideoPathManager } from '../video-path-manager'
25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' 25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
26 26
27/** 27/**
@@ -32,159 +32,162 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
32 */ 32 */
33 33
34// Optimize the original video file and replace it. The resolution is not changed. 34// Optimize the original video file and replace it. The resolution is not changed.
35async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { 35function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) {
36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 36 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
37 const newExtname = '.mp4' 37 const newExtname = '.mp4'
38 38
39 const videoInputPath = getVideoFilePath(video, inputVideoFile) 39 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => {
40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 40 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
41 41
42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) 42 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
43 ? 'quick-transcode' 43 ? 'quick-transcode'
44 : 'video' 44 : 'video'
45 45
46 const resolution = toEven(inputVideoFile.resolution) 46 const resolution = toEven(inputVideoFile.resolution)
47 47
48 const transcodeOptions: TranscodeOptions = { 48 const transcodeOptions: TranscodeOptions = {
49 type: transcodeType, 49 type: transcodeType,
50 50
51 inputPath: videoInputPath, 51 inputPath: videoInputPath,
52 outputPath: videoTranscodedPath, 52 outputPath: videoTranscodedPath,
53 53
54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 54 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
55 profile: CONFIG.TRANSCODING.PROFILE, 55 profile: CONFIG.TRANSCODING.PROFILE,
56 56
57 resolution, 57 resolution,
58 58
59 job 59 job
60 } 60 }
61 61
62 // Could be very long! 62 // Could be very long!
63 await transcode(transcodeOptions) 63 await transcode(transcodeOptions)
64 64
65 try { 65 try {
66 await remove(videoInputPath) 66 await remove(videoInputPath)
67 67
68 // Important to do this before getVideoFilename() to take in account the new filename 68 // Important to do this before getVideoFilename() to take in account the new filename
69 inputVideoFile.extname = newExtname 69 inputVideoFile.extname = newExtname
70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) 70 inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
71 inputVideoFile.storage = VideoStorage.FILE_SYSTEM
71 72
72 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 73 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
73 74
74 await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 75 const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
75 76
76 return transcodeType 77 return { transcodeType, videoFile }
77 } catch (err) { 78 } catch (err) {
78 // Auto destruction... 79 // Auto destruction...
79 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err })) 80 video.destroy().catch(err => logger.error('Cannot destruct video after transcoding failure.', { err }))
80 81
81 throw err 82 throw err
82 } 83 }
84 })
83} 85}
84 86
85// Transcode the original video file to a lower resolution. 87// Transcode the original video file to a lower resolution
86async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { 88// We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
89function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) {
87 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 90 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
88 const extname = '.mp4' 91 const extname = '.mp4'
89 92
90 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed 93 return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => {
91 const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile()) 94 const newVideoFile = new VideoFileModel({
95 resolution,
96 extname,
97 filename: generateWebTorrentVideoFilename(resolution, extname),
98 size: 0,
99 videoId: video.id
100 })
92 101
93 const newVideoFile = new VideoFileModel({ 102 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
94 resolution, 103 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
95 extname,
96 filename: generateWebTorrentVideoFilename(resolution, extname),
97 size: 0,
98 videoId: video.id
99 })
100 104
101 const videoOutputPath = getVideoFilePath(video, newVideoFile) 105 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
102 const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) 106 ? {
107 type: 'only-audio' as 'only-audio',
103 108
104 const transcodeOptions = resolution === VideoResolution.H_NOVIDEO 109 inputPath: videoInputPath,
105 ? { 110 outputPath: videoTranscodedPath,
106 type: 'only-audio' as 'only-audio',
107 111
108 inputPath: videoInputPath, 112 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
109 outputPath: videoTranscodedPath, 113 profile: CONFIG.TRANSCODING.PROFILE,
110 114
111 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 115 resolution,
112 profile: CONFIG.TRANSCODING.PROFILE,
113 116
114 resolution, 117 job
118 }
119 : {
120 type: 'video' as 'video',
121 inputPath: videoInputPath,
122 outputPath: videoTranscodedPath,
115 123
116 job 124 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
117 } 125 profile: CONFIG.TRANSCODING.PROFILE,
118 : {
119 type: 'video' as 'video',
120 inputPath: videoInputPath,
121 outputPath: videoTranscodedPath,
122 126
123 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 127 resolution,
124 profile: CONFIG.TRANSCODING.PROFILE, 128 isPortraitMode: isPortrait,
125 129
126 resolution, 130 job
127 isPortraitMode: isPortrait, 131 }
128 132
129 job 133 await transcode(transcodeOptions)
130 }
131
132 await transcode(transcodeOptions)
133 134
134 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 135 return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
136 })
135} 137}
136 138
137// Merge an image with an audio file to create a video 139// Merge an image with an audio file to create a video
138async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { 140function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) {
139 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 141 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
140 const newExtname = '.mp4' 142 const newExtname = '.mp4'
141 143
142 const inputVideoFile = video.getMinQualityFile() 144 const inputVideoFile = video.getMinQualityFile()
143 145
144 const audioInputPath = getVideoFilePath(video, inputVideoFile) 146 return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => {
145 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 147 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
146 148
147 // If the user updates the video preview during transcoding 149 // If the user updates the video preview during transcoding
148 const previewPath = video.getPreview().getPath() 150 const previewPath = video.getPreview().getPath()
149 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) 151 const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
150 await copyFile(previewPath, tmpPreviewPath) 152 await copyFile(previewPath, tmpPreviewPath)
151 153
152 const transcodeOptions = { 154 const transcodeOptions = {
153 type: 'merge-audio' as 'merge-audio', 155 type: 'merge-audio' as 'merge-audio',
154 156
155 inputPath: tmpPreviewPath, 157 inputPath: tmpPreviewPath,
156 outputPath: videoTranscodedPath, 158 outputPath: videoTranscodedPath,
157 159
158 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 160 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
159 profile: CONFIG.TRANSCODING.PROFILE, 161 profile: CONFIG.TRANSCODING.PROFILE,
160 162
161 audioPath: audioInputPath, 163 audioPath: audioInputPath,
162 resolution, 164 resolution,
163 165
164 job 166 job
165 } 167 }
166 168
167 try { 169 try {
168 await transcode(transcodeOptions) 170 await transcode(transcodeOptions)
169 171
170 await remove(audioInputPath) 172 await remove(audioInputPath)
171 await remove(tmpPreviewPath) 173 await remove(tmpPreviewPath)
172 } catch (err) { 174 } catch (err) {
173 await remove(tmpPreviewPath) 175 await remove(tmpPreviewPath)
174 throw err 176 throw err
175 } 177 }
176 178
177 // Important to do this before getVideoFilename() to take in account the new file extension 179 // Important to do this before getVideoFilename() to take in account the new file extension
178 inputVideoFile.extname = newExtname 180 inputVideoFile.extname = newExtname
179 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) 181 inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
180 182
181 const videoOutputPath = getVideoFilePath(video, inputVideoFile) 183 const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
182 // ffmpeg generated a new video file, so update the video duration 184 // ffmpeg generated a new video file, so update the video duration
183 // See https://trac.ffmpeg.org/ticket/5456 185 // See https://trac.ffmpeg.org/ticket/5456
184 video.duration = await getDurationFromVideoFile(videoTranscodedPath) 186 video.duration = await getDurationFromVideoFile(videoTranscodedPath)
185 await video.save() 187 await video.save()
186 188
187 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 189 return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
190 })
188} 191}
189 192
190// Concat TS segments from a live video to a fragmented mp4 HLS playlist 193// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@@ -258,7 +261,7 @@ async function onWebTorrentVideoFileTranscoding (
258 await VideoFileModel.customUpsert(videoFile, 'video', undefined) 261 await VideoFileModel.customUpsert(videoFile, 'video', undefined)
259 video.VideoFiles = await video.$get('VideoFiles') 262 video.VideoFiles = await video.$get('VideoFiles')
260 263
261 return video 264 return { video, videoFile }
262} 265}
263 266
264async function generateHlsPlaylistCommon (options: { 267async function generateHlsPlaylistCommon (options: {
@@ -335,14 +338,13 @@ async function generateHlsPlaylistCommon (options: {
335 videoStreamingPlaylistId: playlist.id 338 videoStreamingPlaylistId: playlist.id
336 }) 339 })
337 340
338 const videoFilePath = getVideoFilePath(playlist, newVideoFile) 341 const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
339 342
340 // Move files from tmp transcoded directory to the appropriate place 343 // Move files from tmp transcoded directory to the appropriate place
341 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 344 await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
342 await ensureDir(baseHlsDirectory)
343 345
344 // Move playlist file 346 // Move playlist file
345 const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) 347 const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
346 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) 348 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
347 // Move video file 349 // Move video file
348 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) 350 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
@@ -355,7 +357,7 @@ async function generateHlsPlaylistCommon (options: {
355 357
356 await createTorrentAndSetInfoHash(playlist, newVideoFile) 358 await createTorrentAndSetInfoHash(playlist, newVideoFile)
357 359
358 await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 360 const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
359 361
360 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo 362 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
361 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') 363 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
@@ -368,5 +370,5 @@ async function generateHlsPlaylistCommon (options: {
368 await updateMasterHLSPlaylist(video, playlistWithFiles) 370 await updateMasterHLSPlaylist(video, playlistWithFiles)
369 await updateSha256VODSegments(video, playlistWithFiles) 371 await updateSha256VODSegments(video, playlistWithFiles)
370 372
371 return resolutionPlaylistPath 373 return { resolutionPlaylistPath, videoFile: savedVideoFile }
372} 374}
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts
new file mode 100644
index 000000000..4c5d0c89d
--- /dev/null
+++ b/server/lib/video-path-manager.ts
@@ -0,0 +1,139 @@
1import { remove } from 'fs-extra'
2import { extname, join } from 'path'
3import { buildUUID } from '@server/helpers/uuid'
4import { extractVideo } from '@server/helpers/video'
5import { CONFIG } from '@server/initializers/config'
6import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
7import { VideoStorage } from '@shared/models'
8import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
9import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
10
11type MakeAvailableCB <T> = (path: string) => Promise<T> | T
12
13class VideoPathManager {
14
15 private static instance: VideoPathManager
16
17 private constructor () {}
18
19 getFSHLSOutputPath (video: MVideoUUID, filename?: string) {
20 const base = getHLSDirectory(video)
21 if (!filename) return base
22
23 return join(base, filename)
24 }
25
26 getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
27 if (videoFile.isHLS()) {
28 const video = extractVideo(videoOrPlaylist)
29
30 return join(getHLSRedundancyDirectory(video), videoFile.filename)
31 }
32
33 return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename)
34 }
35
36 getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
37 if (videoFile.isHLS()) {
38 const video = extractVideo(videoOrPlaylist)
39
40 return join(getHLSDirectory(video), videoFile.filename)
41 }
42
43 return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
44 }
45
46 async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
47 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
48 return this.makeAvailableFactory(
49 () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile),
50 false,
51 cb
52 )
53 }
54
55 const destination = this.buildTMPDestination(videoFile.filename)
56
57 if (videoFile.isHLS()) {
58 const video = extractVideo(videoOrPlaylist)
59
60 return this.makeAvailableFactory(
61 () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination),
62 true,
63 cb
64 )
65 }
66
67 return this.makeAvailableFactory(
68 () => makeWebTorrentFileAvailable(videoFile.filename, destination),
69 true,
70 cb
71 )
72 }
73
74 async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
75 const filename = getHlsResolutionPlaylistFilename(videoFile.filename)
76
77 if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
78 return this.makeAvailableFactory(
79 () => join(getHLSDirectory(playlist.Video), filename),
80 false,
81 cb
82 )
83 }
84
85 return this.makeAvailableFactory(
86 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
87 true,
88 cb
89 )
90 }
91
92 async makeAvailablePlaylistFile <T> (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB<T>) {
93 if (playlist.storage === VideoStorage.FILE_SYSTEM) {
94 return this.makeAvailableFactory(
95 () => join(getHLSDirectory(playlist.Video), filename),
96 false,
97 cb
98 )
99 }
100
101 return this.makeAvailableFactory(
102 () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
103 true,
104 cb
105 )
106 }
107
108 private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
109 let result: T
110
111 const destination = await method()
112
113 try {
114 result = await cb(destination)
115 } catch (err) {
116 if (destination && clean) await remove(destination)
117 throw err
118 }
119
120 if (clean) await remove(destination)
121
122 return result
123 }
124
125 private buildTMPDestination (filename: string) {
126 return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename))
127
128 }
129
130 static get Instance () {
131 return this.instance || (this.instance = new this())
132 }
133}
134
135// ---------------------------------------------------------------------------
136
137export {
138 VideoPathManager
139}
diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts
new file mode 100644
index 000000000..0613d94bf
--- /dev/null
+++ b/server/lib/video-state.ts
@@ -0,0 +1,99 @@
1import { Transaction } from 'sequelize'
2import { logger } from '@server/helpers/logger'
3import { CONFIG } from '@server/initializers/config'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { MVideoFullLight, MVideoUUID } from '@server/types/models'
8import { VideoState } from '@shared/models'
9import { federateVideoIfNeeded } from './activitypub/videos'
10import { Notifier } from './notifier'
11import { addMoveToObjectStorageJob } from './video'
12
13function buildNextVideoState (currentState?: VideoState) {
14 if (currentState === VideoState.PUBLISHED) {
15 throw new Error('Video is already in its final state')
16 }
17
18 if (
19 currentState !== VideoState.TO_TRANSCODE &&
20 currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
21 CONFIG.TRANSCODING.ENABLED
22 ) {
23 return VideoState.TO_TRANSCODE
24 }
25
26 if (
27 currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
28 CONFIG.OBJECT_STORAGE.ENABLED
29 ) {
30 return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
31 }
32
33 return VideoState.PUBLISHED
34}
35
36function moveToNextState (video: MVideoUUID, isNewVideo = true) {
37 return sequelizeTypescript.transaction(async t => {
38 // Maybe the video changed in database, refresh it
39 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
40 // Video does not exist anymore
41 if (!videoDatabase) return undefined
42
43 // Already in its final state
44 if (videoDatabase.state === VideoState.PUBLISHED) {
45 return federateVideoIfNeeded(videoDatabase, false, t)
46 }
47
48 const newState = buildNextVideoState(videoDatabase.state)
49
50 if (newState === VideoState.PUBLISHED) {
51 return moveToPublishedState(videoDatabase, isNewVideo, t)
52 }
53
54 if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
55 return moveToExternalStorageState(videoDatabase, isNewVideo, t)
56 }
57 })
58}
59
60// ---------------------------------------------------------------------------
61
62export {
63 buildNextVideoState,
64 moveToNextState
65}
66
67// ---------------------------------------------------------------------------
68
69async function moveToPublishedState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
70 logger.info('Publishing video %s.', video.uuid, { tags: [ video.uuid ] })
71
72 const previousState = video.state
73 await video.setNewState(VideoState.PUBLISHED, transaction)
74
75 // If the video was not published, we consider it is a new one for other instances
76 // Live videos are always federated, so it's not a new video
77 await federateVideoIfNeeded(video, isNewVideo, transaction)
78
79 Notifier.Instance.notifyOnNewVideoIfNeeded(video)
80
81 if (previousState === VideoState.TO_TRANSCODE) {
82 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video)
83 }
84}
85
86async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: boolean, transaction: Transaction) {
87 const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction)
88 const pendingTranscode = videoJobInfo?.pendingTranscode || 0
89
90 // We want to wait all transcoding jobs before moving the video on an external storage
91 if (pendingTranscode !== 0) return
92
93 await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, transaction)
94
95 logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
96
97 addMoveToObjectStorageJob(video, isNewVideo)
98 .catch(err => logger.error('Cannot add move to object storage job', { err }))
99}
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts
new file mode 100644
index 000000000..64c2c9bf9
--- /dev/null
+++ b/server/lib/video-urls.ts
@@ -0,0 +1,31 @@
1
2import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
3import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
4
5// ################## Redundancy ##################
6
7function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) {
8 // Base URL used by our HLS player
9 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid
10}
11
12function generateWebTorrentRedundancyUrl (file: MVideoFile) {
13 return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename
14}
15
16// ################## Meta data ##################
17
18function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) {
19 const path = '/api/v1/videos/'
20
21 return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id
22}
23
24// ---------------------------------------------------------------------------
25
26export {
27 getLocalVideoFileMetadataUrl,
28
29 generateWebTorrentRedundancyUrl,
30 generateHLSRedundancyUrl
31}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 61fee4949..0a2b93cc0 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -1,15 +1,13 @@
1import { UploadFiles } from 'express' 1import { UploadFiles } from 'express'
2import { Transaction } from 'sequelize/types' 2import { Transaction } from 'sequelize/types'
3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' 3import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
4import { sequelizeTypescript } from '@server/initializers/database'
5import { TagModel } from '@server/models/video/tag' 4import { TagModel } from '@server/models/video/tag'
6import { VideoModel } from '@server/models/video/video' 5import { VideoModel } from '@server/models/video/video'
6import { VideoJobInfoModel } from '@server/models/video/video-job-info'
7import { FilteredModelAttributes } from '@server/types' 7import { FilteredModelAttributes } from '@server/types'
8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' 9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
10import { federateVideoIfNeeded } from './activitypub/videos' 10import { CreateJobOptions, JobQueue } from './job-queue/job-queue'
11import { JobQueue } from './job-queue/job-queue'
12import { Notifier } from './notifier'
13import { updateVideoMiniatureFromExisting } from './thumbnail' 11import { updateVideoMiniatureFromExisting } from './thumbnail'
14 12
15function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { 13function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
@@ -82,29 +80,6 @@ async function setVideoTags (options: {
82 video.Tags = tagInstances 80 video.Tags = tagInstances
83} 81}
84 82
85async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
86 const result = await sequelizeTypescript.transaction(async t => {
87 // Maybe the video changed in database, refresh it
88 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
89 // Video does not exist anymore
90 if (!videoDatabase) return undefined
91
92 // We transcoded the video file in another format, now we can publish it
93 const videoPublished = await videoDatabase.publishIfNeededAndSave(t)
94
95 // If the video was not published, we consider it is a new one for other instances
96 // Live videos are always federated, so it's not a new video
97 await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t)
98
99 return { videoDatabase, videoPublished }
100 })
101
102 if (result?.videoPublished) {
103 Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase)
104 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase)
105 }
106}
107
108async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { 83async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
109 let dataInput: VideoTranscodingPayload 84 let dataInput: VideoTranscodingPayload
110 85
@@ -127,7 +102,20 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF
127 priority: await getTranscodingJobPriority(user) 102 priority: await getTranscodingJobPriority(user)
128 } 103 }
129 104
130 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions) 105 return addTranscodingJob(dataInput, jobOptions)
106}
107
108async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) {
109 await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
110
111 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options)
112}
113
114async function addMoveToObjectStorageJob (video: MVideoUUID, isNewVideo = true) {
115 await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
116
117 const dataInput = { videoUUID: video.uuid, isNewVideo }
118 return JobQueue.Instance.createJobWithPromise({ type: 'move-to-object-storage', payload: dataInput })
131} 119}
132 120
133async function getTranscodingJobPriority (user: MUserId) { 121async function getTranscodingJobPriority (user: MUserId) {
@@ -143,9 +131,10 @@ async function getTranscodingJobPriority (user: MUserId) {
143 131
144export { 132export {
145 buildLocalVideoFromReq, 133 buildLocalVideoFromReq,
146 publishAndFederateIfNeeded,
147 buildVideoThumbnailsFromReq, 134 buildVideoThumbnailsFromReq,
148 setVideoTags, 135 setVideoTags,
149 addOptimizeOrMergeAudioJob, 136 addOptimizeOrMergeAudioJob,
137 addTranscodingJob,
138 addMoveToObjectStorageJob,
150 getTranscodingJobPriority 139 getTranscodingJobPriority
151} 140}
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index 8a54de3b0..b3c4f390d 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -1,6 +1,6 @@
1import { uuidToShort } from '@server/helpers/uuid' 1import { uuidToShort } from '@server/helpers/uuid'
2import { generateMagnetUri } from '@server/helpers/webtorrent' 2import { generateMagnetUri } from '@server/helpers/webtorrent'
3import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' 3import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
4import { VideoFile } from '@shared/models/videos/video-file.model' 4import { VideoFile } from '@shared/models/videos/video-file.model'
5import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' 5import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects'
6import { Video, VideoDetails } from '../../../../shared/models/videos' 6import { Video, VideoDetails } from '../../../../shared/models/videos'
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts
index 742d19099..75823864d 100644
--- a/server/models/video/sql/shared/video-tables.ts
+++ b/server/models/video/sql/shared/video-tables.ts
@@ -87,7 +87,8 @@ export class VideoTables {
87 'fps', 87 'fps',
88 'metadataUrl', 88 'metadataUrl',
89 'videoStreamingPlaylistId', 89 'videoStreamingPlaylistId',
90 'videoId' 90 'videoId',
91 'storage'
91 ] 92 ]
92 } 93 }
93 94
@@ -102,7 +103,8 @@ export class VideoTables {
102 'segmentsSha256Url', 103 'segmentsSha256Url',
103 'videoId', 104 'videoId',
104 'createdAt', 105 'createdAt',
105 'updatedAt' 106 'updatedAt',
107 'storage'
106 ]) 108 ])
107 } 109 }
108 110
@@ -258,7 +260,8 @@ export class VideoTables {
258 'originallyPublishedAt', 260 'originallyPublishedAt',
259 'channelId', 261 'channelId',
260 'createdAt', 262 'createdAt',
261 'updatedAt' 263 'updatedAt',
264 'moveJobsRunning'
262 ] 265 ]
263 } 266 }
264} 267}
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 09fc5288b..627c95763 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -23,9 +23,11 @@ import validator from 'validator'
23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' 23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
24import { logger } from '@server/helpers/logger' 24import { logger } from '@server/helpers/logger'
25import { extractVideo } from '@server/helpers/video' 25import { extractVideo } from '@server/helpers/video'
26import { getTorrentFilePath } from '@server/lib/video-paths' 26import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
27import { getFSTorrentFilePath } from '@server/lib/paths'
27import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' 28import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
28import { AttributesOnly } from '@shared/core-utils' 29import { AttributesOnly } from '@shared/core-utils'
30import { VideoStorage } from '@shared/models'
29import { 31import {
30 isVideoFileExtnameValid, 32 isVideoFileExtnameValid,
31 isVideoFileInfoHashValid, 33 isVideoFileInfoHashValid,
@@ -214,6 +216,11 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
214 @Column 216 @Column
215 videoId: number 217 videoId: number
216 218
219 @AllowNull(false)
220 @Default(VideoStorage.FILE_SYSTEM)
221 @Column
222 storage: VideoStorage
223
217 @BelongsTo(() => VideoModel, { 224 @BelongsTo(() => VideoModel, {
218 foreignKey: { 225 foreignKey: {
219 allowNull: true 226 allowNull: true
@@ -273,7 +280,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
273 280
274 static async doesOwnedWebTorrentVideoFileExist (filename: string) { 281 static async doesOwnedWebTorrentVideoFileExist (filename: string) {
275 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + 282 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
276 'WHERE "filename" = $filename LIMIT 1' 283 `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1`
277 284
278 return doesExist(query, { filename }) 285 return doesExist(query, { filename })
279 } 286 }
@@ -450,9 +457,20 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
450 return !!this.videoStreamingPlaylistId 457 return !!this.videoStreamingPlaylistId
451 } 458 }
452 459
460 getObjectStorageUrl () {
461 if (this.isHLS()) {
462 return getHLSPublicFileUrl(this.fileUrl)
463 }
464
465 return getWebTorrentPublicFileUrl(this.fileUrl)
466 }
467
453 getFileUrl (video: MVideo) { 468 getFileUrl (video: MVideo) {
454 if (!this.Video) this.Video = video as VideoModel 469 if (this.storage === VideoStorage.OBJECT_STORAGE) {
470 return this.getObjectStorageUrl()
471 }
455 472
473 if (!this.Video) this.Video = video as VideoModel
456 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) 474 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video)
457 475
458 return this.fileUrl 476 return this.fileUrl
@@ -503,7 +521,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
503 removeTorrent () { 521 removeTorrent () {
504 if (!this.torrentFilename) return null 522 if (!this.torrentFilename) return null
505 523
506 const torrentPath = getTorrentFilePath(this) 524 const torrentPath = getFSTorrentFilePath(this)
507 return remove(torrentPath) 525 return remove(torrentPath)
508 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) 526 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
509 } 527 }
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts
new file mode 100644
index 000000000..7c1fe6734
--- /dev/null
+++ b/server/models/video/video-job-info.ts
@@ -0,0 +1,100 @@
1import { Op, QueryTypes, Transaction } from 'sequelize'
2import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript'
3import { AttributesOnly } from '@shared/core-utils'
4import { VideoModel } from './video'
5
6@Table({
7 tableName: 'videoJobInfo',
8 indexes: [
9 {
10 fields: [ 'videoId' ],
11 where: {
12 videoId: {
13 [Op.ne]: null
14 }
15 }
16 }
17 ]
18})
19
20export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfoModel>>> {
21 @CreatedAt
22 createdAt: Date
23
24 @UpdatedAt
25 updatedAt: Date
26
27 @AllowNull(false)
28 @Default(0)
29 @IsInt
30 @Column
31 pendingMove: number
32
33 @AllowNull(false)
34 @Default(0)
35 @IsInt
36 @Column
37 pendingTranscode: number
38
39 @ForeignKey(() => VideoModel)
40 @Unique
41 @Column
42 videoId: number
43
44 @BelongsTo(() => VideoModel, {
45 foreignKey: {
46 allowNull: false
47 },
48 onDelete: 'cascade'
49 })
50 Video: VideoModel
51
52 static load (videoId: number, transaction: Transaction) {
53 const where = {
54 videoId
55 }
56
57 return VideoJobInfoModel.findOne({ where, transaction })
58 }
59
60 static async increaseOrCreate (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> {
61 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
62
63 const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
64 INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt")
65 SELECT
66 "video"."id" AS "videoId", 1, NOW(), NOW()
67 FROM
68 "video"
69 WHERE
70 "video"."uuid" = $videoUUID
71 ON CONFLICT ("videoId") DO UPDATE
72 SET
73 "${column}" = "videoJobInfo"."${column}" + 1,
74 "updatedAt" = NOW()
75 RETURNING
76 "${column}"
77 `, options)
78
79 return pendingMove
80 }
81
82 static async decrease (videoUUID: string, column: 'pendingMove' | 'pendingTranscode'): Promise<number> {
83 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } }
84
85 const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(`
86 UPDATE
87 "videoJobInfo"
88 SET
89 "${column}" = "videoJobInfo"."${column}" - 1,
90 "updatedAt" = NOW()
91 FROM "video"
92 WHERE
93 "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID
94 RETURNING
95 "${column}";
96 `, options)
97
98 return pendingMove
99 }
100}
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index d591a3134..3e9fd97c7 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -1,10 +1,25 @@
1import * as memoizee from 'memoizee' 1import * as memoizee from 'memoizee'
2import { join } from 'path' 2import { join } from 'path'
3import { Op } from 'sequelize' 3import { Op } from 'sequelize'
4import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' 4import {
5 AllowNull,
6 BelongsTo,
7 Column,
8 CreatedAt,
9 DataType,
10 Default,
11 ForeignKey,
12 HasMany,
13 Is,
14 Model,
15 Table,
16 UpdatedAt
17} from 'sequelize-typescript'
18import { getHLSPublicFileUrl } from '@server/lib/object-storage'
5import { VideoFileModel } from '@server/models/video/video-file' 19import { VideoFileModel } from '@server/models/video/video-file'
6import { MStreamingPlaylist, MVideo } from '@server/types/models' 20import { MStreamingPlaylist, MVideo } from '@server/types/models'
7import { AttributesOnly } from '@shared/core-utils' 21import { AttributesOnly } from '@shared/core-utils'
22import { VideoStorage } from '@shared/models'
8import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 23import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
9import { sha1 } from '../../helpers/core-utils' 24import { sha1 } from '../../helpers/core-utils'
10import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 25import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
@@ -81,6 +96,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
81 @Column 96 @Column
82 videoId: number 97 videoId: number
83 98
99 @AllowNull(false)
100 @Default(VideoStorage.FILE_SYSTEM)
101 @Column
102 storage: VideoStorage
103
84 @BelongsTo(() => VideoModel, { 104 @BelongsTo(() => VideoModel, {
85 foreignKey: { 105 foreignKey: {
86 allowNull: false 106 allowNull: false
@@ -185,12 +205,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
185 } 205 }
186 206
187 getMasterPlaylistUrl (video: MVideo) { 207 getMasterPlaylistUrl (video: MVideo) {
208 if (this.storage === VideoStorage.OBJECT_STORAGE) {
209 return getHLSPublicFileUrl(this.playlistUrl)
210 }
211
188 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid) 212 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
189 213
190 return this.playlistUrl 214 return this.playlistUrl
191 } 215 }
192 216
193 getSha256SegmentsUrl (video: MVideo) { 217 getSha256SegmentsUrl (video: MVideo) {
218 if (this.storage === VideoStorage.OBJECT_STORAGE) {
219 return getHLSPublicFileUrl(this.segmentsSha256Url)
220 }
221
194 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive) 222 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
195 223
196 return this.segmentsSha256Url 224 return this.segmentsSha256Url
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 56a5b0e18..874ad168a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -28,14 +28,16 @@ import { buildNSFWFilter } from '@server/helpers/express-utils'
28import { uuidToShort } from '@server/helpers/uuid' 28import { uuidToShort } from '@server/helpers/uuid'
29import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' 29import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
30import { LiveManager } from '@server/lib/live/live-manager' 30import { LiveManager } from '@server/lib/live/live-manager'
31import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' 31import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
32import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
33import { VideoPathManager } from '@server/lib/video-path-manager'
32import { getServerActor } from '@server/models/application/application' 34import { getServerActor } from '@server/models/application/application'
33import { ModelCache } from '@server/models/model-cache' 35import { ModelCache } from '@server/models/model-cache'
34import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' 36import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
35import { VideoFile } from '@shared/models/videos/video-file.model' 37import { VideoFile } from '@shared/models/videos/video-file.model'
36import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' 38import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
37import { VideoObject } from '../../../shared/models/activitypub/objects' 39import { VideoObject } from '../../../shared/models/activitypub/objects'
38import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos' 40import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos'
39import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' 41import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
40import { VideoFilter } from '../../../shared/models/videos/video-query.type' 42import { VideoFilter } from '../../../shared/models/videos/video-query.type'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 43import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -114,6 +116,7 @@ import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel
114import { VideoCommentModel } from './video-comment' 116import { VideoCommentModel } from './video-comment'
115import { VideoFileModel } from './video-file' 117import { VideoFileModel } from './video-file'
116import { VideoImportModel } from './video-import' 118import { VideoImportModel } from './video-import'
119import { VideoJobInfoModel } from './video-job-info'
117import { VideoLiveModel } from './video-live' 120import { VideoLiveModel } from './video-live'
118import { VideoPlaylistElementModel } from './video-playlist-element' 121import { VideoPlaylistElementModel } from './video-playlist-element'
119import { VideoShareModel } from './video-share' 122import { VideoShareModel } from './video-share'
@@ -732,6 +735,15 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
732 }) 735 })
733 VideoCaptions: VideoCaptionModel[] 736 VideoCaptions: VideoCaptionModel[]
734 737
738 @HasOne(() => VideoJobInfoModel, {
739 foreignKey: {
740 name: 'videoId',
741 allowNull: false
742 },
743 onDelete: 'cascade'
744 })
745 VideoJobInfo: VideoJobInfoModel
746
735 @BeforeDestroy 747 @BeforeDestroy
736 static async sendDelete (instance: MVideoAccountLight, options) { 748 static async sendDelete (instance: MVideoAccountLight, options) {
737 if (!instance.isOwned()) return undefined 749 if (!instance.isOwned()) return undefined
@@ -1641,9 +1653,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1641 getMaxQualityResolution () { 1653 getMaxQualityResolution () {
1642 const file = this.getMaxQualityFile() 1654 const file = this.getMaxQualityFile()
1643 const videoOrPlaylist = file.getVideoOrStreamingPlaylist() 1655 const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
1644 const originalFilePath = getVideoFilePath(videoOrPlaylist, file)
1645 1656
1646 return getVideoFileResolution(originalFilePath) 1657 return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => {
1658 return getVideoFileResolution(originalFilePath)
1659 })
1647 } 1660 }
1648 1661
1649 getDescriptionAPIPath () { 1662 getDescriptionAPIPath () {
@@ -1673,16 +1686,24 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1673 } 1686 }
1674 1687
1675 removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { 1688 removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
1676 const filePath = getVideoFilePath(this, videoFile, isRedundancy) 1689 const filePath = isRedundancy
1690 ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile)
1691 : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile)
1677 1692
1678 const promises: Promise<any>[] = [ remove(filePath) ] 1693 const promises: Promise<any>[] = [ remove(filePath) ]
1679 if (!isRedundancy) promises.push(videoFile.removeTorrent()) 1694 if (!isRedundancy) promises.push(videoFile.removeTorrent())
1680 1695
1696 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
1697 promises.push(removeWebTorrentObjectStorage(videoFile))
1698 }
1699
1681 return Promise.all(promises) 1700 return Promise.all(promises)
1682 } 1701 }
1683 1702
1684 async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { 1703 async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
1685 const directoryPath = getHLSDirectory(this, isRedundancy) 1704 const directoryPath = isRedundancy
1705 ? getHLSRedundancyDirectory(this)
1706 : getHLSDirectory(this)
1686 1707
1687 await remove(directoryPath) 1708 await remove(directoryPath)
1688 1709
@@ -1698,6 +1719,10 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1698 await Promise.all( 1719 await Promise.all(
1699 streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) 1720 streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent())
1700 ) 1721 )
1722
1723 if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
1724 await removeHLSObjectStorage(streamingPlaylist, this)
1725 }
1701 } 1726 }
1702 } 1727 }
1703 1728
@@ -1741,16 +1766,16 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1741 this.privacy === VideoPrivacy.INTERNAL 1766 this.privacy === VideoPrivacy.INTERNAL
1742 } 1767 }
1743 1768
1744 async publishIfNeededAndSave (t: Transaction) { 1769 async setNewState (newState: VideoState, transaction: Transaction) {
1745 if (this.state !== VideoState.PUBLISHED) { 1770 if (this.state === newState) throw new Error('Cannot use same state ' + newState)
1746 this.state = VideoState.PUBLISHED 1771
1747 this.publishedAt = new Date() 1772 this.state = newState
1748 await this.save({ transaction: t })
1749 1773
1750 return true 1774 if (this.state === VideoState.PUBLISHED) {
1775 this.publishedAt = new Date()
1751 } 1776 }
1752 1777
1753 return false 1778 await this.save({ transaction })
1754 } 1779 }
1755 1780
1756 getBandwidthBits (videoFile: MVideoFile) { 1781 getBandwidthBits (videoFile: MVideoFile) {
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts
index b62e2f5f7..19301c0b9 100644
--- a/server/tests/api/index.ts
+++ b/server/tests/api/index.ts
@@ -2,6 +2,7 @@
2import './activitypub' 2import './activitypub'
3import './check-params' 3import './check-params'
4import './moderation' 4import './moderation'
5import './object-storage'
5import './notifications' 6import './notifications'
6import './redundancy' 7import './redundancy'
7import './search' 8import './search'
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 8f1fb78a5..6c4ea90ca 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -15,7 +15,9 @@ import {
15 stopFfmpeg, 15 stopFfmpeg,
16 testFfmpegStreamError, 16 testFfmpegStreamError,
17 wait, 17 wait,
18 waitJobs 18 waitJobs,
19 waitUntilLivePublishedOnAllServers,
20 waitUntilLiveSavedOnAllServers
19} from '@shared/extra-utils' 21} from '@shared/extra-utils'
20import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' 22import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
21 23
@@ -66,18 +68,6 @@ describe('Save replay setting', function () {
66 } 68 }
67 } 69 }
68 70
69 async function waitUntilLivePublishedOnAllServers (videoId: string) {
70 for (const server of servers) {
71 await server.live.waitUntilPublished({ videoId })
72 }
73 }
74
75 async function waitUntilLiveSavedOnAllServers (videoId: string) {
76 for (const server of servers) {
77 await server.live.waitUntilSaved({ videoId })
78 }
79 }
80
81 before(async function () { 71 before(async function () {
82 this.timeout(120000) 72 this.timeout(120000)
83 73
@@ -127,7 +117,7 @@ describe('Save replay setting', function () {
127 117
128 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 118 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
129 119
130 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 120 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
131 121
132 await waitJobs(servers) 122 await waitJobs(servers)
133 123
@@ -160,7 +150,7 @@ describe('Save replay setting', function () {
160 150
161 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 151 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
162 152
163 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 153 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
164 154
165 await waitJobs(servers) 155 await waitJobs(servers)
166 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 156 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -189,7 +179,7 @@ describe('Save replay setting', function () {
189 179
190 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 180 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
191 181
192 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 182 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
193 183
194 await waitJobs(servers) 184 await waitJobs(servers)
195 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 185 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -224,7 +214,7 @@ describe('Save replay setting', function () {
224 this.timeout(20000) 214 this.timeout(20000)
225 215
226 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 216 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
227 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 217 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
228 218
229 await waitJobs(servers) 219 await waitJobs(servers)
230 220
@@ -237,7 +227,7 @@ describe('Save replay setting', function () {
237 227
238 await stopFfmpeg(ffmpegCommand) 228 await stopFfmpeg(ffmpegCommand)
239 229
240 await waitUntilLiveSavedOnAllServers(liveVideoUUID) 230 await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID)
241 await waitJobs(servers) 231 await waitJobs(servers)
242 232
243 // Live has been transcoded 233 // Live has been transcoded
@@ -268,7 +258,7 @@ describe('Save replay setting', function () {
268 liveVideoUUID = await createLiveWrapper(true) 258 liveVideoUUID = await createLiveWrapper(true)
269 259
270 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 260 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
271 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 261 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
272 262
273 await waitJobs(servers) 263 await waitJobs(servers)
274 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 264 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -296,7 +286,7 @@ describe('Save replay setting', function () {
296 liveVideoUUID = await createLiveWrapper(true) 286 liveVideoUUID = await createLiveWrapper(true)
297 287
298 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 288 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
299 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 289 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
300 290
301 await waitJobs(servers) 291 await waitJobs(servers)
302 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 292 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts
new file mode 100644
index 000000000..f319d6ef5
--- /dev/null
+++ b/server/tests/api/object-storage/index.ts
@@ -0,0 +1,3 @@
1export * from './live'
2export * from './video-imports'
3export * from './videos'
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
new file mode 100644
index 000000000..d3e6777f2
--- /dev/null
+++ b/server/tests/api/object-storage/live.ts
@@ -0,0 +1,136 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { FfmpegCommand } from 'fluent-ffmpeg'
6import {
7 areObjectStorageTestsDisabled,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 killallServers,
12 makeRawRequest,
13 ObjectStorageCommand,
14 PeerTubeServer,
15 setAccessTokensToServers,
16 setDefaultVideoChannel,
17 stopFfmpeg,
18 waitJobs,
19 waitUntilLivePublishedOnAllServers,
20 waitUntilLiveSavedOnAllServers
21} from '@shared/extra-utils'
22import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models'
23
24const expect = chai.expect
25
26async function createLive (server: PeerTubeServer) {
27 const attributes: LiveVideoCreate = {
28 channelId: server.store.channel.id,
29 privacy: VideoPrivacy.PUBLIC,
30 name: 'my super live',
31 saveReplay: true
32 }
33
34 const { uuid } = await server.live.create({ fields: attributes })
35
36 return uuid
37}
38
39async function checkFiles (files: VideoFile[]) {
40 for (const file of files) {
41 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
42
43 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
44 }
45}
46
47describe('Object storage for lives', function () {
48 if (areObjectStorageTestsDisabled()) return
49
50 let ffmpegCommand: FfmpegCommand
51 let servers: PeerTubeServer[]
52 let videoUUID: string
53
54 before(async function () {
55 this.timeout(120000)
56
57 await ObjectStorageCommand.prepareDefaultBuckets()
58
59 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig())
60
61 await setAccessTokensToServers(servers)
62 await setDefaultVideoChannel(servers)
63 await doubleFollow(servers[0], servers[1])
64
65 await servers[0].config.enableTranscoding()
66 })
67
68 describe('Without live transcoding', async function () {
69
70 before(async function () {
71 await servers[0].config.enableLive({ transcoding: false })
72
73 videoUUID = await createLive(servers[0])
74 })
75
76 it('Should create a live and save the replay on object storage', async function () {
77 this.timeout(220000)
78
79 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
80 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
81
82 await stopFfmpeg(ffmpegCommand)
83
84 await waitUntilLiveSavedOnAllServers(servers, videoUUID)
85 await waitJobs(servers)
86
87 for (const server of servers) {
88 const video = await server.videos.get({ id: videoUUID })
89
90 expect(video.files).to.have.lengthOf(0)
91 expect(video.streamingPlaylists).to.have.lengthOf(1)
92
93 const files = video.streamingPlaylists[0].files
94
95 await checkFiles(files)
96 }
97 })
98 })
99
100 describe('With live transcoding', async function () {
101
102 before(async function () {
103 await servers[0].config.enableLive({ transcoding: true })
104
105 videoUUID = await createLive(servers[0])
106 })
107
108 it('Should import a video and have sent it to object storage', async function () {
109 this.timeout(240000)
110
111 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
112 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
113
114 await stopFfmpeg(ffmpegCommand)
115
116 await waitUntilLiveSavedOnAllServers(servers, videoUUID)
117 await waitJobs(servers)
118
119 for (const server of servers) {
120 const video = await server.videos.get({ id: videoUUID })
121
122 expect(video.files).to.have.lengthOf(0)
123 expect(video.streamingPlaylists).to.have.lengthOf(1)
124
125 const files = video.streamingPlaylists[0].files
126 expect(files).to.have.lengthOf(4)
127
128 await checkFiles(files)
129 }
130 })
131 })
132
133 after(async function () {
134 await killallServers(servers)
135 })
136})
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts
new file mode 100644
index 000000000..efc01f550
--- /dev/null
+++ b/server/tests/api/object-storage/video-imports.ts
@@ -0,0 +1,112 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import {
6 areObjectStorageTestsDisabled,
7 createSingleServer,
8 expectStartWith,
9 FIXTURE_URLS,
10 killallServers,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 setDefaultVideoChannel,
16 waitJobs
17} from '@shared/extra-utils'
18import { HttpStatusCode, VideoPrivacy } from '@shared/models'
19
20const expect = chai.expect
21
22async function importVideo (server: PeerTubeServer) {
23 const attributes = {
24 name: 'import 2',
25 privacy: VideoPrivacy.PUBLIC,
26 channelId: server.store.channel.id,
27 targetUrl: FIXTURE_URLS.goodVideo720
28 }
29
30 const { video: { uuid } } = await server.imports.importVideo({ attributes })
31
32 return uuid
33}
34
35describe('Object storage for video import', function () {
36 if (areObjectStorageTestsDisabled()) return
37
38 let server: PeerTubeServer
39
40 before(async function () {
41 this.timeout(120000)
42
43 await ObjectStorageCommand.prepareDefaultBuckets()
44
45 server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig())
46
47 await setAccessTokensToServers([ server ])
48 await setDefaultVideoChannel([ server ])
49
50 await server.config.enableImports()
51 })
52
53 describe('Without transcoding', async function () {
54
55 before(async function () {
56 await server.config.disableTranscoding()
57 })
58
59 it('Should import a video and have sent it to object storage', async function () {
60 this.timeout(120000)
61
62 const uuid = await importVideo(server)
63 await waitJobs(server)
64
65 const video = await server.videos.get({ id: uuid })
66
67 expect(video.files).to.have.lengthOf(1)
68 expect(video.streamingPlaylists).to.have.lengthOf(0)
69
70 const fileUrl = video.files[0].fileUrl
71 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
72
73 await makeRawRequest(fileUrl, HttpStatusCode.OK_200)
74 })
75 })
76
77 describe('With transcoding', async function () {
78
79 before(async function () {
80 await server.config.enableTranscoding()
81 })
82
83 it('Should import a video and have sent it to object storage', async function () {
84 this.timeout(120000)
85
86 const uuid = await importVideo(server)
87 await waitJobs(server)
88
89 const video = await server.videos.get({ id: uuid })
90
91 expect(video.files).to.have.lengthOf(4)
92 expect(video.streamingPlaylists).to.have.lengthOf(1)
93 expect(video.streamingPlaylists[0].files).to.have.lengthOf(4)
94
95 for (const file of video.files) {
96 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
97
98 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
99 }
100
101 for (const file of video.streamingPlaylists[0].files) {
102 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
103
104 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 }
106 })
107 })
108
109 after(async function () {
110 await killallServers([ server ])
111 })
112})
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
new file mode 100644
index 000000000..3958bd3d7
--- /dev/null
+++ b/server/tests/api/object-storage/videos.ts
@@ -0,0 +1,391 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { merge } from 'lodash'
6import {
7 areObjectStorageTestsDisabled,
8 checkTmpIsEmpty,
9 cleanupTests,
10 createMultipleServers,
11 createSingleServer,
12 doubleFollow,
13 expectStartWith,
14 killallServers,
15 makeRawRequest,
16 MockObjectStorage,
17 ObjectStorageCommand,
18 PeerTubeServer,
19 setAccessTokensToServers,
20 waitJobs,
21 webtorrentAdd
22} from '@shared/extra-utils'
23import { HttpStatusCode, VideoDetails } from '@shared/models'
24
25const expect = chai.expect
26
27async function checkFiles (options: {
28 video: VideoDetails
29
30 baseMockUrl?: string
31
32 playlistBucket: string
33 playlistPrefix?: string
34
35 webtorrentBucket: string
36 webtorrentPrefix?: string
37}) {
38 const {
39 video,
40 playlistBucket,
41 webtorrentBucket,
42 baseMockUrl,
43 playlistPrefix,
44 webtorrentPrefix
45 } = options
46
47 let allFiles = video.files
48
49 for (const file of video.files) {
50 const baseUrl = baseMockUrl
51 ? `${baseMockUrl}/${webtorrentBucket}/`
52 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
53
54 const prefix = webtorrentPrefix || ''
55 const start = baseUrl + prefix
56
57 expectStartWith(file.fileUrl, start)
58
59 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
60 const location = res.headers['location']
61 expectStartWith(location, start)
62
63 await makeRawRequest(location, HttpStatusCode.OK_200)
64 }
65
66 const hls = video.streamingPlaylists[0]
67
68 if (hls) {
69 allFiles = allFiles.concat(hls.files)
70
71 const baseUrl = baseMockUrl
72 ? `${baseMockUrl}/${playlistBucket}/`
73 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
74
75 const prefix = playlistPrefix || ''
76 const start = baseUrl + prefix
77
78 expectStartWith(hls.playlistUrl, start)
79 expectStartWith(hls.segmentsSha256Url, start)
80
81 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
82
83 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
84 expect(JSON.stringify(resSha.body)).to.not.throw
85
86 for (const file of hls.files) {
87 expectStartWith(file.fileUrl, start)
88
89 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
90 const location = res.headers['location']
91 expectStartWith(location, start)
92
93 await makeRawRequest(location, HttpStatusCode.OK_200)
94 }
95 }
96
97 for (const file of allFiles) {
98 const torrent = await webtorrentAdd(file.magnetUri, true)
99
100 expect(torrent.files).to.be.an('array')
101 expect(torrent.files.length).to.equal(1)
102 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
103
104 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 expect(res.body).to.have.length.above(100)
106 }
107
108 return allFiles.map(f => f.fileUrl)
109}
110
111function runTestSuite (options: {
112 playlistBucket: string
113 playlistPrefix?: string
114
115 webtorrentBucket: string
116 webtorrentPrefix?: string
117
118 useMockBaseUrl?: boolean
119
120 maxUploadPart?: string
121}) {
122 const mockObjectStorage = new MockObjectStorage()
123 let baseMockUrl: string
124
125 let servers: PeerTubeServer[]
126
127 let keptUrls: string[] = []
128
129 const uuidsToDelete: string[] = []
130 let deletedUrls: string[] = []
131
132 before(async function () {
133 this.timeout(120000)
134
135 const port = await mockObjectStorage.initialize()
136 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
137
138 await ObjectStorageCommand.createBucket(options.playlistBucket)
139 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
140
141 const config = {
142 object_storage: {
143 enabled: true,
144 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
145 region: ObjectStorageCommand.getRegion(),
146
147 credentials: ObjectStorageCommand.getCredentialsConfig(),
148
149 max_upload_part: options.maxUploadPart || '2MB',
150
151 streaming_playlists: {
152 bucket_name: options.playlistBucket,
153 prefix: options.playlistPrefix,
154 base_url: baseMockUrl
155 ? `${baseMockUrl}/${options.playlistBucket}`
156 : undefined
157 },
158
159 videos: {
160 bucket_name: options.webtorrentBucket,
161 prefix: options.webtorrentPrefix,
162 base_url: baseMockUrl
163 ? `${baseMockUrl}/${options.webtorrentBucket}`
164 : undefined
165 }
166 }
167 }
168
169 servers = await createMultipleServers(2, config)
170
171 await setAccessTokensToServers(servers)
172 await doubleFollow(servers[0], servers[1])
173
174 for (const server of servers) {
175 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
176 await waitJobs(servers)
177
178 const files = await server.videos.listFiles({ id: uuid })
179 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
180 }
181 })
182
183 it('Should upload a video and move it to the object storage without transcoding', async function () {
184 this.timeout(20000)
185
186 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
187 uuidsToDelete.push(uuid)
188
189 await waitJobs(servers)
190
191 for (const server of servers) {
192 const video = await server.videos.get({ id: uuid })
193 const files = await checkFiles({ ...options, video, baseMockUrl })
194
195 deletedUrls = deletedUrls.concat(files)
196 }
197 })
198
199 it('Should upload a video and move it to the object storage with transcoding', async function () {
200 this.timeout(40000)
201
202 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
203 uuidsToDelete.push(uuid)
204
205 await waitJobs(servers)
206
207 for (const server of servers) {
208 const video = await server.videos.get({ id: uuid })
209 const files = await checkFiles({ ...options, video, baseMockUrl })
210
211 deletedUrls = deletedUrls.concat(files)
212 }
213 })
214
215 it('Should correctly delete the files', async function () {
216 await servers[0].videos.remove({ id: uuidsToDelete[0] })
217 await servers[1].videos.remove({ id: uuidsToDelete[1] })
218
219 await waitJobs(servers)
220
221 for (const url of deletedUrls) {
222 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
223 }
224 })
225
226 it('Should have kept other files', async function () {
227 for (const url of keptUrls) {
228 await makeRawRequest(url, HttpStatusCode.OK_200)
229 }
230 })
231
232 it('Should have an empty tmp directory', async function () {
233 for (const server of servers) {
234 await checkTmpIsEmpty(server)
235 }
236 })
237
238 after(async function () {
239 mockObjectStorage.terminate()
240
241 await cleanupTests(servers)
242 })
243}
244
245describe('Object storage for videos', function () {
246 if (areObjectStorageTestsDisabled()) return
247
248 describe('Test config', function () {
249 let server: PeerTubeServer
250
251 const baseConfig = {
252 object_storage: {
253 enabled: true,
254 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
255 region: ObjectStorageCommand.getRegion(),
256
257 credentials: ObjectStorageCommand.getCredentialsConfig(),
258
259 streaming_playlists: {
260 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
261 },
262
263 videos: {
264 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
265 }
266 }
267 }
268
269 const badCredentials = {
270 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
271 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
272 }
273
274 it('Should fail with same bucket names without prefix', function (done) {
275 const config = merge({}, baseConfig, {
276 object_storage: {
277 streaming_playlists: {
278 bucket_name: 'aaa'
279 },
280
281 videos: {
282 bucket_name: 'aaa'
283 }
284 }
285 })
286
287 createSingleServer(1, config)
288 .then(() => done(new Error('Did not throw')))
289 .catch(() => done())
290 })
291
292 it('Should fail with bad credentials', async function () {
293 this.timeout(60000)
294
295 await ObjectStorageCommand.prepareDefaultBuckets()
296
297 const config = merge({}, baseConfig, {
298 object_storage: {
299 credentials: badCredentials
300 }
301 })
302
303 server = await createSingleServer(1, config)
304 await setAccessTokensToServers([ server ])
305
306 const { uuid } = await server.videos.quickUpload({ name: 'video' })
307
308 await waitJobs([ server ], true)
309 const video = await server.videos.get({ id: uuid })
310
311 expectStartWith(video.files[0].fileUrl, server.url)
312
313 await killallServers([ server ])
314 })
315
316 it('Should succeed with credentials from env', async function () {
317 this.timeout(60000)
318
319 await ObjectStorageCommand.prepareDefaultBuckets()
320
321 const config = merge({}, baseConfig, {
322 object_storage: {
323 credentials: {
324 access_key_id: '',
325 secret_access_key: ''
326 }
327 }
328 })
329
330 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
331
332 server = await createSingleServer(1, config, {
333 env: {
334 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
335 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
336 }
337 })
338
339 await setAccessTokensToServers([ server ])
340
341 const { uuid } = await server.videos.quickUpload({ name: 'video' })
342
343 await waitJobs([ server ], true)
344 const video = await server.videos.get({ id: uuid })
345
346 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
347 })
348
349 after(async function () {
350 await killallServers([ server ])
351 })
352 })
353
354 describe('Test simple object storage', function () {
355 runTestSuite({
356 playlistBucket: 'streaming-playlists',
357 webtorrentBucket: 'videos'
358 })
359 })
360
361 describe('Test object storage with prefix', function () {
362 runTestSuite({
363 playlistBucket: 'mybucket',
364 webtorrentBucket: 'mybucket',
365
366 playlistPrefix: 'streaming-playlists_',
367 webtorrentPrefix: 'webtorrent_'
368 })
369 })
370
371 describe('Test object storage with prefix and base URL', function () {
372 runTestSuite({
373 playlistBucket: 'mybucket',
374 webtorrentBucket: 'mybucket',
375
376 playlistPrefix: 'streaming-playlists_',
377 webtorrentPrefix: 'webtorrent_',
378
379 useMockBaseUrl: true
380 })
381 })
382
383 describe('Test object storage with small upload part', function () {
384 runTestSuite({
385 playlistBucket: 'streaming-playlists',
386 webtorrentBucket: 'videos',
387
388 maxUploadPart: '5KB'
389 })
390 })
391})
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index e1a12f5f8..3400b1d9a 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -207,14 +207,14 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) 207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
208 } 208 }
209 209
210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' 210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID
211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls' 211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID
212 212
213 const video = await servers[0].videos.get({ id: videoUUID }) 213 const video = await servers[0].videos.get({ id: videoUUID })
214 const hlsPlaylist = video.streamingPlaylists[0] 214 const hlsPlaylist = video.streamingPlaylists[0]
215 215
216 for (const resolution of [ 240, 360, 480, 720 ]) { 216 for (const resolution of [ 240, 360, 480, 720 ]) {
217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) 217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist })
218 } 218 }
219 219
220 const { hlsFilenames } = await ensureSameFilenames(videoUUID) 220 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
index 961f0e617..2c829f532 100644
--- a/server/tests/api/videos/video-hls.ts
+++ b/server/tests/api/videos/video-hls.ts
@@ -5,6 +5,7 @@ import * as chai from 'chai'
5import { basename, join } from 'path' 5import { basename, join } from 'path'
6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' 6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
7import { 7import {
8 areObjectStorageTestsDisabled,
8 checkDirectoryIsEmpty, 9 checkDirectoryIsEmpty,
9 checkResolutionsInMasterPlaylist, 10 checkResolutionsInMasterPlaylist,
10 checkSegmentHash, 11 checkSegmentHash,
@@ -12,7 +13,9 @@ import {
12 cleanupTests, 13 cleanupTests,
13 createMultipleServers, 14 createMultipleServers,
14 doubleFollow, 15 doubleFollow,
16 expectStartWith,
15 makeRawRequest, 17 makeRawRequest,
18 ObjectStorageCommand,
16 PeerTubeServer, 19 PeerTubeServer,
17 setAccessTokensToServers, 20 setAccessTokensToServers,
18 waitJobs, 21 waitJobs,
@@ -23,8 +26,19 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
23 26
24const expect = chai.expect 27const expect = chai.expect
25 28
26async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { 29async function checkHlsPlaylist (options: {
27 for (const server of servers) { 30 servers: PeerTubeServer[]
31 videoUUID: string
32 hlsOnly: boolean
33
34 resolutions?: number[]
35 objectStorageBaseUrl: string
36}) {
37 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
38
39 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
40
41 for (const server of options.servers) {
28 const videoDetails = await server.videos.get({ id: videoUUID }) 42 const videoDetails = await server.videos.get({ id: videoUUID })
29 const baseUrl = `http://${videoDetails.account.host}` 43 const baseUrl = `http://${videoDetails.account.host}`
30 44
@@ -48,9 +62,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
48 expect(file.torrentUrl).to.match( 62 expect(file.torrentUrl).to.match(
49 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) 63 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
50 ) 64 )
51 expect(file.fileUrl).to.match( 65
52 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) 66 if (objectStorageBaseUrl) {
53 ) 67 expectStartWith(file.fileUrl, objectStorageBaseUrl)
68 } else {
69 expect(file.fileUrl).to.match(
70 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
71 )
72 }
73
54 expect(file.resolution.label).to.equal(resolution + 'p') 74 expect(file.resolution.label).to.equal(resolution + 'p')
55 75
56 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) 76 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
@@ -80,9 +100,11 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
80 const file = hlsFiles.find(f => f.resolution.id === resolution) 100 const file = hlsFiles.find(f => f.resolution.id === resolution)
81 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' 101 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
82 102
83 const subPlaylist = await server.streamingPlaylists.get({ 103 const url = objectStorageBaseUrl
84 url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` 104 ? `${objectStorageBaseUrl}hls_${videoUUID}/${playlistName}`
85 }) 105 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
106
107 const subPlaylist = await server.streamingPlaylists.get({ url })
86 108
87 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) 109 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
88 expect(subPlaylist).to.contain(basename(file.fileUrl)) 110 expect(subPlaylist).to.contain(basename(file.fileUrl))
@@ -90,14 +112,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
90 } 112 }
91 113
92 { 114 {
93 const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' 115 const baseUrlAndPath = objectStorageBaseUrl
116 ? objectStorageBaseUrl + 'hls_' + videoUUID
117 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
94 118
95 for (const resolution of resolutions) { 119 for (const resolution of resolutions) {
96 await checkSegmentHash({ 120 await checkSegmentHash({
97 server, 121 server,
98 baseUrlPlaylist: baseUrlAndPath, 122 baseUrlPlaylist: baseUrlAndPath,
99 baseUrlSegment: baseUrlAndPath, 123 baseUrlSegment: baseUrlAndPath,
100 videoUUID,
101 resolution, 124 resolution,
102 hlsPlaylist 125 hlsPlaylist
103 }) 126 })
@@ -111,7 +134,7 @@ describe('Test HLS videos', function () {
111 let videoUUID = '' 134 let videoUUID = ''
112 let videoAudioUUID = '' 135 let videoAudioUUID = ''
113 136
114 function runTestSuite (hlsOnly: boolean) { 137 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
115 138
116 it('Should upload a video and transcode it to HLS', async function () { 139 it('Should upload a video and transcode it to HLS', async function () {
117 this.timeout(120000) 140 this.timeout(120000)
@@ -121,7 +144,7 @@ describe('Test HLS videos', function () {
121 144
122 await waitJobs(servers) 145 await waitJobs(servers)
123 146
124 await checkHlsPlaylist(servers, videoUUID, hlsOnly) 147 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
125 }) 148 })
126 149
127 it('Should upload an audio file and transcode it to HLS', async function () { 150 it('Should upload an audio file and transcode it to HLS', async function () {
@@ -132,7 +155,13 @@ describe('Test HLS videos', function () {
132 155
133 await waitJobs(servers) 156 await waitJobs(servers)
134 157
135 await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ]) 158 await checkHlsPlaylist({
159 servers,
160 videoUUID: videoAudioUUID,
161 hlsOnly,
162 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
163 objectStorageBaseUrl
164 })
136 }) 165 })
137 166
138 it('Should update the video', async function () { 167 it('Should update the video', async function () {
@@ -142,7 +171,7 @@ describe('Test HLS videos', function () {
142 171
143 await waitJobs(servers) 172 await waitJobs(servers)
144 173
145 await checkHlsPlaylist(servers, videoUUID, hlsOnly) 174 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
146 }) 175 })
147 176
148 it('Should delete videos', async function () { 177 it('Should delete videos', async function () {
@@ -229,6 +258,22 @@ describe('Test HLS videos', function () {
229 runTestSuite(true) 258 runTestSuite(true)
230 }) 259 })
231 260
261 describe('With object storage enabled', function () {
262 if (areObjectStorageTestsDisabled()) return
263
264 before(async function () {
265 this.timeout(120000)
266
267 const configOverride = ObjectStorageCommand.getDefaultConfig()
268 await ObjectStorageCommand.prepareDefaultBuckets()
269
270 await servers[0].kill()
271 await servers[0].run(configOverride)
272 })
273
274 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
275 })
276
232 after(async function () { 277 after(async function () {
233 await cleanupTests(servers) 278 await cleanupTests(servers)
234 }) 279 })
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
index bddcff5e7..9f1b57a2e 100644
--- a/server/tests/cli/create-import-video-file-job.ts
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -2,8 +2,19 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' 5import {
6import { VideoFile } from '@shared/models' 6 areObjectStorageTestsDisabled,
7 cleanupTests,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 waitJobs
16} from '@shared/extra-utils'
17import { HttpStatusCode, VideoDetails, VideoFile } from '@shared/models'
7 18
8const expect = chai.expect 19const expect = chai.expect
9 20
@@ -17,22 +28,35 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s
17 if (size) expect(video.size).to.equal(size) 28 if (size) expect(video.size).to.equal(size)
18} 29}
19 30
20describe('Test create import video jobs', function () { 31async function checkFiles (video: VideoDetails, objectStorage: boolean) {
21 this.timeout(60000) 32 for (const file of video.files) {
33 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
22 34
23 let servers: PeerTubeServer[] = [] 35 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
36 }
37}
38
39function runTests (objectStorage: boolean) {
24 let video1UUID: string 40 let video1UUID: string
25 let video2UUID: string 41 let video2UUID: string
26 42
43 let servers: PeerTubeServer[] = []
44
27 before(async function () { 45 before(async function () {
28 this.timeout(90000) 46 this.timeout(90000)
29 47
48 const config = objectStorage
49 ? ObjectStorageCommand.getDefaultConfig()
50 : {}
51
30 // Run server 2 to have transcoding enabled 52 // Run server 2 to have transcoding enabled
31 servers = await createMultipleServers(2) 53 servers = await createMultipleServers(2, config)
32 await setAccessTokensToServers(servers) 54 await setAccessTokensToServers(servers)
33 55
34 await doubleFollow(servers[0], servers[1]) 56 await doubleFollow(servers[0], servers[1])
35 57
58 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
59
36 // Upload two videos for our needs 60 // Upload two videos for our needs
37 { 61 {
38 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) 62 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video1' } })
@@ -44,7 +68,6 @@ describe('Test create import video jobs', function () {
44 video2UUID = uuid 68 video2UUID = uuid
45 } 69 }
46 70
47 // Transcoding
48 await waitJobs(servers) 71 await waitJobs(servers)
49 }) 72 })
50 73
@@ -65,6 +88,8 @@ describe('Test create import video jobs', function () {
65 const [ originalVideo, transcodedVideo ] = videoDetails.files 88 const [ originalVideo, transcodedVideo ] = videoDetails.files
66 assertVideoProperties(originalVideo, 720, 'webm', 218910) 89 assertVideoProperties(originalVideo, 720, 'webm', 218910)
67 assertVideoProperties(transcodedVideo, 480, 'webm', 69217) 90 assertVideoProperties(transcodedVideo, 480, 'webm', 69217)
91
92 await checkFiles(videoDetails, objectStorage)
68 } 93 }
69 }) 94 })
70 95
@@ -87,6 +112,8 @@ describe('Test create import video jobs', function () {
87 assertVideoProperties(transcodedVideo420, 480, 'mp4') 112 assertVideoProperties(transcodedVideo420, 480, 'mp4')
88 assertVideoProperties(transcodedVideo320, 360, 'mp4') 113 assertVideoProperties(transcodedVideo320, 360, 'mp4')
89 assertVideoProperties(transcodedVideo240, 240, 'mp4') 114 assertVideoProperties(transcodedVideo240, 240, 'mp4')
115
116 await checkFiles(videoDetails, objectStorage)
90 } 117 }
91 }) 118 })
92 119
@@ -107,10 +134,25 @@ describe('Test create import video jobs', function () {
107 const [ video720, video480 ] = videoDetails.files 134 const [ video720, video480 ] = videoDetails.files
108 assertVideoProperties(video720, 720, 'webm', 942961) 135 assertVideoProperties(video720, 720, 'webm', 942961)
109 assertVideoProperties(video480, 480, 'webm', 69217) 136 assertVideoProperties(video480, 480, 'webm', 69217)
137
138 await checkFiles(videoDetails, objectStorage)
110 } 139 }
111 }) 140 })
112 141
113 after(async function () { 142 after(async function () {
114 await cleanupTests(servers) 143 await cleanupTests(servers)
115 }) 144 })
145}
146
147describe('Test create import video jobs', function () {
148
149 describe('On filesystem', function () {
150 runTests(false)
151 })
152
153 describe('On object storage', function () {
154 if (areObjectStorageTestsDisabled()) return
155
156 runTests(true)
157 })
116}) 158})
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index df787ccdc..3313a492f 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -2,10 +2,15 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { HttpStatusCode, VideoFile } from '@shared/models'
5import { 6import {
7 areObjectStorageTestsDisabled,
6 cleanupTests, 8 cleanupTests,
7 createMultipleServers, 9 createMultipleServers,
8 doubleFollow, 10 doubleFollow,
11 expectStartWith,
12 makeRawRequest,
13 ObjectStorageCommand,
9 PeerTubeServer, 14 PeerTubeServer,
10 setAccessTokensToServers, 15 setAccessTokensToServers,
11 waitJobs 16 waitJobs
@@ -13,39 +18,39 @@ import {
13 18
14const expect = chai.expect 19const expect = chai.expect
15 20
16describe('Test create transcoding jobs', function () { 21async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') {
17 let servers: PeerTubeServer[] = [] 22 for (const file of files) {
18 const videosUUID: string[] = [] 23 const shouldStartWith = type === 'webtorrent'
24 ? ObjectStorageCommand.getWebTorrentBaseUrl()
25 : ObjectStorageCommand.getPlaylistBaseUrl()
19 26
20 const config = { 27 expectStartWith(file.fileUrl, shouldStartWith)
21 transcoding: { 28
22 enabled: false, 29 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
23 resolutions: {
24 '240p': true,
25 '360p': true,
26 '480p': true,
27 '720p': true,
28 '1080p': true,
29 '1440p': true,
30 '2160p': true
31 },
32 hls: {
33 enabled: false
34 }
35 }
36 } 30 }
31}
32
33function runTests (objectStorage: boolean) {
34 let servers: PeerTubeServer[] = []
35 const videosUUID: string[] = []
37 36
38 before(async function () { 37 before(async function () {
39 this.timeout(60000) 38 this.timeout(60000)
40 39
40 const config = objectStorage
41 ? ObjectStorageCommand.getDefaultConfig()
42 : {}
43
41 // Run server 2 to have transcoding enabled 44 // Run server 2 to have transcoding enabled
42 servers = await createMultipleServers(2) 45 servers = await createMultipleServers(2, config)
43 await setAccessTokensToServers(servers) 46 await setAccessTokensToServers(servers)
44 47
45 await servers[0].config.updateCustomSubConfig({ newConfig: config }) 48 await servers[0].config.disableTranscoding()
46 49
47 await doubleFollow(servers[0], servers[1]) 50 await doubleFollow(servers[0], servers[1])
48 51
52 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
53
49 for (let i = 1; i <= 5; i++) { 54 for (let i = 1; i <= 5; i++) {
50 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) 55 const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
51 videosUUID.push(uuid) 56 videosUUID.push(uuid)
@@ -81,27 +86,29 @@ describe('Test create transcoding jobs', function () {
81 let infoHashes: { [id: number]: string } 86 let infoHashes: { [id: number]: string }
82 87
83 for (const video of data) { 88 for (const video of data) {
84 const videoDetail = await server.videos.get({ id: video.uuid }) 89 const videoDetails = await server.videos.get({ id: video.uuid })
85 90
86 if (video.uuid === videosUUID[1]) { 91 if (video.uuid === videosUUID[1]) {
87 expect(videoDetail.files).to.have.lengthOf(4) 92 expect(videoDetails.files).to.have.lengthOf(4)
88 expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) 93 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
94
95 if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
89 96
90 if (!infoHashes) { 97 if (!infoHashes) {
91 infoHashes = {} 98 infoHashes = {}
92 99
93 for (const file of videoDetail.files) { 100 for (const file of videoDetails.files) {
94 infoHashes[file.resolution.id.toString()] = file.magnetUri 101 infoHashes[file.resolution.id.toString()] = file.magnetUri
95 } 102 }
96 } else { 103 } else {
97 for (const resolution of Object.keys(infoHashes)) { 104 for (const resolution of Object.keys(infoHashes)) {
98 const file = videoDetail.files.find(f => f.resolution.id.toString() === resolution) 105 const file = videoDetails.files.find(f => f.resolution.id.toString() === resolution)
99 expect(file.magnetUri).to.equal(infoHashes[resolution]) 106 expect(file.magnetUri).to.equal(infoHashes[resolution])
100 } 107 }
101 } 108 }
102 } else { 109 } else {
103 expect(videoDetail.files).to.have.lengthOf(1) 110 expect(videoDetails.files).to.have.lengthOf(1)
104 expect(videoDetail.streamingPlaylists).to.have.lengthOf(0) 111 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
105 } 112 }
106 } 113 }
107 } 114 }
@@ -125,6 +132,8 @@ describe('Test create transcoding jobs', function () {
125 expect(videoDetails.files[1].resolution.id).to.equal(480) 132 expect(videoDetails.files[1].resolution.id).to.equal(480)
126 133
127 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) 134 expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
135
136 if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
128 } 137 }
129 }) 138 })
130 139
@@ -139,11 +148,15 @@ describe('Test create transcoding jobs', function () {
139 const videoDetails = await server.videos.get({ id: videosUUID[2] }) 148 const videoDetails = await server.videos.get({ id: videosUUID[2] })
140 149
141 expect(videoDetails.files).to.have.lengthOf(1) 150 expect(videoDetails.files).to.have.lengthOf(1)
151 if (objectStorage) await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
152
142 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) 153 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
143 154
144 const files = videoDetails.streamingPlaylists[0].files 155 const files = videoDetails.streamingPlaylists[0].files
145 expect(files).to.have.lengthOf(1) 156 expect(files).to.have.lengthOf(1)
146 expect(files[0].resolution.id).to.equal(480) 157 expect(files[0].resolution.id).to.equal(480)
158
159 if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
147 } 160 }
148 }) 161 })
149 162
@@ -160,6 +173,8 @@ describe('Test create transcoding jobs', function () {
160 const files = videoDetails.streamingPlaylists[0].files 173 const files = videoDetails.streamingPlaylists[0].files
161 expect(files).to.have.lengthOf(1) 174 expect(files).to.have.lengthOf(1)
162 expect(files[0].resolution.id).to.equal(480) 175 expect(files[0].resolution.id).to.equal(480)
176
177 if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
163 } 178 }
164 }) 179 })
165 180
@@ -178,15 +193,15 @@ describe('Test create transcoding jobs', function () {
178 193
179 const files = videoDetails.streamingPlaylists[0].files 194 const files = videoDetails.streamingPlaylists[0].files
180 expect(files).to.have.lengthOf(4) 195 expect(files).to.have.lengthOf(4)
196
197 if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
181 } 198 }
182 }) 199 })
183 200
184 it('Should optimize the video file and generate HLS videos if enabled in config', async function () { 201 it('Should optimize the video file and generate HLS videos if enabled in config', async function () {
185 this.timeout(120000) 202 this.timeout(120000)
186 203
187 config.transcoding.hls.enabled = true 204 await servers[0].config.enableTranscoding()
188 await servers[0].config.updateCustomSubConfig({ newConfig: config })
189
190 await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`) 205 await servers[0].cli.execWithEnv(`npm run create-transcoding-job -- -v ${videosUUID[4]}`)
191 206
192 await waitJobs(servers) 207 await waitJobs(servers)
@@ -197,10 +212,28 @@ describe('Test create transcoding jobs', function () {
197 expect(videoDetails.files).to.have.lengthOf(4) 212 expect(videoDetails.files).to.have.lengthOf(4)
198 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) 213 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
199 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4) 214 expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(4)
215
216 if (objectStorage) {
217 await checkFilesInObjectStorage(videoDetails.files, 'webtorrent')
218 await checkFilesInObjectStorage(videoDetails.streamingPlaylists[0].files, 'playlist')
219 }
200 } 220 }
201 }) 221 })
202 222
203 after(async function () { 223 after(async function () {
204 await cleanupTests(servers) 224 await cleanupTests(servers)
205 }) 225 })
226}
227
228describe('Test create transcoding jobs', function () {
229
230 describe('On filesystem', function () {
231 runTests(false)
232 })
233
234 describe('On object storage', function () {
235 if (areObjectStorageTestsDisabled()) return
236
237 runTests(true)
238 })
206}) 239})
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts
index 7f7873df3..c9a2eb831 100644
--- a/server/tests/helpers/request.ts
+++ b/server/tests/helpers/request.ts
@@ -13,7 +13,7 @@ describe('Request helpers', function () {
13 13
14 it('Should throw an error when the bytes limit is exceeded for request', async function () { 14 it('Should throw an error when the bytes limit is exceeded for request', async function () {
15 try { 15 try {
16 await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 3 }) 16 await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 })
17 } catch { 17 } catch {
18 return 18 return
19 } 19 }
@@ -23,7 +23,7 @@ describe('Request helpers', function () {
23 23
24 it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { 24 it('Should throw an error when the bytes limit is exceeded for request and save file', async function () {
25 try { 25 try {
26 await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath1, { bodyKBLimit: 3 }) 26 await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 })
27 } catch { 27 } catch {
28 28
29 await wait(500) 29 await wait(500)
@@ -35,8 +35,8 @@ describe('Request helpers', function () {
35 }) 35 })
36 36
37 it('Should succeed if the file is below the limit', async function () { 37 it('Should succeed if the file is below the limit', async function () {
38 await doRequest(FIXTURE_URLS.video4K, { bodyKBLimit: 5 }) 38 await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 })
39 await doRequestAndSaveToFile(FIXTURE_URLS.video4K, destPath2, { bodyKBLimit: 5 }) 39 await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 })
40 40
41 expect(await pathExists(destPath2)).to.be.true 41 expect(await pathExists(destPath2)).to.be.true
42 }) 42 })
diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts
index 7fc92f804..aa2c8e8fa 100644
--- a/shared/extra-utils/miscs/checks.ts
+++ b/shared/extra-utils/miscs/checks.ts
@@ -16,6 +16,10 @@ function dateIsValid (dateString: string, interval = 300000) {
16 return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval 16 return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval
17} 17}
18 18
19function expectStartWith (str: string, start: string) {
20 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
21}
22
19async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { 23async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
20 const res = await makeGetRequest({ 24 const res = await makeGetRequest({
21 url, 25 url,
@@ -42,5 +46,6 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f
42export { 46export {
43 dateIsValid, 47 dateIsValid,
44 testImage, 48 testImage,
45 testFileExistsOrNot 49 testFileExistsOrNot,
50 expectStartWith
46} 51}
diff --git a/shared/extra-utils/miscs/tests.ts b/shared/extra-utils/miscs/tests.ts
index 3dfb2487e..dd86041fe 100644
--- a/shared/extra-utils/miscs/tests.ts
+++ b/shared/extra-utils/miscs/tests.ts
@@ -28,7 +28,9 @@ const FIXTURE_URLS = {
28 28
29 badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', 29 badVideo: 'https://download.cpy.re/peertube/bad_video.mp4',
30 goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', 30 goodVideo: 'https://download.cpy.re/peertube/good_video.mp4',
31 video4K: 'https://download.cpy.re/peertube/4k_file.txt' 31 goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4',
32
33 file4K: 'https://download.cpy.re/peertube/4k_file.txt'
32} 34}
33 35
34function parallelTests () { 36function parallelTests () {
@@ -42,7 +44,15 @@ function isGithubCI () {
42function areHttpImportTestsDisabled () { 44function areHttpImportTestsDisabled () {
43 const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' 45 const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true'
44 46
45 if (disabled) console.log('Import tests are disabled') 47 if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled')
48
49 return disabled
50}
51
52function areObjectStorageTestsDisabled () {
53 const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true'
54
55 if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled')
46 56
47 return disabled 57 return disabled
48} 58}
@@ -89,6 +99,7 @@ export {
89 buildAbsoluteFixturePath, 99 buildAbsoluteFixturePath,
90 getFileSize, 100 getFileSize,
91 buildRequestStub, 101 buildRequestStub,
102 areObjectStorageTestsDisabled,
92 wait, 103 wait,
93 root 104 root
94} 105}
diff --git a/shared/extra-utils/mock-servers/index.ts b/shared/extra-utils/mock-servers/index.ts
index 0ec07f685..93c00c788 100644
--- a/shared/extra-utils/mock-servers/index.ts
+++ b/shared/extra-utils/mock-servers/index.ts
@@ -2,3 +2,4 @@ export * from './mock-email'
2export * from './mock-instances-index' 2export * from './mock-instances-index'
3export * from './mock-joinpeertube-versions' 3export * from './mock-joinpeertube-versions'
4export * from './mock-plugin-blocklist' 4export * from './mock-plugin-blocklist'
5export * from './mock-object-storage'
diff --git a/shared/extra-utils/mock-servers/mock-object-storage.ts b/shared/extra-utils/mock-servers/mock-object-storage.ts
new file mode 100644
index 000000000..19ea7c87c
--- /dev/null
+++ b/shared/extra-utils/mock-servers/mock-object-storage.ts
@@ -0,0 +1,42 @@
1import * as express from 'express'
2import got, { RequestError } from 'got'
3import { Server } from 'http'
4import { pipeline } from 'stream'
5import { randomInt } from '@shared/core-utils'
6import { ObjectStorageCommand } from '../server'
7
8export class MockObjectStorage {
9 private server: Server
10
11 initialize () {
12 return new Promise<number>(res => {
13 const app = express()
14
15 app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {
16 const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}`
17
18 if (process.env.DEBUG) {
19 console.log('Receiving request on mocked server %s.', req.url)
20 console.log('Proxifying request to %s', url)
21 }
22
23 return pipeline(
24 got.stream(url, { throwHttpErrors: false }),
25 res,
26 (err: RequestError) => {
27 if (!err) return
28
29 console.error('Pipeline failed.', err)
30 }
31 )
32 })
33
34 const port = 42301 + randomInt(1, 100)
35 this.server = app.listen(port, () => res(port))
36 })
37 }
38
39 terminate () {
40 if (this.server) this.server.close()
41 }
42}
diff --git a/shared/extra-utils/requests/requests.ts b/shared/extra-utils/requests/requests.ts
index 70f790222..e3ecd1af2 100644
--- a/shared/extra-utils/requests/requests.ts
+++ b/shared/extra-utils/requests/requests.ts
@@ -121,6 +121,20 @@ function unwrapText (test: request.Test): Promise<string> {
121 return test.then(res => res.text) 121 return test.then(res => res.text)
122} 122}
123 123
124function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> {
125 return test.then(res => {
126 if (res.body instanceof Buffer) {
127 return JSON.parse(new TextDecoder().decode(res.body))
128 }
129
130 return res.body
131 })
132}
133
134function unwrapTextOrDecode (test: request.Test): Promise<string> {
135 return test.then(res => res.text || new TextDecoder().decode(res.body))
136}
137
124// --------------------------------------------------------------------------- 138// ---------------------------------------------------------------------------
125 139
126export { 140export {
@@ -134,6 +148,8 @@ export {
134 makeRawRequest, 148 makeRawRequest,
135 makeActivityPubGetRequest, 149 makeActivityPubGetRequest,
136 unwrapBody, 150 unwrapBody,
151 unwrapTextOrDecode,
152 unwrapBodyOrDecodeToJSON,
137 unwrapText 153 unwrapText
138} 154}
139 155
diff --git a/shared/extra-utils/server/config-command.ts b/shared/extra-utils/server/config-command.ts
index 11148aa46..51d04fa63 100644
--- a/shared/extra-utils/server/config-command.ts
+++ b/shared/extra-utils/server/config-command.ts
@@ -18,6 +18,70 @@ export class ConfigCommand extends AbstractCommand {
18 } 18 }
19 } 19 }
20 20
21 enableImports () {
22 return this.updateExistingSubConfig({
23 newConfig: {
24 import: {
25 videos: {
26 http: {
27 enabled: true
28 },
29
30 torrent: {
31 enabled: true
32 }
33 }
34 }
35 }
36 })
37 }
38
39 enableLive (options: {
40 allowReplay?: boolean
41 transcoding?: boolean
42 } = {}) {
43 return this.updateExistingSubConfig({
44 newConfig: {
45 live: {
46 enabled: true,
47 allowReplay: options.allowReplay ?? true,
48 transcoding: {
49 enabled: options.transcoding ?? true,
50 resolutions: ConfigCommand.getCustomConfigResolutions(true)
51 }
52 }
53 }
54 })
55 }
56
57 disableTranscoding () {
58 return this.updateExistingSubConfig({
59 newConfig: {
60 transcoding: {
61 enabled: false
62 }
63 }
64 })
65 }
66
67 enableTranscoding (webtorrent = true, hls = true) {
68 return this.updateExistingSubConfig({
69 newConfig: {
70 transcoding: {
71 enabled: true,
72 resolutions: ConfigCommand.getCustomConfigResolutions(true),
73
74 webtorrent: {
75 enabled: webtorrent
76 },
77 hls: {
78 enabled: hls
79 }
80 }
81 }
82 })
83 }
84
21 getConfig (options: OverrideCommandOptions = {}) { 85 getConfig (options: OverrideCommandOptions = {}) {
22 const path = '/api/v1/config' 86 const path = '/api/v1/config'
23 87
@@ -81,6 +145,14 @@ export class ConfigCommand extends AbstractCommand {
81 }) 145 })
82 } 146 }
83 147
148 async updateExistingSubConfig (options: OverrideCommandOptions & {
149 newConfig: DeepPartial<CustomConfig>
150 }) {
151 const existing = await this.getCustomConfig(options)
152
153 return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) })
154 }
155
84 updateCustomSubConfig (options: OverrideCommandOptions & { 156 updateCustomSubConfig (options: OverrideCommandOptions & {
85 newConfig: DeepPartial<CustomConfig> 157 newConfig: DeepPartial<CustomConfig>
86 }) { 158 }) {
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts
index 9055dfc57..92ff7a0f9 100644
--- a/shared/extra-utils/server/index.ts
+++ b/shared/extra-utils/server/index.ts
@@ -6,6 +6,7 @@ export * from './follows-command'
6export * from './follows' 6export * from './follows'
7export * from './jobs' 7export * from './jobs'
8export * from './jobs-command' 8export * from './jobs-command'
9export * from './object-storage-command'
9export * from './plugins-command' 10export * from './plugins-command'
10export * from './plugins' 11export * from './plugins'
11export * from './redundancy-command' 12export * from './redundancy-command'
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts
index c4eb12dc2..91771c176 100644
--- a/shared/extra-utils/server/jobs-command.ts
+++ b/shared/extra-utils/server/jobs-command.ts
@@ -5,6 +5,16 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared'
5 5
6export class JobsCommand extends AbstractCommand { 6export class JobsCommand extends AbstractCommand {
7 7
8 async getLatest (options: OverrideCommandOptions & {
9 jobType: JobType
10 }) {
11 const { data } = await this.getJobsList({ ...options, start: 0, count: 1, sort: '-createdAt' })
12
13 if (data.length === 0) return undefined
14
15 return data[0]
16 }
17
8 getJobsList (options: OverrideCommandOptions & { 18 getJobsList (options: OverrideCommandOptions & {
9 state?: JobState 19 state?: JobState
10 jobType?: JobType 20 jobType?: JobType
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts
index 64a0353eb..27104bfdf 100644
--- a/shared/extra-utils/server/jobs.ts
+++ b/shared/extra-utils/server/jobs.ts
@@ -3,7 +3,7 @@ import { JobState } from '../../models'
3import { wait } from '../miscs' 3import { wait } from '../miscs'
4import { PeerTubeServer } from './server' 4import { PeerTubeServer } from './server'
5 5
6async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { 6async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer, skipDelayed = false) {
7 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT 7 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT
8 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) 8 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10)
9 : 250 9 : 250
@@ -13,7 +13,9 @@ async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) {
13 if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] 13 if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ]
14 else servers = serversArg as PeerTubeServer[] 14 else servers = serversArg as PeerTubeServer[]
15 15
16 const states: JobState[] = [ 'waiting', 'active', 'delayed' ] 16 const states: JobState[] = [ 'waiting', 'active' ]
17 if (!skipDelayed) states.push('delayed')
18
17 const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] 19 const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ]
18 let pendingRequests: boolean 20 let pendingRequests: boolean
19 21
diff --git a/shared/extra-utils/server/object-storage-command.ts b/shared/extra-utils/server/object-storage-command.ts
new file mode 100644
index 000000000..b4de8f4cb
--- /dev/null
+++ b/shared/extra-utils/server/object-storage-command.ts
@@ -0,0 +1,77 @@
1
2import { HttpStatusCode } from '@shared/models'
3import { makePostBodyRequest } from '../requests'
4import { AbstractCommand } from '../shared'
5
6export class ObjectStorageCommand extends AbstractCommand {
7 static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists'
8 static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos'
9
10 static getDefaultConfig () {
11 return {
12 object_storage: {
13 enabled: true,
14 endpoint: 'http://' + this.getEndpointHost(),
15 region: this.getRegion(),
16
17 credentials: this.getCredentialsConfig(),
18
19 streaming_playlists: {
20 bucket_name: this.DEFAULT_PLAYLIST_BUCKET
21 },
22
23 videos: {
24 bucket_name: this.DEFAULT_WEBTORRENT_BUCKET
25 }
26 }
27 }
28 }
29
30 static getCredentialsConfig () {
31 return {
32 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
33 secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
34 }
35 }
36
37 static getEndpointHost () {
38 return 'localhost:9444'
39 }
40
41 static getRegion () {
42 return 'us-east-1'
43 }
44
45 static getWebTorrentBaseUrl () {
46 return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/`
47 }
48
49 static getPlaylistBaseUrl () {
50 return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/`
51 }
52
53 static async prepareDefaultBuckets () {
54 await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET)
55 await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET)
56 }
57
58 static async createBucket (name: string) {
59 await makePostBodyRequest({
60 url: this.getEndpointHost(),
61 path: '/ui/' + name + '?delete',
62 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
63 })
64
65 await makePostBodyRequest({
66 url: this.getEndpointHost(),
67 path: '/ui/' + name + '?create',
68 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
69 })
70
71 await makePostBodyRequest({
72 url: this.getEndpointHost(),
73 path: '/ui/' + name + '?make-public',
74 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
75 })
76 }
77}
diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts
index 3c335b8e4..bc5e1cd5f 100644
--- a/shared/extra-utils/server/server.ts
+++ b/shared/extra-utils/server/server.ts
@@ -38,11 +38,13 @@ import { PluginsCommand } from './plugins-command'
38import { RedundancyCommand } from './redundancy-command' 38import { RedundancyCommand } from './redundancy-command'
39import { ServersCommand } from './servers-command' 39import { ServersCommand } from './servers-command'
40import { StatsCommand } from './stats-command' 40import { StatsCommand } from './stats-command'
41import { ObjectStorageCommand } from './object-storage-command'
41 42
42export type RunServerOptions = { 43export type RunServerOptions = {
43 hideLogs?: boolean 44 hideLogs?: boolean
44 nodeArgs?: string[] 45 nodeArgs?: string[]
45 peertubeArgs?: string[] 46 peertubeArgs?: string[]
47 env?: { [ id: string ]: string }
46} 48}
47 49
48export class PeerTubeServer { 50export class PeerTubeServer {
@@ -121,6 +123,7 @@ export class PeerTubeServer {
121 servers?: ServersCommand 123 servers?: ServersCommand
122 login?: LoginCommand 124 login?: LoginCommand
123 users?: UsersCommand 125 users?: UsersCommand
126 objectStorage?: ObjectStorageCommand
124 videos?: VideosCommand 127 videos?: VideosCommand
125 128
126 constructor (options: { serverNumber: number } | { url: string }) { 129 constructor (options: { serverNumber: number } | { url: string }) {
@@ -202,6 +205,10 @@ export class PeerTubeServer {
202 env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() 205 env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
203 env['NODE_CONFIG'] = JSON.stringify(configOverride) 206 env['NODE_CONFIG'] = JSON.stringify(configOverride)
204 207
208 if (options.env) {
209 Object.assign(env, options.env)
210 }
211
205 const forkOptions = { 212 const forkOptions = {
206 silent: true, 213 silent: true,
207 env, 214 env,
@@ -209,10 +216,17 @@ export class PeerTubeServer {
209 execArgv: options.nodeArgs || [] 216 execArgv: options.nodeArgs || []
210 } 217 }
211 218
212 return new Promise<void>(res => { 219 return new Promise<void>((res, rej) => {
213 const self = this 220 const self = this
214 221
215 this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) 222 this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions)
223
224 const onExit = function () {
225 return rej(new Error('Process exited'))
226 }
227
228 this.app.on('exit', onExit)
229
216 this.app.stdout.on('data', function onStdout (data) { 230 this.app.stdout.on('data', function onStdout (data) {
217 let dontContinue = false 231 let dontContinue = false
218 232
@@ -241,6 +255,7 @@ export class PeerTubeServer {
241 console.log(data.toString()) 255 console.log(data.toString())
242 } else { 256 } else {
243 self.app.stdout.removeListener('data', onStdout) 257 self.app.stdout.removeListener('data', onStdout)
258 self.app.removeListener('exit', onExit)
244 } 259 }
245 260
246 process.on('exit', () => { 261 process.on('exit', () => {
@@ -365,5 +380,6 @@ export class PeerTubeServer {
365 this.login = new LoginCommand(this) 380 this.login = new LoginCommand(this)
366 this.users = new UsersCommand(this) 381 this.users = new UsersCommand(this)
367 this.videos = new VideosCommand(this) 382 this.videos = new VideosCommand(this)
383 this.objectStorage = new ObjectStorageCommand(this)
368 } 384 }
369} 385}
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts
index f0622feb0..21ab9405b 100644
--- a/shared/extra-utils/server/servers.ts
+++ b/shared/extra-utils/server/servers.ts
@@ -10,11 +10,11 @@ async function createSingleServer (serverNumber: number, configOverride?: Object
10 return server 10 return server
11} 11}
12 12
13function createMultipleServers (totalServers: number, configOverride?: Object) { 13function createMultipleServers (totalServers: number, configOverride?: Object, options: RunServerOptions = {}) {
14 const serverPromises: Promise<PeerTubeServer>[] = [] 14 const serverPromises: Promise<PeerTubeServer>[] = []
15 15
16 for (let i = 1; i <= totalServers; i++) { 16 for (let i = 1; i <= totalServers; i++) {
17 serverPromises.push(createSingleServer(i, configOverride)) 17 serverPromises.push(createSingleServer(i, configOverride, options))
18 } 18 }
19 19
20 return Promise.all(serverPromises) 20 return Promise.all(serverPromises)
diff --git a/shared/extra-utils/videos/live-command.ts b/shared/extra-utils/videos/live-command.ts
index 81ae458e0..74f5d3089 100644
--- a/shared/extra-utils/videos/live-command.ts
+++ b/shared/extra-utils/videos/live-command.ts
@@ -126,7 +126,7 @@ export class LiveCommand extends AbstractCommand {
126 video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) 126 video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId })
127 127
128 await wait(500) 128 await wait(500)
129 } while (video.isLive === true && video.state.id !== VideoState.PUBLISHED) 129 } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED)
130 } 130 }
131 131
132 async countPlaylists (options: OverrideCommandOptions & { 132 async countPlaylists (options: OverrideCommandOptions & {
diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts
index 9a6df07a8..29f99ed6d 100644
--- a/shared/extra-utils/videos/live.ts
+++ b/shared/extra-utils/videos/live.ts
@@ -89,6 +89,12 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi
89 } 89 }
90} 90}
91 91
92async function waitUntilLiveSavedOnAllServers (servers: PeerTubeServer[], videoId: string) {
93 for (const server of servers) {
94 await server.live.waitUntilSaved({ videoId })
95 }
96}
97
92async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { 98async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
93 const basePath = server.servers.buildDirectory('streaming-playlists') 99 const basePath = server.servers.buildDirectory('streaming-playlists')
94 const hlsPath = join(basePath, 'hls', videoUUID) 100 const hlsPath = join(basePath, 'hls', videoUUID)
@@ -126,5 +132,6 @@ export {
126 testFfmpegStreamError, 132 testFfmpegStreamError,
127 stopFfmpeg, 133 stopFfmpeg,
128 waitUntilLivePublishedOnAllServers, 134 waitUntilLivePublishedOnAllServers,
135 waitUntilLiveSavedOnAllServers,
129 checkLiveCleanupAfterSave 136 checkLiveCleanupAfterSave
130} 137}
diff --git a/shared/extra-utils/videos/streaming-playlists-command.ts b/shared/extra-utils/videos/streaming-playlists-command.ts
index 9662685da..5d40d35cb 100644
--- a/shared/extra-utils/videos/streaming-playlists-command.ts
+++ b/shared/extra-utils/videos/streaming-playlists-command.ts
@@ -1,5 +1,5 @@
1import { HttpStatusCode } from '@shared/models' 1import { HttpStatusCode } from '@shared/models'
2import { unwrapBody, unwrapText } from '../requests' 2import { unwrapBody, unwrapTextOrDecode, unwrapBodyOrDecodeToJSON } from '../requests'
3import { AbstractCommand, OverrideCommandOptions } from '../shared' 3import { AbstractCommand, OverrideCommandOptions } from '../shared'
4 4
5export class StreamingPlaylistsCommand extends AbstractCommand { 5export class StreamingPlaylistsCommand extends AbstractCommand {
@@ -7,7 +7,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
7 get (options: OverrideCommandOptions & { 7 get (options: OverrideCommandOptions & {
8 url: string 8 url: string
9 }) { 9 }) {
10 return unwrapText(this.getRawRequest({ 10 return unwrapTextOrDecode(this.getRawRequest({
11 ...options, 11 ...options,
12 12
13 url: options.url, 13 url: options.url,
@@ -33,7 +33,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
33 getSegmentSha256 (options: OverrideCommandOptions & { 33 getSegmentSha256 (options: OverrideCommandOptions & {
34 url: string 34 url: string
35 }) { 35 }) {
36 return unwrapBody<{ [ id: string ]: string }>(this.getRawRequest({ 36 return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({
37 ...options, 37 ...options,
38 38
39 url: options.url, 39 url: options.url,
diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts
index a224b8f5f..6671e3fa6 100644
--- a/shared/extra-utils/videos/streaming-playlists.ts
+++ b/shared/extra-utils/videos/streaming-playlists.ts
@@ -9,17 +9,16 @@ async function checkSegmentHash (options: {
9 server: PeerTubeServer 9 server: PeerTubeServer
10 baseUrlPlaylist: string 10 baseUrlPlaylist: string
11 baseUrlSegment: string 11 baseUrlSegment: string
12 videoUUID: string
13 resolution: number 12 resolution: number
14 hlsPlaylist: VideoStreamingPlaylist 13 hlsPlaylist: VideoStreamingPlaylist
15}) { 14}) {
16 const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options 15 const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist } = options
17 const command = server.streamingPlaylists 16 const command = server.streamingPlaylists
18 17
19 const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) 18 const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
20 const videoName = basename(file.fileUrl) 19 const videoName = basename(file.fileUrl)
21 20
22 const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) 21 const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8` })
23 22
24 const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) 23 const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
25 24
@@ -28,7 +27,7 @@ async function checkSegmentHash (options: {
28 const range = `${offset}-${offset + length - 1}` 27 const range = `${offset}-${offset + length - 1}`
29 28
30 const segmentBody = await command.getSegment({ 29 const segmentBody = await command.getSegment({
31 url: `${baseUrlSegment}/${videoUUID}/${videoName}`, 30 url: `${baseUrlSegment}/${videoName}`,
32 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, 31 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206,
33 range: `bytes=${range}` 32 range: `bytes=${range}`
34 }) 33 })
diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts
index 33725bfdc..d35339c8d 100644
--- a/shared/extra-utils/videos/videos-command.ts
+++ b/shared/extra-utils/videos/videos-command.ts
@@ -188,6 +188,17 @@ export class VideosCommand extends AbstractCommand {
188 return id 188 return id
189 } 189 }
190 190
191 async listFiles (options: OverrideCommandOptions & {
192 id: number | string
193 }) {
194 const video = await this.get(options)
195
196 const files = video.files || []
197 const hlsFiles = video.streamingPlaylists[0]?.files || []
198
199 return files.concat(hlsFiles)
200 }
201
191 // --------------------------------------------------------------------------- 202 // ---------------------------------------------------------------------------
192 203
193 listMyVideos (options: OverrideCommandOptions & { 204 listMyVideos (options: OverrideCommandOptions & {
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 4ab249e0b..ff96283a4 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -19,6 +19,7 @@ export type JobType =
19 | 'video-redundancy' 19 | 'video-redundancy'
20 | 'video-live-ending' 20 | 'video-live-ending'
21 | 'actor-keys' 21 | 'actor-keys'
22 | 'move-to-object-storage'
22 23
23export interface Job { 24export interface Job {
24 id: number 25 id: number
@@ -136,3 +137,8 @@ export interface VideoLiveEndingPayload {
136export interface ActorKeysPayload { 137export interface ActorKeysPayload {
137 actorId: number 138 actorId: number
138} 139}
140
141export interface MoveObjectStoragePayload {
142 videoUUID: string
143 isNewVideo: boolean
144}
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts
index faa9b9868..733c433a0 100644
--- a/shared/models/videos/index.ts
+++ b/shared/models/videos/index.ts
@@ -26,6 +26,7 @@ export * from './video-resolution.enum'
26export * from './video-schedule-update.model' 26export * from './video-schedule-update.model'
27export * from './video-sort-field.type' 27export * from './video-sort-field.type'
28export * from './video-state.enum' 28export * from './video-state.enum'
29export * from './video-storage.enum'
29 30
30export * from './video-streaming-playlist.model' 31export * from './video-streaming-playlist.model'
31export * from './video-streaming-playlist.type' 32export * from './video-streaming-playlist.type'
diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts
index 49d997f24..c6af481e7 100644
--- a/shared/models/videos/video-state.enum.ts
+++ b/shared/models/videos/video-state.enum.ts
@@ -3,5 +3,6 @@ export const enum VideoState {
3 TO_TRANSCODE = 2, 3 TO_TRANSCODE = 2,
4 TO_IMPORT = 3, 4 TO_IMPORT = 3,
5 WAITING_FOR_LIVE = 4, 5 WAITING_FOR_LIVE = 4,
6 LIVE_ENDED = 5 6 LIVE_ENDED = 5,
7 TO_MOVE_TO_EXTERNAL_STORAGE = 6
7} 8}
diff --git a/shared/models/videos/video-storage.enum.ts b/shared/models/videos/video-storage.enum.ts
new file mode 100644
index 000000000..7c6690db2
--- /dev/null
+++ b/shared/models/videos/video-storage.enum.ts
@@ -0,0 +1,4 @@
1export const enum VideoStorage {
2 FILE_SYSTEM,
3 OBJECT_STORAGE,
4}
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml
index ce0f89d7b..1b474582a 100644
--- a/support/docker/production/config/custom-environment-variables.yaml
+++ b/support/docker/production/config/custom-environment-variables.yaml
@@ -45,6 +45,29 @@ smtp:
45 __format: "json" 45 __format: "json"
46 from_address: "PEERTUBE_SMTP_FROM" 46 from_address: "PEERTUBE_SMTP_FROM"
47 47
48object_storage:
49 enabled:
50 __name: "PEERTUBE_OBJECT_STORAGE_ENABLED"
51 __format: "json"
52
53 endpoint: "PEERTUBE_OBJECT_STORAGE_ENDPOINT"
54
55 region: "PEERTUBE_OBJECT_STORAGE_REGION"
56
57 max_upload_part:
58 __name: "PEERTUBE_OBJECT_STORAGE_MAX_UPLOAD_PART"
59 __format: "json"
60
61 streaming_playlists:
62 bucket_name: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BUCKET_NAME"
63 prefix: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_PREFIX"
64 base_url: "PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BASE_URL"
65
66 videos:
67 bucket_name: "PEERTUBE_OBJECT_STORAGE_VIDEOS_BUCKET_NAME"
68 prefix: "PEERTUBE_OBJECT_STORAGE_VIDEOS_PREFIX"
69 base_url: "PEERTUBE_OBJECT_STORAGE_VIDEOS_BASE_URL"
70
48log: 71log:
49 level: "PEERTUBE_LOG_LEVEL" 72 level: "PEERTUBE_LOG_LEVEL"
50 log_ping_requests: 73 log_ping_requests:
diff --git a/yarn.lock b/yarn.lock
index 5590d255d..0f4fe3938 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -49,6 +49,770 @@
49 resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" 49 resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06"
50 integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== 50 integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg==
51 51
52"@aws-crypto/crc32@^1.0.0":
53 version "1.1.0"
54 resolved "https://registry.yarnpkg.com/@aws-crypto/crc32/-/crc32-1.1.0.tgz#aff048e207798fad0b0e7765b12d474c273779b6"
55 integrity sha512-ifvfaaJVvT+JUTi3zSkX4wtuGGVJrAcjN7ftg+JiE/frNBP3zNwo4xipzWBsMLZfNuzMZuaesEYyqkZcs5tzCQ==
56 dependencies:
57 tslib "^1.11.1"
58
59"@aws-crypto/ie11-detection@^1.0.0":
60 version "1.0.0"
61 resolved "https://registry.yarnpkg.com/@aws-crypto/ie11-detection/-/ie11-detection-1.0.0.tgz#d3a6af29ba7f15458f79c41d1cd8cac3925e726a"
62 integrity sha512-kCKVhCF1oDxFYgQrxXmIrS5oaWulkvRcPz+QBDMsUr2crbF4VGgGT6+uQhSwJFdUAQ2A//Vq+uT83eJrkzFgXA==
63 dependencies:
64 tslib "^1.11.1"
65
66"@aws-crypto/sha256-browser@^1.0.0":
67 version "1.1.1"
68 resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-browser/-/sha256-browser-1.1.1.tgz#85dddf13e8f8d74c0d6592d993e4bf401da9f420"
69 integrity sha512-nS4vdan97It6HcweV58WXtjPbPSc0JXd3sAwlw3Ou5Mc3WllSycAS32Tv2LRn8butNQoU9AE3jEQAOgiMdNC1Q==
70 dependencies:
71 "@aws-crypto/ie11-detection" "^1.0.0"
72 "@aws-crypto/sha256-js" "^1.1.0"
73 "@aws-crypto/supports-web-crypto" "^1.0.0"
74 "@aws-sdk/types" "^3.1.0"
75 "@aws-sdk/util-locate-window" "^3.0.0"
76 "@aws-sdk/util-utf8-browser" "^3.0.0"
77 tslib "^1.11.1"
78
79"@aws-crypto/sha256-js@^1.0.0", "@aws-crypto/sha256-js@^1.1.0":
80 version "1.1.0"
81 resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-js/-/sha256-js-1.1.0.tgz#a58386ad18186e392e0f1d98d18831261d27b071"
82 integrity sha512-VIhuqbPgXDVr8sZe2yhgQcDRRmzf4CI8fmC1A3bHiRfE6wlz1d8KpeemqbuoEHotz/Dch9yOxlshyQDNjNFeHA==
83 dependencies:
84 "@aws-sdk/types" "^3.1.0"
85 "@aws-sdk/util-utf8-browser" "^3.0.0"
86 tslib "^1.11.1"
87
88"@aws-crypto/supports-web-crypto@^1.0.0":
89 version "1.0.0"
90 resolved "https://registry.yarnpkg.com/@aws-crypto/supports-web-crypto/-/supports-web-crypto-1.0.0.tgz#c40901bc17ac1e875e248df16a2b47ad8bfd9a93"
91 integrity sha512-IHLfv+WmVH89EW4n6a5eE8/hUlz6qkWGMn/v4r5ZgzcXdTC5nolii2z3k46y01hWRiC2PPhOdeSLzMUCUMco7g==
92 dependencies:
93 tslib "^1.11.1"
94
95"@aws-sdk/abort-controller@3.25.0":
96 version "3.25.0"
97 resolved "https://registry.yarnpkg.com/@aws-sdk/abort-controller/-/abort-controller-3.25.0.tgz#a9ea250140de378d8beb6d2f427067fa30423e9e"
98 integrity sha512-uEVKqKkPVz6atbCxCNJY5O7V+ieSK8crUswXo8/WePyEbGEgxJ4t9x/WG4lV8kBjelmvQHDR4GqfJmb5Sh9xSg==
99 dependencies:
100 "@aws-sdk/types" "3.25.0"
101 tslib "^2.3.0"
102
103"@aws-sdk/chunked-blob-reader-native@3.23.0":
104 version "3.23.0"
105 resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.23.0.tgz#72d711e3cc904bb380e99cdd60c59deacd1596ac"
106 integrity sha512-Ya5f8Ntv0EyZw+AHkpV6n6qqHzpCDNlkX50uj/dwFCMmPiHFWsWMvd0Qu04Y7miycJINEatRrJ5V8r/uVvZIDg==
107 dependencies:
108 "@aws-sdk/util-base64-browser" "3.23.0"
109 tslib "^2.3.0"
110
111"@aws-sdk/chunked-blob-reader@3.23.0":
112 version "3.23.0"
113 resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader/-/chunked-blob-reader-3.23.0.tgz#83eb6a437172b671e699850378bcb558e15374ec"
114 integrity sha512-gmJhCuXrKOOumppviE4K30NvsIQIqqxbGDNptrJrMYBO0qXCbK8/BypZ/hS/oT3loDzlSIxG2z5GDL/va9lbFw==
115 dependencies:
116 tslib "^2.3.0"
117
118"@aws-sdk/client-s3@^3.23.0":
119 version "3.25.0"
120 resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.25.0.tgz#6b8146c18e76606378c5f736554cb41ad4ce229e"
121 integrity sha512-yVDPo6x7DCt9t833SjqWI+AQBx81/m54gLF/ePQZBeHL5mPMEyMXTF0o22yUP5t8f92U2VAyRCP2NvKtB9WgBg==
122 dependencies:
123 "@aws-crypto/sha256-browser" "^1.0.0"
124 "@aws-crypto/sha256-js" "^1.0.0"
125 "@aws-sdk/client-sts" "3.25.0"
126 "@aws-sdk/config-resolver" "3.25.0"
127 "@aws-sdk/credential-provider-node" "3.25.0"
128 "@aws-sdk/eventstream-serde-browser" "3.25.0"
129 "@aws-sdk/eventstream-serde-config-resolver" "3.25.0"
130 "@aws-sdk/eventstream-serde-node" "3.25.0"
131 "@aws-sdk/fetch-http-handler" "3.25.0"
132 "@aws-sdk/hash-blob-browser" "3.25.0"
133 "@aws-sdk/hash-node" "3.25.0"
134 "@aws-sdk/hash-stream-node" "3.25.0"
135 "@aws-sdk/invalid-dependency" "3.25.0"
136 "@aws-sdk/md5-js" "3.25.0"
137 "@aws-sdk/middleware-apply-body-checksum" "3.25.0"
138 "@aws-sdk/middleware-bucket-endpoint" "3.25.0"
139 "@aws-sdk/middleware-content-length" "3.25.0"
140 "@aws-sdk/middleware-expect-continue" "3.25.0"
141 "@aws-sdk/middleware-host-header" "3.25.0"
142 "@aws-sdk/middleware-location-constraint" "3.25.0"
143 "@aws-sdk/middleware-logger" "3.25.0"
144 "@aws-sdk/middleware-retry" "3.25.0"
145 "@aws-sdk/middleware-sdk-s3" "3.25.0"
146 "@aws-sdk/middleware-serde" "3.25.0"
147 "@aws-sdk/middleware-signing" "3.25.0"
148 "@aws-sdk/middleware-ssec" "3.25.0"
149 "@aws-sdk/middleware-stack" "3.25.0"
150 "@aws-sdk/middleware-user-agent" "3.25.0"
151 "@aws-sdk/node-config-provider" "3.25.0"
152 "@aws-sdk/node-http-handler" "3.25.0"
153 "@aws-sdk/protocol-http" "3.25.0"
154 "@aws-sdk/smithy-client" "3.25.0"
155 "@aws-sdk/types" "3.25.0"
156 "@aws-sdk/url-parser" "3.25.0"
157 "@aws-sdk/util-base64-browser" "3.23.0"
158 "@aws-sdk/util-base64-node" "3.23.0"
159 "@aws-sdk/util-body-length-browser" "3.23.0"
160 "@aws-sdk/util-body-length-node" "3.23.0"
161 "@aws-sdk/util-user-agent-browser" "3.25.0"
162 "@aws-sdk/util-user-agent-node" "3.25.0"
163 "@aws-sdk/util-utf8-browser" "3.23.0"
164 "@aws-sdk/util-utf8-node" "3.23.0"
165 "@aws-sdk/util-waiter" "3.25.0"
166 "@aws-sdk/xml-builder" "3.23.0"
167 entities "2.2.0"
168 fast-xml-parser "3.19.0"
169 tslib "^2.3.0"
170
171"@aws-sdk/client-sso@3.25.0":
172 version "3.25.0"
173 resolved "https://registry.yarnpkg.com/@aws-sdk/client-sso/-/client-sso-3.25.0.tgz#9756178afb08e399b5aef5d12dfece3825bc2e26"
174 integrity sha512-b8v4tb7rncnqE5ktBlQEckFdNT+Pk2mBg4e1Uc9C1Z3XmZM+wOWtlbu+KRvgMgDWSx2FzLIjAKe3mLaM4o1Xhg==
175 dependencies:
176 "@aws-crypto/sha256-browser" "^1.0.0"
177 "@aws-crypto/sha256-js" "^1.0.0"
178 "@aws-sdk/config-resolver" "3.25.0"
179 "@aws-sdk/fetch-http-handler" "3.25.0"
180 "@aws-sdk/hash-node" "3.25.0"
181 "@aws-sdk/invalid-dependency" "3.25.0"
182 "@aws-sdk/middleware-content-length" "3.25.0"
183 "@aws-sdk/middleware-host-header" "3.25.0"
184 "@aws-sdk/middleware-logger" "3.25.0"
185 "@aws-sdk/middleware-retry" "3.25.0"
186 "@aws-sdk/middleware-serde" "3.25.0"
187 "@aws-sdk/middleware-stack" "3.25.0"
188 "@aws-sdk/middleware-user-agent" "3.25.0"
189 "@aws-sdk/node-config-provider" "3.25.0"
190 "@aws-sdk/node-http-handler" "3.25.0"
191 "@aws-sdk/protocol-http" "3.25.0"
192 "@aws-sdk/smithy-client" "3.25.0"
193 "@aws-sdk/types" "3.25.0"
194 "@aws-sdk/url-parser" "3.25.0"
195 "@aws-sdk/util-base64-browser" "3.23.0"
196 "@aws-sdk/util-base64-node" "3.23.0"
197 "@aws-sdk/util-body-length-browser" "3.23.0"
198 "@aws-sdk/util-body-length-node" "3.23.0"
199 "@aws-sdk/util-user-agent-browser" "3.25.0"
200 "@aws-sdk/util-user-agent-node" "3.25.0"
201 "@aws-sdk/util-utf8-browser" "3.23.0"
202 "@aws-sdk/util-utf8-node" "3.23.0"
203 tslib "^2.3.0"
204
205"@aws-sdk/client-sts@3.25.0":
206 version "3.25.0"
207 resolved "https://registry.yarnpkg.com/@aws-sdk/client-sts/-/client-sts-3.25.0.tgz#e189c46d560daaa56b872330a5e7d125d00d5a1f"
208 integrity sha512-VQoG4GX+Pf5U/WtUgVgXLF2xC1jK6o4YmOxz09GhPfKT0y26x8hh42jY3zRCys7ldA3VKkfTLCeqMm3UKqXJZg==
209 dependencies:
210 "@aws-crypto/sha256-browser" "^1.0.0"
211 "@aws-crypto/sha256-js" "^1.0.0"
212 "@aws-sdk/config-resolver" "3.25.0"
213 "@aws-sdk/credential-provider-node" "3.25.0"
214 "@aws-sdk/fetch-http-handler" "3.25.0"
215 "@aws-sdk/hash-node" "3.25.0"
216 "@aws-sdk/invalid-dependency" "3.25.0"
217 "@aws-sdk/middleware-content-length" "3.25.0"
218 "@aws-sdk/middleware-host-header" "3.25.0"
219 "@aws-sdk/middleware-logger" "3.25.0"
220 "@aws-sdk/middleware-retry" "3.25.0"
221 "@aws-sdk/middleware-sdk-sts" "3.25.0"
222 "@aws-sdk/middleware-serde" "3.25.0"
223 "@aws-sdk/middleware-signing" "3.25.0"
224 "@aws-sdk/middleware-stack" "3.25.0"
225 "@aws-sdk/middleware-user-agent" "3.25.0"
226 "@aws-sdk/node-config-provider" "3.25.0"
227 "@aws-sdk/node-http-handler" "3.25.0"
228 "@aws-sdk/protocol-http" "3.25.0"
229 "@aws-sdk/smithy-client" "3.25.0"
230 "@aws-sdk/types" "3.25.0"
231 "@aws-sdk/url-parser" "3.25.0"
232 "@aws-sdk/util-base64-browser" "3.23.0"
233 "@aws-sdk/util-base64-node" "3.23.0"
234 "@aws-sdk/util-body-length-browser" "3.23.0"
235 "@aws-sdk/util-body-length-node" "3.23.0"
236 "@aws-sdk/util-user-agent-browser" "3.25.0"
237 "@aws-sdk/util-user-agent-node" "3.25.0"
238 "@aws-sdk/util-utf8-browser" "3.23.0"
239 "@aws-sdk/util-utf8-node" "3.23.0"
240 entities "2.2.0"
241 fast-xml-parser "3.19.0"
242 tslib "^2.3.0"
243
244"@aws-sdk/config-resolver@3.25.0":
245 version "3.25.0"
246 resolved "https://registry.yarnpkg.com/@aws-sdk/config-resolver/-/config-resolver-3.25.0.tgz#d7caba201a00aeb9d60aeddb8901b7e58f7f5a2b"
247 integrity sha512-t5CE90jYkxQyGGxG22atf8040lHuL17wptGp1kN8nSxaG6PudKhxQuHPAGYt6FHgrqqeyFccp/P3jiDSjqUaVw==
248 dependencies:
249 "@aws-sdk/signature-v4" "3.25.0"
250 "@aws-sdk/types" "3.25.0"
251 tslib "^2.3.0"
252
253"@aws-sdk/credential-provider-env@3.25.0":
254 version "3.25.0"
255 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.25.0.tgz#9899ff627f40f09223126d6d2f1153b3ade2e804"
256 integrity sha512-I65/PNGQG+ktt1QSHCWwQ8v7QRK1eRdLkQl3zB5rwBuANbQ3Yu+vA+lAwU+IbpGCOEpHJO3lDN330It5B4Rtvg==
257 dependencies:
258 "@aws-sdk/property-provider" "3.25.0"
259 "@aws-sdk/types" "3.25.0"
260 tslib "^2.3.0"
261
262"@aws-sdk/credential-provider-imds@3.25.0":
263 version "3.25.0"
264 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.25.0.tgz#c40b76bb6a4561fb4c5fd94ce437aac938aaa23f"
265 integrity sha512-BhPM89tjeXsa0KXxz2UTLeAY798Qg1cddFXPZXaJyHQ6eWsrDSoKbSOaeP+rznp037NNLnLX6PB8MOtfu3MAzw==
266 dependencies:
267 "@aws-sdk/property-provider" "3.25.0"
268 "@aws-sdk/types" "3.25.0"
269 tslib "^2.3.0"
270
271"@aws-sdk/credential-provider-ini@3.25.0":
272 version "3.25.0"
273 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.25.0.tgz#32652f30247f84dd49e4c96ecec91577f972f2e3"
274 integrity sha512-p6yvqcZMN+eNZbJXnrFQgLpA06pVA2XagGJdkdDb3q9J4HYoWQduocWUfr3dy0HJdjDZ01BVT/ldBanUyhznQQ==
275 dependencies:
276 "@aws-sdk/credential-provider-env" "3.25.0"
277 "@aws-sdk/credential-provider-imds" "3.25.0"
278 "@aws-sdk/credential-provider-sso" "3.25.0"
279 "@aws-sdk/credential-provider-web-identity" "3.25.0"
280 "@aws-sdk/property-provider" "3.25.0"
281 "@aws-sdk/shared-ini-file-loader" "3.23.0"
282 "@aws-sdk/types" "3.25.0"
283 "@aws-sdk/util-credentials" "3.23.0"
284 tslib "^2.3.0"
285
286"@aws-sdk/credential-provider-node@3.25.0":
287 version "3.25.0"
288 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.25.0.tgz#f8f4c9b8ae51a89f44c11fbbf999e1363424f39e"
289 integrity sha512-GZedy79oSpnDr2I54su3EE1fwpTRFBw/Sn4RBE4VWCM8AWq7ZNk7IKAmbnBrmt+gpFpr9k2PifUIJ7fAcbNvJQ==
290 dependencies:
291 "@aws-sdk/credential-provider-env" "3.25.0"
292 "@aws-sdk/credential-provider-imds" "3.25.0"
293 "@aws-sdk/credential-provider-ini" "3.25.0"
294 "@aws-sdk/credential-provider-process" "3.25.0"
295 "@aws-sdk/credential-provider-sso" "3.25.0"
296 "@aws-sdk/credential-provider-web-identity" "3.25.0"
297 "@aws-sdk/property-provider" "3.25.0"
298 "@aws-sdk/shared-ini-file-loader" "3.23.0"
299 "@aws-sdk/types" "3.25.0"
300 "@aws-sdk/util-credentials" "3.23.0"
301 tslib "^2.3.0"
302
303"@aws-sdk/credential-provider-process@3.25.0":
304 version "3.25.0"
305 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.25.0.tgz#472938d6582152252fb69247531125ed24017d4e"
306 integrity sha512-qMldWWDvvy6Q+HMcTAVWUJP7MLjLXqf0P08Vb5oGYOlyh4TCJDorccRVVsQvutjQggpBaIMTQdzjdamqtZ1y+w==
307 dependencies:
308 "@aws-sdk/property-provider" "3.25.0"
309 "@aws-sdk/shared-ini-file-loader" "3.23.0"
310 "@aws-sdk/types" "3.25.0"
311 "@aws-sdk/util-credentials" "3.23.0"
312 tslib "^2.3.0"
313
314"@aws-sdk/credential-provider-sso@3.25.0":
315 version "3.25.0"
316 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.25.0.tgz#e2065ee6aec63a647acc816732ffcd270eb3c669"
317 integrity sha512-cGP1Zcw2fZHn4CYGgq4soody4x5TrsWk0Pf9F8yCjRMSSZqs3rj0+PrXy4xqkiLCvTSrse6p4e4wMMpaFAm7Tg==
318 dependencies:
319 "@aws-sdk/client-sso" "3.25.0"
320 "@aws-sdk/property-provider" "3.25.0"
321 "@aws-sdk/shared-ini-file-loader" "3.23.0"
322 "@aws-sdk/types" "3.25.0"
323 "@aws-sdk/util-credentials" "3.23.0"
324 tslib "^2.3.0"
325
326"@aws-sdk/credential-provider-web-identity@3.25.0":
327 version "3.25.0"
328 resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.25.0.tgz#9c330322eea3a5f1f0166556c1f18ecc0992b0bf"
329 integrity sha512-6NvOaynsXGuNYbrGzT5h+kkGMaKtAI6zKgPqS/20NKlO5PJc9Eo56Hdbq0gBohXSBzRJE5Jx/1OOrTdvRlwniw==
330 dependencies:
331 "@aws-sdk/property-provider" "3.25.0"
332 "@aws-sdk/types" "3.25.0"
333 tslib "^2.3.0"
334
335"@aws-sdk/eventstream-marshaller@3.25.0":
336 version "3.25.0"
337 resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-marshaller/-/eventstream-marshaller-3.25.0.tgz#8db1f633a638f50d8e37441f01d739238d374549"
338 integrity sha512-gUZIIxupgCIGyspiIV6bEplSRWnhAR9MkyrCJbHhbs4GjWIYlFqp7W0+Y7HY1tIeeXCUf0O8KE3paUMszKPXtg==
339 dependencies:
340 "@aws-crypto/crc32" "^1.0.0"
341 "@aws-sdk/types" "3.25.0"
342 "@aws-sdk/util-hex-encoding" "3.23.0"
343 tslib "^2.3.0"
344
345"@aws-sdk/eventstream-serde-browser@3.25.0":
346 version "3.25.0"
347 resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-browser/-/eventstream-serde-browser-3.25.0.tgz#55481e23acb454d876948fd3b6e609a79977fa7d"
348 integrity sha512-QJF08OIZiufoBPPoVcRwBPvZIpKMSZpISZfpCHcY1GaTpMIzz35N7Nkd10JGpfzpUO9oFcgcmm2q3XHo1XJyyw==
349 dependencies:
350 "@aws-sdk/eventstream-marshaller" "3.25.0"
351 "@aws-sdk/eventstream-serde-universal" "3.25.0"
352 "@aws-sdk/types" "3.25.0"
353 tslib "^2.3.0"
354
355"@aws-sdk/eventstream-serde-config-resolver@3.25.0":
356 version "3.25.0"
357 resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.25.0.tgz#5b8f4ef24fb1bf6c9f0353fb219a68206bad5eb4"
358 integrity sha512-Fb4VS3waKNzc6pK6tQBmWM+JmCNQJYNG/QBfb8y4AoJOZ+I7yX0Qgo90drh8IiUcIKDeprUFjSi/cGIa/KHIsg==
359 dependencies:
360 "@aws-sdk/types" "3.25.0"
361 tslib "^2.3.0"
362
363"@aws-sdk/eventstream-serde-node@3.25.0":
364 version "3.25.0"
365 resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-node/-/eventstream-serde-node-3.25.0.tgz#7ae7fcb8db1e554638f8f1c0fea514cfb07e2524"
366 integrity sha512-gPs+6w0zXf+p0PuOxxmpAlCvP/7E7+8oAar8Ys27exnLXNgqJJK1k5hMBSrfR9GLVti3EhJ1M9x5Seg1SN0/SA==
367 dependencies:
368 "@aws-sdk/eventstream-marshaller" "3.25.0"
369 "@aws-sdk/eventstream-serde-universal" "3.25.0"
370 "@aws-sdk/types" "3.25.0"
371 tslib "^2.3.0"
372
373"@aws-sdk/eventstream-serde-universal@3.25.0":
374 version "3.25.0"
375 resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-serde-universal/-/eventstream-serde-universal-3.25.0.tgz#bf84056fcad894c14f7239272248ea5b3ff39d47"
376 integrity sha512-NgsQk5dXg7NlRDEKGRUdiAx7WESQGD1jEhXitklL3/PHRZ7Y9BJugEFlBvKpU7tiHZBcomTbl/gE2o6i2op/jA==
377 dependencies:
378 "@aws-sdk/eventstream-marshaller" "3.25.0"
379 "@aws-sdk/types" "3.25.0"
380 tslib "^2.3.0"
381
382"@aws-sdk/fetch-http-handler@3.25.0":
383 version "3.25.0"
384 resolved "https://registry.yarnpkg.com/@aws-sdk/fetch-http-handler/-/fetch-http-handler-3.25.0.tgz#0ba013ced267b8ead120be1fcba5bdbbc379b82f"
385 integrity sha512-792kkbfSRBdiFb7Q2cDJts9MKxzAwuQSwUIwRKAOMazU8HkKbKnXXAFSsK3T7VasOFOh7O7YEGN0q9UgEw1q+g==
386 dependencies:
387 "@aws-sdk/protocol-http" "3.25.0"
388 "@aws-sdk/querystring-builder" "3.25.0"
389 "@aws-sdk/types" "3.25.0"
390 "@aws-sdk/util-base64-browser" "3.23.0"
391 tslib "^2.3.0"
392
393"@aws-sdk/hash-blob-browser@3.25.0":
394 version "3.25.0"
395 resolved "https://registry.yarnpkg.com/@aws-sdk/hash-blob-browser/-/hash-blob-browser-3.25.0.tgz#2708daf0f2b53c6670a94276c1048a9a34706108"
396 integrity sha512-dsvV/nkW8v9wIotd3xJn3TQ8AxVLl56H82WkGkHcfw61csRxj3eSUNv0apUBopCcQPK8OK4l2nHAg08r0+LWXg==
397 dependencies:
398 "@aws-sdk/chunked-blob-reader" "3.23.0"
399 "@aws-sdk/chunked-blob-reader-native" "3.23.0"
400 "@aws-sdk/types" "3.25.0"
401 tslib "^2.3.0"
402
403"@aws-sdk/hash-node@3.25.0":
404 version "3.25.0"
405 resolved "https://registry.yarnpkg.com/@aws-sdk/hash-node/-/hash-node-3.25.0.tgz#b149ddf170f4038c7cc3afe8f12e21b0f63e0771"
406 integrity sha512-qRn6iqG9VLt8D29SBABcbauDLn92ssMjtpyVApiOhDYyFm2VA2avomOHD6y2PRBMwM5FMQAygZbpA2HIN2F96w==
407 dependencies:
408 "@aws-sdk/types" "3.25.0"
409 "@aws-sdk/util-buffer-from" "3.23.0"
410 tslib "^2.3.0"
411
412"@aws-sdk/hash-stream-node@3.25.0":
413 version "3.25.0"
414 resolved "https://registry.yarnpkg.com/@aws-sdk/hash-stream-node/-/hash-stream-node-3.25.0.tgz#6fa38cc349a9037367f20ce2601ff0510035dfa2"
415 integrity sha512-pzScUO9pPEEHQ5YQk1sl1bPlU2tt0OCblxUwboZJ9mRgNnWwkMWxe7Mec5IfyMWVUcbIznUHn7qRYEvJQ9JXmw==
416 dependencies:
417 "@aws-sdk/types" "3.25.0"
418 tslib "^2.3.0"
419
420"@aws-sdk/invalid-dependency@3.25.0":
421 version "3.25.0"
422 resolved "https://registry.yarnpkg.com/@aws-sdk/invalid-dependency/-/invalid-dependency-3.25.0.tgz#a75dfb7e86a0e1eb6083b61397dc49a1db041434"
423 integrity sha512-ZBXjBAF2JSiO/wGBa1oaXsd1q5YG3diS8TfIUMXeQoe9O66R5LGoGOQeAbB/JjlwFot6DZfAcfocvl6CtWwqkw==
424 dependencies:
425 "@aws-sdk/types" "3.25.0"
426 tslib "^2.3.0"
427
428"@aws-sdk/is-array-buffer@3.23.0":
429 version "3.23.0"
430 resolved "https://registry.yarnpkg.com/@aws-sdk/is-array-buffer/-/is-array-buffer-3.23.0.tgz#3a5d601b0102ea3a4d832bde647509c8405b2ec9"
431 integrity sha512-XN20/scFthok0lCbjtinW77CoIBoar8cbOzmu+HkYTnBBpJrF6Ai5g9sgglO8r+X+OLn4PrDrTP+BxdpNuIh9g==
432 dependencies:
433 tslib "^2.3.0"
434
435"@aws-sdk/md5-js@3.25.0":
436 version "3.25.0"
437 resolved "https://registry.yarnpkg.com/@aws-sdk/md5-js/-/md5-js-3.25.0.tgz#32cefc43a8c0ee1d85586b95eba0be4912cde534"
438 integrity sha512-97MtL1VF3JCkyJJnwi8LcXpqItnH1VtgoqtVqmaASYp5GXnlsnA1WDnB0754ufPHlssS1aBj/gkLzMZ0Htw/Rg==
439 dependencies:
440 "@aws-sdk/types" "3.25.0"
441 "@aws-sdk/util-utf8-browser" "3.23.0"
442 "@aws-sdk/util-utf8-node" "3.23.0"
443 tslib "^2.3.0"
444
445"@aws-sdk/middleware-apply-body-checksum@3.25.0":
446 version "3.25.0"
447 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-apply-body-checksum/-/middleware-apply-body-checksum-3.25.0.tgz#4263ea8c8e1808e5a4a278fb704ebe7aa891f698"
448 integrity sha512-162qFG7eap4vDKuKrpXWQYE4tbIETNrpTQX6jrPgqostOy1O0Nc5Bn1COIoOMgeMVnkOAZV7qV1J/XAYGz32Yw==
449 dependencies:
450 "@aws-sdk/is-array-buffer" "3.23.0"
451 "@aws-sdk/protocol-http" "3.25.0"
452 "@aws-sdk/types" "3.25.0"
453 tslib "^2.3.0"
454
455"@aws-sdk/middleware-bucket-endpoint@3.25.0":
456 version "3.25.0"
457 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.25.0.tgz#d698230ae552533a1b8ded2c3e6885b4a8374795"
458 integrity sha512-r/6ECFiw/TNjzhAuZzUx3M/1mAtezHTp3e8twB4dDbRRQqABrEZ/dynXi1VxrT2kKW0ZgZNXqEer/NfPOtWB8g==
459 dependencies:
460 "@aws-sdk/protocol-http" "3.25.0"
461 "@aws-sdk/types" "3.25.0"
462 "@aws-sdk/util-arn-parser" "3.23.0"
463 tslib "^2.3.0"
464
465"@aws-sdk/middleware-content-length@3.25.0":
466 version "3.25.0"
467 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-content-length/-/middleware-content-length-3.25.0.tgz#71031d326e52f788396e0ed8216410840059ac53"
468 integrity sha512-uOXus0MmZi/mucRIr5yfwM1vDhYG66CujNfnhyEaq5f4kcDA1Q5qPWSn9dkQPV9JWTZK3WTuYiOPSgtmlAYTAg==
469 dependencies:
470 "@aws-sdk/protocol-http" "3.25.0"
471 "@aws-sdk/types" "3.25.0"
472 tslib "^2.3.0"
473
474"@aws-sdk/middleware-expect-continue@3.25.0":
475 version "3.25.0"
476 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.25.0.tgz#bb41ea9d716c6ce04c4d8fb2cc2dd5fd37f6ccd9"
477 integrity sha512-o3euv8NIO0zlHML81krtfs4TrF5gZwoxBYtY+6tRHXlgutsHe1yfg1wrhWnJNbJg1QhPwXxbMNfYX7MM83D8Ng==
478 dependencies:
479 "@aws-sdk/middleware-header-default" "3.25.0"
480 "@aws-sdk/protocol-http" "3.25.0"
481 "@aws-sdk/types" "3.25.0"
482 tslib "^2.3.0"
483
484"@aws-sdk/middleware-header-default@3.25.0":
485 version "3.25.0"
486 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-header-default/-/middleware-header-default-3.25.0.tgz#17fec9b1941e81059a1374eba58b52230da35a2b"
487 integrity sha512-xkFfZcctPL0VTxmEKITf6/MSDv/8rY+8uA9OMt/YZqfbg0RfeqR2+R1xlDNDxeHeK/v+g5gTNIYTQLM8L2unNA==
488 dependencies:
489 "@aws-sdk/protocol-http" "3.25.0"
490 "@aws-sdk/types" "3.25.0"
491 tslib "^2.3.0"
492
493"@aws-sdk/middleware-host-header@3.25.0":
494 version "3.25.0"
495 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.25.0.tgz#f08dd8c45362cf5cb152c478027092e3d1f4aa58"
496 integrity sha512-xKD/CfsUS3ul2VaQ3IgIUXgA7jU2/Guo/DUhYKrLZTOxm0nuvsIFw0RqSCtRBCLptE5Qi+unkc1LcFDbfqrRbg==
497 dependencies:
498 "@aws-sdk/protocol-http" "3.25.0"
499 "@aws-sdk/types" "3.25.0"
500 tslib "^2.3.0"
501
502"@aws-sdk/middleware-location-constraint@3.25.0":
503 version "3.25.0"
504 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.25.0.tgz#7ba5798aa46cd08c90823f649fcdae0ce5227095"
505 integrity sha512-diwmJ+MRQrq3H9VH+8CNAT4dImf2j3CLewlMrUEY+HsJN9xl2mtU6GQaluQg60iw6FjurLUKKGTTZCul4PGkIQ==
506 dependencies:
507 "@aws-sdk/types" "3.25.0"
508 tslib "^2.3.0"
509
510"@aws-sdk/middleware-logger@3.25.0":
511 version "3.25.0"
512 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.25.0.tgz#03294611be7a2f4aba06e9d80e04318c0991d769"
513 integrity sha512-M1F7BlAsDKoEM8hBaU2pHlLSM40rzzgtZ6jFNhfmTwGcjxe1N7JXCH5QPa7aI8wnJq2RoIRHVfVsUH4GwvOZnA==
514 dependencies:
515 "@aws-sdk/types" "3.25.0"
516 tslib "^2.3.0"
517
518"@aws-sdk/middleware-retry@3.25.0":
519 version "3.25.0"
520 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-retry/-/middleware-retry-3.25.0.tgz#e9f1b011494142aa27ece3ef881e8a3d4866797c"
521 integrity sha512-SzdWPo4ESUR6AXvIf4eC8s5sko2G9Hou6cUIr+BWI4h7whA32j/aWUmvcMHxWT/eaSuPeruXrnvKyLvuM0RjJg==
522 dependencies:
523 "@aws-sdk/protocol-http" "3.25.0"
524 "@aws-sdk/service-error-classification" "3.25.0"
525 "@aws-sdk/types" "3.25.0"
526 tslib "^2.3.0"
527 uuid "^8.3.2"
528
529"@aws-sdk/middleware-sdk-s3@3.25.0":
530 version "3.25.0"
531 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.25.0.tgz#64278bbc97c3a2c26411f155642cc35e8de38887"
532 integrity sha512-Y1P6JnpAdj7p5Q43aSLSuYBCc3hKpZ/mrqFSGN8VFXl7Tzo7tYfjpd9SVRxNGJK7O7tDAUsPNmuGqBrdA2tj8w==
533 dependencies:
534 "@aws-sdk/protocol-http" "3.25.0"
535 "@aws-sdk/types" "3.25.0"
536 "@aws-sdk/util-arn-parser" "3.23.0"
537 tslib "^2.3.0"
538
539"@aws-sdk/middleware-sdk-sts@3.25.0":
540 version "3.25.0"
541 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.25.0.tgz#15d4836958f70187cbb6819a0c0742b751fb44ed"
542 integrity sha512-1SoZZTVejo+32eH0WqXaFvt/NIkVEYWquh3OJpkghMi2oOnMfeIRI0uSoqshL6949f4iSfUvvtuzDpyA7XNCQA==
543 dependencies:
544 "@aws-sdk/middleware-signing" "3.25.0"
545 "@aws-sdk/property-provider" "3.25.0"
546 "@aws-sdk/protocol-http" "3.25.0"
547 "@aws-sdk/signature-v4" "3.25.0"
548 "@aws-sdk/types" "3.25.0"
549 tslib "^2.3.0"
550
551"@aws-sdk/middleware-serde@3.25.0":
552 version "3.25.0"
553 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-serde/-/middleware-serde-3.25.0.tgz#e1284ed4af64b4444cfeb7b5275f489418fa2f58"
554 integrity sha512-065Kugo8yXzBkcVAxctxFCHKlHcINnaQRsJ8ifvgc+UOEgvTG9+LfGWDwfdgarW9CkF7RkCoZOyaqFsO+HJWsg==
555 dependencies:
556 "@aws-sdk/types" "3.25.0"
557 tslib "^2.3.0"
558
559"@aws-sdk/middleware-signing@3.25.0":
560 version "3.25.0"
561 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-signing/-/middleware-signing-3.25.0.tgz#de19f5b27c34161081553a87285f1b5690e2cb9a"
562 integrity sha512-FkhxGMV3UY5HIAwUcarfxdq/CF/tYukdg+bkbTNluMpkcJczqn6shpEIQAGa5FFQP3Lya+STL1NuNXfOP7bG9w==
563 dependencies:
564 "@aws-sdk/property-provider" "3.25.0"
565 "@aws-sdk/protocol-http" "3.25.0"
566 "@aws-sdk/signature-v4" "3.25.0"
567 "@aws-sdk/types" "3.25.0"
568 tslib "^2.3.0"
569
570"@aws-sdk/middleware-ssec@3.25.0":
571 version "3.25.0"
572 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.25.0.tgz#f8cf5bb6fe48d842b1df77f35ccb0f77f1a07b71"
573 integrity sha512-bnrHb8oddW+vDexbNzZtpfshshKru+skcmq3dyXlL8LB/NlJsMiQJE8xoGbq5odTLiflIgaDBt527m5q58i+fg==
574 dependencies:
575 "@aws-sdk/types" "3.25.0"
576 tslib "^2.3.0"
577
578"@aws-sdk/middleware-stack@3.25.0":
579 version "3.25.0"
580 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-stack/-/middleware-stack-3.25.0.tgz#8fc022c90b030c80308bf2930c4a7040052234b4"
581 integrity sha512-s2VgdsasOVKHY3/SIGsw9AeZMMsdcIbBGWim9n5IO3j8C8y54EdRLVCEja8ePvMDZKIzuummwatYPHaUrnqPtQ==
582 dependencies:
583 tslib "^2.3.0"
584
585"@aws-sdk/middleware-user-agent@3.25.0":
586 version "3.25.0"
587 resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.25.0.tgz#2033da6cdcfbf4641b991e3ee3c60ba9809898e7"
588 integrity sha512-HXd/Qknq8Cp7fzJYU7jDDpN7ReJ3arUrnt+dAPNaDDrhmrBbCZp+24UXN6X6DAj0JICRoRuF/l7KxjwdF5FShw==
589 dependencies:
590 "@aws-sdk/protocol-http" "3.25.0"
591 "@aws-sdk/types" "3.25.0"
592 tslib "^2.3.0"
593
594"@aws-sdk/node-config-provider@3.25.0":
595 version "3.25.0"
596 resolved "https://registry.yarnpkg.com/@aws-sdk/node-config-provider/-/node-config-provider-3.25.0.tgz#6ec3e9031b7ff0c51d6e0b33aeff3547ea5619b3"
597 integrity sha512-95FiUDuh1YGo0Giti0Xz9l2TV0Wzw75M1xx0TduFcm1dpLKl+znxTgYh+4G+MOSMHNGy+6K91yxurv4PGYgCWw==
598 dependencies:
599 "@aws-sdk/property-provider" "3.25.0"
600 "@aws-sdk/shared-ini-file-loader" "3.23.0"
601 "@aws-sdk/types" "3.25.0"
602 tslib "^2.3.0"
603
604"@aws-sdk/node-http-handler@3.25.0":
605 version "3.25.0"
606 resolved "https://registry.yarnpkg.com/@aws-sdk/node-http-handler/-/node-http-handler-3.25.0.tgz#b636ea2c39b4a47cf9bffd4cdb6a41c603b99bff"
607 integrity sha512-zVeAM/bXewZiuMtcUZI/xGDID6knkzOv73ueVkzUbP0Ki8bfao7diR3hMbIt5Fy/r8cAVjJce9v6zFqo4sr1WA==
608 dependencies:
609 "@aws-sdk/abort-controller" "3.25.0"
610 "@aws-sdk/protocol-http" "3.25.0"
611 "@aws-sdk/querystring-builder" "3.25.0"
612 "@aws-sdk/types" "3.25.0"
613 tslib "^2.3.0"
614
615"@aws-sdk/property-provider@3.25.0":
616 version "3.25.0"
617 resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.25.0.tgz#2fd7246917b9b6ff448a599163a479bc417a1421"
618 integrity sha512-jUnPDguLWsyGLPfdxGdeaXe3j/CjS3kxBmctvI+soZg57rA2hntP9rm7SUZ2+5rj4mmJaI3bzchiaY3kE3JmpA==
619 dependencies:
620 "@aws-sdk/types" "3.25.0"
621 tslib "^2.3.0"
622
623"@aws-sdk/protocol-http@3.25.0":
624 version "3.25.0"
625 resolved "https://registry.yarnpkg.com/@aws-sdk/protocol-http/-/protocol-http-3.25.0.tgz#4b638cb90672fc2d6cb6d15bebc8bb1fb297da2e"
626 integrity sha512-4Jebt5G8uIFa+HZO7KOgOtA66E/CXysQekiV5dfAsU8ca+rX5PB6qhpWZ2unX/l6He+oDQ0zMoW70JkNiP4/4w==
627 dependencies:
628 "@aws-sdk/types" "3.25.0"
629 tslib "^2.3.0"
630
631"@aws-sdk/querystring-builder@3.25.0":
632 version "3.25.0"
633 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.25.0.tgz#9e6f5eaa5d6805fbf45ae4a47ccbaf823584a4a2"
634 integrity sha512-o/R3/viOxjWckI+kepkxJSL7fIdg1hHYOW/rOpo9HbXS0CJrHVnB8vlBb+Xwl1IFyY2gg+5YZTjiufcgpgRBkw==
635 dependencies:
636 "@aws-sdk/types" "3.25.0"
637 "@aws-sdk/util-uri-escape" "3.23.0"
638 tslib "^2.3.0"
639
640"@aws-sdk/querystring-parser@3.25.0":
641 version "3.25.0"
642 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.25.0.tgz#7fe0a3ddf95a4e5475f53be056fce435fb24b774"
643 integrity sha512-FCNyaOLFLVS5j43MhVA7/VJUDX0t/9RyNTNulHgzFjj6ffsgqcY0uwUq1RO3QCL4asl56zOrLVJgK+Z7wMbvFg==
644 dependencies:
645 "@aws-sdk/types" "3.25.0"
646 tslib "^2.3.0"
647
648"@aws-sdk/service-error-classification@3.25.0":
649 version "3.25.0"
650 resolved "https://registry.yarnpkg.com/@aws-sdk/service-error-classification/-/service-error-classification-3.25.0.tgz#1f24fe74f0a89f00d4f6f2ad1d7bb6b0e2f871e7"
651 integrity sha512-66FfIab87LnnHtOLrGrVOht9Pw6lE8appyOpBdtoeoU5DP7ARSWuDdsYmKdGdRCWvn/RaVFbSYua9k0M1WsGqg==
652
653"@aws-sdk/shared-ini-file-loader@3.23.0":
654 version "3.23.0"
655 resolved "https://registry.yarnpkg.com/@aws-sdk/shared-ini-file-loader/-/shared-ini-file-loader-3.23.0.tgz#574901a31e65e425632a9cae6a64f6382a2b76e8"
656 integrity sha512-YUp46l6E3dLKHp1cKMkZI4slTjsVc/Lm7nPCTVc3oQvZ1MvC99N/jMCmZ7X5YYofuAUSdc9eJ8sYiF2BnUww9g==
657 dependencies:
658 tslib "^2.3.0"
659
660"@aws-sdk/signature-v4@3.25.0":
661 version "3.25.0"
662 resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4/-/signature-v4-3.25.0.tgz#c7fb8184a09593ef6dc62029ca45e252b51247b2"
663 integrity sha512-6KDRRz9XVrj9RxrBLC6dzfnb2TDl3CjIzcNpLdRuKFgzEEdwV+5D+EZuAQU3MuHG5pWTIwG72k/dmCbJ2MDPUQ==
664 dependencies:
665 "@aws-sdk/is-array-buffer" "3.23.0"
666 "@aws-sdk/types" "3.25.0"
667 "@aws-sdk/util-hex-encoding" "3.23.0"
668 "@aws-sdk/util-uri-escape" "3.23.0"
669 tslib "^2.3.0"
670
671"@aws-sdk/smithy-client@3.25.0":
672 version "3.25.0"
673 resolved "https://registry.yarnpkg.com/@aws-sdk/smithy-client/-/smithy-client-3.25.0.tgz#bfdf77f1fa82b26bb7893f16056e8e60e49a140a"
674 integrity sha512-+/iMCNziL5/muaY/gl3xkRsSZyeoVCUSjSbbZjDIXbqDbB9SOz4o3UAIgWHoCgYNfsF25GQR6rThLi61FrSyoQ==
675 dependencies:
676 "@aws-sdk/middleware-stack" "3.25.0"
677 "@aws-sdk/types" "3.25.0"
678 tslib "^2.3.0"
679
680"@aws-sdk/types@3.25.0", "@aws-sdk/types@^3.1.0":
681 version "3.25.0"
682 resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.25.0.tgz#981210272dae2d259130f6dca8429522d9a564bb"
683 integrity sha512-vS0+cTKwj6CujlR07HmeEBxzWPWSrdmZMYnxn/QC9KW9dFu0lsyCGSCqWsFluI6GI0flsnYYWNkP5y4bfD9tqg==
684
685"@aws-sdk/url-parser@3.25.0":
686 version "3.25.0"
687 resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.25.0.tgz#668c7d9d4bc21854c10bfb8bdf762a9206776fae"
688 integrity sha512-qZ3Vq0NjHsE7Qq6R5NVRswIAsiyYjCDnAV+/Vt4jU/K0V3mGumiasiJyRyblW4Da8R6kfcJk0mHSMFRJfoHh8Q==
689 dependencies:
690 "@aws-sdk/querystring-parser" "3.25.0"
691 "@aws-sdk/types" "3.25.0"
692 tslib "^2.3.0"
693
694"@aws-sdk/util-arn-parser@3.23.0":
695 version "3.23.0"
696 resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.23.0.tgz#7372460ba98a6826f97d9622759764bcf09add79"
697 integrity sha512-J3+/wnC21kbb3UAHo7x31aCZxzIa7GBijt6Q7nad/j2aF38EZtE3SI0aZpD8250Vi+9zsZ4672QDUeSZ5BR5kg==
698 dependencies:
699 tslib "^2.3.0"
700
701"@aws-sdk/util-base64-browser@3.23.0":
702 version "3.23.0"
703 resolved "https://registry.yarnpkg.com/@aws-sdk/util-base64-browser/-/util-base64-browser-3.23.0.tgz#61594ac9529756361c81ece287548ab5b8c5a768"
704 integrity sha512-xlI/qw+uhLJWa3k0mRtRHQ42v5QzsMFEUXScredQMfJ/34qzXyocsG6OHPOTV1I8WSANrxnHR5m1Ae3iU6JuVw==
705 dependencies:
706 tslib "^2.3.0"
707
708"@aws-sdk/util-base64-node@3.23.0":
709 version "3.23.0"
710 resolved "https://registry.yarnpkg.com/@aws-sdk/util-base64-node/-/util-base64-node-3.23.0.tgz#d0da9ed6b8aaa7513ba4b36a20b4794c72c074ce"
711 integrity sha512-Kf8JIAUtjrPcD5CJzrig2B5CtegWswUNpW4zBarww/UJhHlp8WzKlCxxA+yNS1ghT0ZMjrRvxPabKDGpkyUfmQ==
712 dependencies:
713 "@aws-sdk/util-buffer-from" "3.23.0"
714 tslib "^2.3.0"
715
716"@aws-sdk/util-body-length-browser@3.23.0":
717 version "3.23.0"
718 resolved "https://registry.yarnpkg.com/@aws-sdk/util-body-length-browser/-/util-body-length-browser-3.23.0.tgz#1a5c5e7ea5e15d93bd178021c54d2ea41faeb1cd"
719 integrity sha512-Bi6u/5omQbOBSB5BxqVvaPgVplLRjhhSuqK3XAukbeBPh7lcibIBdy7YvbhQyl4i8Hb2QjFnqqfzA0lNBe5eiw==
720 dependencies:
721 tslib "^2.3.0"
722
723"@aws-sdk/util-body-length-node@3.23.0":
724 version "3.23.0"
725 resolved "https://registry.yarnpkg.com/@aws-sdk/util-body-length-node/-/util-body-length-node-3.23.0.tgz#2a7890b4fa6de78a042db9537a67f90ccb2a3034"
726 integrity sha512-8kSczloA78mikPaJ742SU9Wpwfcz3HOruoXiP/pOy69UZEsMe4P7zTZI1bo8BAp7j6IFUPCXth9E3UAtkbz+CQ==
727 dependencies:
728 tslib "^2.3.0"
729
730"@aws-sdk/util-buffer-from@3.23.0":
731 version "3.23.0"
732 resolved "https://registry.yarnpkg.com/@aws-sdk/util-buffer-from/-/util-buffer-from-3.23.0.tgz#3bc02f50c6e8a5c2b9db61faeb3bebc9de701c3b"
733 integrity sha512-axXy1FvEOM1uECgMPmyHF1S3Hd7JI+BerhhcAlGig0bbqUsZVQUNL9yhOsWreA+nf1v08Ucj8P2SHPCT9Hvpgg==
734 dependencies:
735 "@aws-sdk/is-array-buffer" "3.23.0"
736 tslib "^2.3.0"
737
738"@aws-sdk/util-credentials@3.23.0":
739 version "3.23.0"
740 resolved "https://registry.yarnpkg.com/@aws-sdk/util-credentials/-/util-credentials-3.23.0.tgz#6b3138c3853c72adc93c3f57e8fb28f58ffdc364"
741 integrity sha512-6TDGZnFa0kZr+vSsWXXMfWt347jbMGKtzGnBxbrmiQgZMijz9s/wLYxsjglZ+CyqI/QrSMOTtqy6mEgJxdnGWQ==
742 dependencies:
743 "@aws-sdk/shared-ini-file-loader" "3.23.0"
744 tslib "^2.3.0"
745
746"@aws-sdk/util-hex-encoding@3.23.0":
747 version "3.23.0"
748 resolved "https://registry.yarnpkg.com/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.23.0.tgz#a8de34faf9e51dd4be379be0e9d3bdc093ae6bf4"
749 integrity sha512-RFDCwNrJMmmPSMVRadxRNePqTXGwtL9s4844x44D0bbGg1TdC42rrg0PRKYkxFL7wd1FbibVQOzciZAvzF+Z+w==
750 dependencies:
751 tslib "^2.3.0"
752
753"@aws-sdk/util-locate-window@^3.0.0":
754 version "3.23.0"
755 resolved "https://registry.yarnpkg.com/@aws-sdk/util-locate-window/-/util-locate-window-3.23.0.tgz#e9bf2a023dce2ea1d13ec2e8c7c92abb333a1442"
756 integrity sha512-mM8kWW7SWIxCshkNllpYqCQi5SzwJ+sv5nURhtquOB5/H3qGqZm0V5lUE3qpE1AYmqKwk6qbGUy1woFn1T5nrw==
757 dependencies:
758 tslib "^2.3.0"
759
760"@aws-sdk/util-uri-escape@3.23.0":
761 version "3.23.0"
762 resolved "https://registry.yarnpkg.com/@aws-sdk/util-uri-escape/-/util-uri-escape-3.23.0.tgz#52539674966eb456d65408d9028ed114e94dfd49"
763 integrity sha512-SvQx2E/FDlI5vLT67wwn/k1j2R/G58tYj4Te6GNgEwPGL43X2+7c0+d/WTgndMaRvxSBHZMUTxBYh1HOeU7loA==
764 dependencies:
765 tslib "^2.3.0"
766
767"@aws-sdk/util-user-agent-browser@3.25.0":
768 version "3.25.0"
769 resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.25.0.tgz#a0f480f1a5b10350370643445b09413102187935"
770 integrity sha512-qGqiWfs49NRmQVXPsBXgMRVkjDZocicU0V2wak98e0t7TOI+KmP8hnwsTkE6c4KwhsFOOUhAzjn5zk3kOwi6tQ==
771 dependencies:
772 "@aws-sdk/types" "3.25.0"
773 bowser "^2.11.0"
774 tslib "^2.3.0"
775
776"@aws-sdk/util-user-agent-node@3.25.0":
777 version "3.25.0"
778 resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.25.0.tgz#db22cb64893c4635adf17086c5cb4a5070c4ac16"
779 integrity sha512-4AWyCNP3n/qxv36OS+WH3l4ooRvwyfdbYWFXNXeGcxMcLANDG0upJQRT1g7H8+/afMaJ6v/BQM/H6tdocJSKjQ==
780 dependencies:
781 "@aws-sdk/node-config-provider" "3.25.0"
782 "@aws-sdk/types" "3.25.0"
783 tslib "^2.3.0"
784
785"@aws-sdk/util-utf8-browser@3.23.0", "@aws-sdk/util-utf8-browser@^3.0.0":
786 version "3.23.0"
787 resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.23.0.tgz#dff7e891c67936de677b7d7a6c796e5c2e1b1510"
788 integrity sha512-fSB95AKnvCnAbCd7o0xLbErfAgD9wnLCaEu23AgfGAiaG3nFF8Z2+wtjebU/9Z4RI9d/x83Ho/yguRnJdkMsPA==
789 dependencies:
790 tslib "^2.3.0"
791
792"@aws-sdk/util-utf8-node@3.23.0":
793 version "3.23.0"
794 resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-node/-/util-utf8-node-3.23.0.tgz#9f9fe76745c79c8a148f15d78e9a5c03d2bf0441"
795 integrity sha512-yao8+8okyfCxRvxZe3GBdO7lJnQEBf3P6rDgleOQD/0DZmMjOQGXCvDd42oagE2TegXhkUnJfVOZU2GqdoR0hg==
796 dependencies:
797 "@aws-sdk/util-buffer-from" "3.23.0"
798 tslib "^2.3.0"
799
800"@aws-sdk/util-waiter@3.25.0":
801 version "3.25.0"
802 resolved "https://registry.yarnpkg.com/@aws-sdk/util-waiter/-/util-waiter-3.25.0.tgz#cd2252c99f335e461134f55c3b7eb89ef6893dca"
803 integrity sha512-rhJ7Q2fcPD8y4H0qNEpaspkSUya0OaNcVrca9wCZKs7jWnropPzrQ+e2MH7fWJ/8jgcBV890+Txr4fWkD4J01g==
804 dependencies:
805 "@aws-sdk/abort-controller" "3.25.0"
806 "@aws-sdk/types" "3.25.0"
807 tslib "^2.3.0"
808
809"@aws-sdk/xml-builder@3.23.0":
810 version "3.23.0"
811 resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.23.0.tgz#e318f539b68fa9c0a36da49e85a96cdca13a8113"
812 integrity sha512-5LEGdhQIJtGTwg4dIYyNtpz5QvPcQoxsqJygmj+VB8KLd+mWorH1IOpiL74z0infeK9N+ZFUUPKIzPJa9xLPqw==
813 dependencies:
814 tslib "^2.3.0"
815
52"@babel/code-frame@7.12.11": 816"@babel/code-frame@7.12.11":
53 version "7.12.11" 817 version "7.12.11"
54 resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" 818 resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f"
@@ -1652,6 +2416,11 @@ boolean@3.0.4:
1652 resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.4.tgz#aa1df8749af41d7211b66b4eee584722ff428c27" 2416 resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.4.tgz#aa1df8749af41d7211b66b4eee584722ff428c27"
1653 integrity sha512-5pyOr+w2LNN72F2mAq6J0ckHUfJYSgRKma7e/wlcMMhgOLV9OI0ERhERYXxUqo+dPyVxcbXKy9n+wg13+LpNnA== 2417 integrity sha512-5pyOr+w2LNN72F2mAq6J0ckHUfJYSgRKma7e/wlcMMhgOLV9OI0ERhERYXxUqo+dPyVxcbXKy9n+wg13+LpNnA==
1654 2418
2419bowser@^2.11.0:
2420 version "2.11.0"
2421 resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f"
2422 integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==
2423
1655boxen@^4.2.0: 2424boxen@^4.2.0:
1656 version "4.2.0" 2425 version "4.2.0"
1657 resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64" 2426 resolved "https://registry.yarnpkg.com/boxen/-/boxen-4.2.0.tgz#e411b62357d6d6d36587c8ac3d5d974daa070e64"
@@ -2909,7 +3678,7 @@ enquirer@^2.3.5:
2909 dependencies: 3678 dependencies:
2910 ansi-colors "^4.1.1" 3679 ansi-colors "^4.1.1"
2911 3680
2912entities@^2.0.0: 3681entities@2.2.0, entities@^2.0.0:
2913 version "2.2.0" 3682 version "2.2.0"
2914 resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" 3683 resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
2915 integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== 3684 integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
@@ -3403,7 +4172,7 @@ fast-safe-stringify@^2.0.4, fast-safe-stringify@^2.0.7:
3403 resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f" 4172 resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz#dc2af48c46cf712b683e849b2bbd446b32de936f"
3404 integrity sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag== 4173 integrity sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag==
3405 4174
3406fast-xml-parser@^3.19.0: 4175fast-xml-parser@3.19.0, fast-xml-parser@^3.19.0:
3407 version "3.19.0" 4176 version "3.19.0"
3408 resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" 4177 resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01"
3409 integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== 4178 integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg==
@@ -7947,7 +8716,7 @@ tsconfig-paths@^3.9.0:
7947 minimist "^1.2.0" 8716 minimist "^1.2.0"
7948 strip-bom "^3.0.0" 8717 strip-bom "^3.0.0"
7949 8718
7950tslib@^1.8.1, tslib@^1.9.0: 8719tslib@^1.11.1, tslib@^1.8.1, tslib@^1.9.0:
7951 version "1.14.1" 8720 version "1.14.1"
7952 resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" 8721 resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
7953 integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== 8722 integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
@@ -7957,6 +8726,11 @@ tslib@^2.0.0, tslib@^2.2.0:
7957 resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" 8726 resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e"
7958 integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== 8727 integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==
7959 8728
8729tslib@^2.3.0:
8730 version "2.3.1"
8731 resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
8732 integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==
8733
7960tsutils@^3.21.0: 8734tsutils@^3.21.0:
7961 version "3.21.0" 8735 version "3.21.0"
7962 resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" 8736 resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"