aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/upload.ts12
-rw-r--r--server/helpers/database-utils.ts31
-rw-r--r--server/helpers/ffmpeg-utils.ts15
-rw-r--r--server/helpers/webtorrent.ts5
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/initializers/migrations/0655-streaming-playlist-filenames.ts66
-rw-r--r--server/lib/activitypub/videos/shared/abstract-builder.ts12
-rw-r--r--server/lib/activitypub/videos/shared/object-to-model-attributes.ts13
-rw-r--r--server/lib/hls.ts35
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts3
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts15
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts3
-rw-r--r--server/lib/live/live-manager.ts26
-rw-r--r--server/lib/live/shared/muxing-session.ts7
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts7
-rw-r--r--server/lib/transcoding/video-transcoding.ts71
-rw-r--r--server/lib/video-paths.ts29
-rw-r--r--server/lib/video.ts4
-rw-r--r--server/models/actor/actor-follow.ts10
-rw-r--r--server/models/redundancy/video-redundancy.ts4
-rw-r--r--server/models/video/formatter/video-format-utils.ts8
-rw-r--r--server/models/video/sql/shared/video-tables.ts3
-rw-r--r--server/models/video/video-file.ts45
-rw-r--r--server/models/video/video-streaming-playlist.ts85
-rw-r--r--server/models/video/video.ts12
-rw-r--r--server/tests/api/live/live-constraints.ts4
-rw-r--r--server/tests/api/live/live-save-replay.ts14
-rw-r--r--server/tests/api/live/live.ts14
-rw-r--r--server/tests/api/users/users-multiple-servers.ts4
-rw-r--r--server/tests/api/videos/resumable-upload.ts7
-rw-r--r--server/tests/api/videos/video-hls.ts11
-rw-r--r--server/tests/cli/optimize-old-videos.ts3
-rw-r--r--server/tests/cli/prune-storage.ts41
-rw-r--r--server/tests/cli/update-host.ts11
-rw-r--r--server/tests/plugins/plugin-transcoding.ts5
-rw-r--r--server/types/models/video/video-streaming-playlist.ts2
36 files changed, 429 insertions, 210 deletions
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
index 7792ae3fc..408f677ff 100644
--- a/server/controllers/api/videos/upload.ts
+++ b/server/controllers/api/videos/upload.ts
@@ -209,10 +209,12 @@ async function addVideo (options: {
209 }) 209 })
210 210
211 createTorrentFederate(video, videoFile) 211 createTorrentFederate(video, videoFile)
212 .then(() => {
213 if (video.state !== VideoState.TO_TRANSCODE) return
212 214
213 if (video.state === VideoState.TO_TRANSCODE) { 215 return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
214 await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) 216 })
215 } 217 .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
216 218
217 Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) 219 Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
218 220
@@ -259,9 +261,9 @@ async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoF
259 return refreshedFile.save() 261 return refreshedFile.save()
260} 262}
261 263
262function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void { 264function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile) {
263 // Create the torrent file in async way because it could be long 265 // Create the torrent file in async way because it could be long
264 createTorrentAndSetInfoHashAsync(video, videoFile) 266 return createTorrentAndSetInfoHashAsync(video, videoFile)
265 .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) })) 267 .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) }))
266 .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)) 268 .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
267 .then(refreshedVideo => { 269 .then(refreshedVideo => {
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
index cbd7aa401..422774022 100644
--- a/server/helpers/database-utils.ts
+++ b/server/helpers/database-utils.ts
@@ -1,6 +1,6 @@
1import * as retry from 'async/retry' 1import * as retry from 'async/retry'
2import * as Bluebird from 'bluebird' 2import * as Bluebird from 'bluebird'
3import { QueryTypes, Transaction } from 'sequelize' 3import { BindOrReplacements, QueryTypes, Transaction } from 'sequelize'
4import { Model } from 'sequelize-typescript' 4import { Model } from 'sequelize-typescript'
5import { sequelizeTypescript } from '@server/initializers/database' 5import { sequelizeTypescript } from '@server/initializers/database'
6import { logger } from './logger' 6import { logger } from './logger'
@@ -84,13 +84,15 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
84 }) 84 })
85} 85}
86 86
87function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> ( 87function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> (
88 fromDatabase: T[], 88 fromDatabase: T[],
89 newModels: T[], 89 newModels: T[]
90 t: Transaction
91) { 90) {
92 return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f))) 91 return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
93 .map(f => f.destroy({ transaction: t })) 92}
93
94function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
95 return Promise.all(models.map(f => f.destroy({ transaction })))
94} 96}
95 97
96// Sequelize always skip the update if we only update updatedAt field 98// Sequelize always skip the update if we only update updatedAt field
@@ -121,13 +123,28 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) {
121 123
122// --------------------------------------------------------------------------- 124// ---------------------------------------------------------------------------
123 125
126function doesExist (query: string, bind?: BindOrReplacements) {
127 const options = {
128 type: QueryTypes.SELECT as QueryTypes.SELECT,
129 bind,
130 raw: true
131 }
132
133 return sequelizeTypescript.query(query, options)
134 .then(results => results.length === 1)
135}
136
137// ---------------------------------------------------------------------------
138
124export { 139export {
125 resetSequelizeInstance, 140 resetSequelizeInstance,
126 retryTransactionWrapper, 141 retryTransactionWrapper,
127 transactionRetryer, 142 transactionRetryer,
128 updateInstanceWithAnother, 143 updateInstanceWithAnother,
129 afterCommitIfTransaction, 144 afterCommitIfTransaction,
130 deleteNonExistingModels, 145 filterNonExistingModels,
146 deleteAllModels,
131 setAsUpdated, 147 setAsUpdated,
132 runInReadCommittedTransaction 148 runInReadCommittedTransaction,
149 doesExist
133} 150}
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 6f5a71b4a..9ad4b7f3b 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -212,14 +212,17 @@ async function transcode (options: TranscodeOptions) {
212 212
213async function getLiveTranscodingCommand (options: { 213async function getLiveTranscodingCommand (options: {
214 rtmpUrl: string 214 rtmpUrl: string
215
215 outPath: string 216 outPath: string
217 masterPlaylistName: string
218
216 resolutions: number[] 219 resolutions: number[]
217 fps: number 220 fps: number
218 221
219 availableEncoders: AvailableEncoders 222 availableEncoders: AvailableEncoders
220 profile: string 223 profile: string
221}) { 224}) {
222 const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options 225 const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile, masterPlaylistName } = options
223 const input = rtmpUrl 226 const input = rtmpUrl
224 227
225 const command = getFFmpeg(input, 'live') 228 const command = getFFmpeg(input, 'live')
@@ -301,14 +304,14 @@ async function getLiveTranscodingCommand (options: {
301 304
302 command.complexFilter(complexFilter) 305 command.complexFilter(complexFilter)
303 306
304 addDefaultLiveHLSParams(command, outPath) 307 addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
305 308
306 command.outputOption('-var_stream_map', varStreamMap.join(' ')) 309 command.outputOption('-var_stream_map', varStreamMap.join(' '))
307 310
308 return command 311 return command
309} 312}
310 313
311function getLiveMuxingCommand (rtmpUrl: string, outPath: string) { 314function getLiveMuxingCommand (rtmpUrl: string, outPath: string, masterPlaylistName: string) {
312 const command = getFFmpeg(rtmpUrl, 'live') 315 const command = getFFmpeg(rtmpUrl, 'live')
313 316
314 command.outputOption('-c:v copy') 317 command.outputOption('-c:v copy')
@@ -316,7 +319,7 @@ function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
316 command.outputOption('-map 0:a?') 319 command.outputOption('-map 0:a?')
317 command.outputOption('-map 0:v?') 320 command.outputOption('-map 0:v?')
318 321
319 addDefaultLiveHLSParams(command, outPath) 322 addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
320 323
321 return command 324 return command
322} 325}
@@ -371,12 +374,12 @@ function addDefaultEncoderParams (options: {
371 } 374 }
372} 375}
373 376
374function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) { 377function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, masterPlaylistName: string) {
375 command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS) 378 command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
376 command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) 379 command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
377 command.outputOption('-hls_flags delete_segments+independent_segments') 380 command.outputOption('-hls_flags delete_segments+independent_segments')
378 command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) 381 command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
379 command.outputOption('-master_pl_name master.m3u8') 382 command.outputOption('-master_pl_name ' + masterPlaylistName)
380 command.outputOption(`-f hls`) 383 command.outputOption(`-f hls`)
381 384
382 command.output(join(outPath, '%v.m3u8')) 385 command.output(join(outPath, '%v.m3u8'))
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index d8220ba9c..ecf63e93e 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -103,6 +103,11 @@ async function createTorrentAndSetInfoHash (
103 103
104 await writeFile(torrentPath, torrent) 104 await writeFile(torrentPath, torrent)
105 105
106 // Remove old torrent file if it existed
107 if (videoFile.hasTorrent()) {
108 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
109 }
110
106 const parsedTorrent = parseTorrent(torrent) 111 const parsedTorrent = parseTorrent(torrent)
107 videoFile.infoHash = parsedTorrent.infoHash 112 videoFile.infoHash = parsedTorrent.infoHash
108 videoFile.torrentFilename = torrentFilename 113 videoFile.torrentFilename = torrentFilename
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index ab59320eb..ee4503b2c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
24 24
25// --------------------------------------------------------------------------- 25// ---------------------------------------------------------------------------
26 26
27const LAST_MIGRATION_VERSION = 650 27const LAST_MIGRATION_VERSION = 655
28 28
29// --------------------------------------------------------------------------- 29// ---------------------------------------------------------------------------
30 30
diff --git a/server/initializers/migrations/0655-streaming-playlist-filenames.ts b/server/initializers/migrations/0655-streaming-playlist-filenames.ts
new file mode 100644
index 000000000..9172a22c4
--- /dev/null
+++ b/server/initializers/migrations/0655-streaming-playlist-filenames.ts
@@ -0,0 +1,66 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction
5 queryInterface: Sequelize.QueryInterface
6 sequelize: Sequelize.Sequelize
7 db: any
8}): Promise<void> {
9 {
10 for (const column of [ 'playlistUrl', 'segmentsSha256Url' ]) {
11 const data = {
12 type: Sequelize.STRING,
13 allowNull: true,
14 defaultValue: null
15 }
16
17 await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
18 }
19 }
20
21 {
22 await utils.sequelize.query(
23 `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = NULL, "segmentsSha256Url" = NULL ` +
24 `WHERE "videoId" IN (SELECT id FROM video WHERE remote IS FALSE)`
25 )
26 }
27
28 {
29 for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
30 const data = {
31 type: Sequelize.STRING,
32 allowNull: true,
33 defaultValue: null
34 }
35
36 await utils.queryInterface.addColumn('videoStreamingPlaylist', column, data)
37 }
38 }
39
40 {
41 await utils.sequelize.query(
42 `UPDATE "videoStreamingPlaylist" SET "playlistFilename" = 'master.m3u8', "segmentsSha256Filename" = 'segments-sha256.json'`
43 )
44 }
45
46 {
47 for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
48 const data = {
49 type: Sequelize.STRING,
50 allowNull: false,
51 defaultValue: null
52 }
53
54 await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
55 }
56 }
57}
58
59function down (options) {
60 throw new Error('Not implemented.')
61}
62
63export {
64 up,
65 down
66}
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts
index e89c94bcd..f995fe637 100644
--- a/server/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/lib/activitypub/videos/shared/abstract-builder.ts
@@ -1,6 +1,6 @@
1import { Transaction } from 'sequelize/types' 1import { Transaction } from 'sequelize/types'
2import { checkUrlsSameHost } from '@server/helpers/activitypub' 2import { checkUrlsSameHost } from '@server/helpers/activitypub'
3import { deleteNonExistingModels } from '@server/helpers/database-utils' 3import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
4import { logger, LoggerTagsFn } from '@server/helpers/logger' 4import { logger, LoggerTagsFn } from '@server/helpers/logger'
5import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' 5import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
6import { setVideoTags } from '@server/lib/video' 6import { setVideoTags } from '@server/lib/video'
@@ -111,8 +111,7 @@ export abstract class APVideoAbstractBuilder {
111 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) 111 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
112 112
113 // Remove video files that do not exist anymore 113 // Remove video files that do not exist anymore
114 const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t) 114 await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t)
115 await Promise.all(destroyTasks)
116 115
117 // Update or add other one 116 // Update or add other one
118 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) 117 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
@@ -124,13 +123,11 @@ export abstract class APVideoAbstractBuilder {
124 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) 123 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
125 124
126 // Remove video playlists that do not exist anymore 125 // Remove video playlists that do not exist anymore
127 const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t) 126 await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)
128 await Promise.all(destroyTasks)
129 127
130 video.VideoStreamingPlaylists = [] 128 video.VideoStreamingPlaylists = []
131 129
132 for (const playlistAttributes of streamingPlaylistAttributes) { 130 for (const playlistAttributes of streamingPlaylistAttributes) {
133
134 const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) 131 const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
135 streamingPlaylistModel.Video = video 132 streamingPlaylistModel.Video = video
136 133
@@ -163,8 +160,7 @@ export abstract class APVideoAbstractBuilder {
163 160
164 const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) 161 const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
165 162
166 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t) 163 await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t)
167 await Promise.all(destroyTasks)
168 164
169 // Update or add other one 165 // Update or add other one
170 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) 166 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index 85548428c..1fa16295d 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -7,10 +7,11 @@ import { logger } from '@server/helpers/logger'
7import { getExtFromMimetype } from '@server/helpers/video' 7import { getExtFromMimetype } from '@server/helpers/video'
8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' 8import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
9import { generateTorrentFileName } from '@server/lib/video-paths' 9import { generateTorrentFileName } from '@server/lib/video-paths'
10import { VideoCaptionModel } from '@server/models/video/video-caption'
10import { VideoFileModel } from '@server/models/video/video-file' 11import { VideoFileModel } from '@server/models/video/video-file'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 12import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { FilteredModelAttributes } from '@server/types' 13import { FilteredModelAttributes } from '@server/types'
13import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models' 14import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
14import { 15import {
15 ActivityHashTagObject, 16 ActivityHashTagObject,
16 ActivityMagnetUrlObject, 17 ActivityMagnetUrlObject,
@@ -23,7 +24,6 @@ import {
23 VideoPrivacy, 24 VideoPrivacy,
24 VideoStreamingPlaylistType 25 VideoStreamingPlaylistType
25} from '@shared/models' 26} from '@shared/models'
26import { VideoCaptionModel } from '@server/models/video/video-caption'
27 27
28function getThumbnailFromIcons (videoObject: VideoObject) { 28function getThumbnailFromIcons (videoObject: VideoObject) {
29 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) 29 let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -80,8 +80,8 @@ function getFileAttributesFromUrl (
80 80
81 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) 81 const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
82 const resolution = fileUrl.height 82 const resolution = fileUrl.height
83 const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id 83 const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id
84 const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null 84 const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null
85 85
86 const attribute = { 86 const attribute = {
87 extname, 87 extname,
@@ -130,8 +130,13 @@ function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject:
130 130
131 const attribute = { 131 const attribute = {
132 type: VideoStreamingPlaylistType.HLS, 132 type: VideoStreamingPlaylistType.HLS,
133
134 playlistFilename: basename(playlistUrlObject.href),
133 playlistUrl: playlistUrlObject.href, 135 playlistUrl: playlistUrlObject.href,
136
137 segmentsSha256Filename: basename(segmentsSha256UrlObject.href),
134 segmentsSha256Url: segmentsSha256UrlObject.href, 138 segmentsSha256Url: segmentsSha256UrlObject.href,
139
135 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), 140 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
136 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, 141 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
137 videoId: video.id, 142 videoId: video.id,
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 212bd095b..32b02bc26 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,7 +1,7 @@
1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' 1import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
2import { flatten, uniq } from 'lodash' 2import { flatten, uniq } from 'lodash'
3import { basename, dirname, join } from 'path' 3import { basename, dirname, join } from 'path'
4import { MVideoWithFile } from '@server/types/models' 4import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
5import { sha256 } from '../helpers/core-utils' 5import { sha256 } from '../helpers/core-utils'
6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' 6import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
7import { logger } from '../helpers/logger' 7import { logger } from '../helpers/logger'
@@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from
12import { sequelizeTypescript } from '../initializers/database' 12import { sequelizeTypescript } from '../initializers/database'
13import { VideoFileModel } from '../models/video/video-file' 13import { VideoFileModel } from '../models/video/video-file'
14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
15import { getVideoFilePath } from './video-paths' 15import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths'
16 16
17async function updateStreamingPlaylistsInfohashesIfNeeded () { 17async function updateStreamingPlaylistsInfohashesIfNeeded () {
18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() 18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@@ -22,27 +22,29 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
22 await sequelizeTypescript.transaction(async t => { 22 await sequelizeTypescript.transaction(async t => {
23 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) 23 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
24 24
25 playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) 25 playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
26 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION 26 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
27
27 await playlist.save({ transaction: t }) 28 await playlist.save({ transaction: t })
28 }) 29 })
29 } 30 }
30} 31}
31 32
32async function updateMasterHLSPlaylist (video: MVideoWithFile) { 33async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
33 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 34 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
35
34 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] 36 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
35 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
36 const streamingPlaylist = video.getHLSPlaylist()
37 37
38 for (const file of streamingPlaylist.VideoFiles) { 38 const masterPlaylistPath = join(directory, playlist.playlistFilename)
39 const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution) 39
40 for (const file of playlist.VideoFiles) {
41 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
40 42
41 // If we did not generated a playlist for this resolution, skip 43 // If we did not generated a playlist for this resolution, skip
42 const filePlaylistPath = join(directory, playlistFilename) 44 const filePlaylistPath = join(directory, playlistFilename)
43 if (await pathExists(filePlaylistPath) === false) continue 45 if (await pathExists(filePlaylistPath) === false) continue
44 46
45 const videoFilePath = getVideoFilePath(streamingPlaylist, file) 47 const videoFilePath = getVideoFilePath(playlist, file)
46 48
47 const size = await getVideoStreamSize(videoFilePath) 49 const size = await getVideoStreamSize(videoFilePath)
48 50
@@ -66,23 +68,22 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) {
66 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') 68 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
67} 69}
68 70
69async function updateSha256VODSegments (video: MVideoWithFile) { 71async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
70 const json: { [filename: string]: { [range: string]: string } } = {} 72 const json: { [filename: string]: { [range: string]: string } } = {}
71 73
72 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 74 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
73 const hlsPlaylist = video.getHLSPlaylist()
74 75
75 // For all the resolutions available for this video 76 // For all the resolutions available for this video
76 for (const file of hlsPlaylist.VideoFiles) { 77 for (const file of playlist.VideoFiles) {
77 const rangeHashes: { [range: string]: string } = {} 78 const rangeHashes: { [range: string]: string } = {}
78 79
79 const videoPath = getVideoFilePath(hlsPlaylist, file) 80 const videoPath = getVideoFilePath(playlist, file)
80 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) 81 const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
81 82
82 // Maybe the playlist is not generated for this resolution yet 83 // Maybe the playlist is not generated for this resolution yet
83 if (!await pathExists(playlistPath)) continue 84 if (!await pathExists(resolutionPlaylistPath)) continue
84 85
85 const playlistContent = await readFile(playlistPath) 86 const playlistContent = await readFile(resolutionPlaylistPath)
86 const ranges = getRangesFromPlaylist(playlistContent.toString()) 87 const ranges = getRangesFromPlaylist(playlistContent.toString())
87 88
88 const fd = await open(videoPath, 'r') 89 const fd = await open(videoPath, 'r')
@@ -98,7 +99,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) {
98 json[videoFilename] = rangeHashes 99 json[videoFilename] = rangeHashes
99 } 100 }
100 101
101 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) 102 const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename)
102 await outputJSON(outputPath, json) 103 await outputJSON(outputPath, json)
103} 104}
104 105
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 1783f206a..4d199f247 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -61,8 +61,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
61 61
62 if (currentVideoFile) { 62 if (currentVideoFile) {
63 // Remove old file and old torrent 63 // Remove old file and old torrent
64 await video.removeFile(currentVideoFile) 64 await video.removeFileAndTorrent(currentVideoFile)
65 await currentVideoFile.removeTorrent()
66 // Remove the old video file from the array 65 // Remove the old video file from the array
67 video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) 66 video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
68 67
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 9eba41bf8..386ccdc7b 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -7,12 +7,12 @@ import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server
7import { generateVideoMiniature } from '@server/lib/thumbnail' 7import { generateVideoMiniature } from '@server/lib/thumbnail'
8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' 8import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
9import { publishAndFederateIfNeeded } from '@server/lib/video' 9import { publishAndFederateIfNeeded } from '@server/lib/video'
10import { getHLSDirectory } from '@server/lib/video-paths' 10import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths'
11import { VideoModel } from '@server/models/video/video' 11import { VideoModel } from '@server/models/video/video'
12import { VideoFileModel } from '@server/models/video/video-file' 12import { VideoFileModel } from '@server/models/video/video-file'
13import { VideoLiveModel } from '@server/models/video/video-live' 13import { VideoLiveModel } from '@server/models/video/video-live'
14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 14import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
15import { MVideo, MVideoLive } from '@server/types/models' 15import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models'
16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 16import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
17import { logger } from '../../../helpers/logger' 17import { logger } from '../../../helpers/logger'
18 18
@@ -43,7 +43,7 @@ async function processVideoLiveEnding (job: Bull.Job) {
43 return cleanupLive(video, streamingPlaylist) 43 return cleanupLive(video, streamingPlaylist)
44 } 44 }
45 45
46 return saveLive(video, live) 46 return saveLive(video, live, streamingPlaylist)
47} 47}
48 48
49// --------------------------------------------------------------------------- 49// ---------------------------------------------------------------------------
@@ -54,14 +54,14 @@ export {
54 54
55// --------------------------------------------------------------------------- 55// ---------------------------------------------------------------------------
56 56
57async function saveLive (video: MVideo, live: MVideoLive) { 57async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
58 const hlsDirectory = getHLSDirectory(video, false) 58 const hlsDirectory = getHLSDirectory(video, false)
59 const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY) 59 const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
60 60
61 const rootFiles = await readdir(hlsDirectory) 61 const rootFiles = await readdir(hlsDirectory)
62 62
63 const playlistFiles = rootFiles.filter(file => { 63 const playlistFiles = rootFiles.filter(file => {
64 return file.endsWith('.m3u8') && file !== 'master.m3u8' 64 return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
65 }) 65 })
66 66
67 await cleanupLiveFiles(hlsDirectory) 67 await cleanupLiveFiles(hlsDirectory)
@@ -80,7 +80,12 @@ async function saveLive (video: MVideo, live: MVideoLive) {
80 80
81 const hlsPlaylist = videoWithFiles.getHLSPlaylist() 81 const hlsPlaylist = videoWithFiles.getHLSPlaylist()
82 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) 82 await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
83
84 // Reset playlist
83 hlsPlaylist.VideoFiles = [] 85 hlsPlaylist.VideoFiles = []
86 hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
87 hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
88 await hlsPlaylist.save()
84 89
85 let durationDone = false 90 let durationDone = false
86 91
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index f5ba6f435..36d9594af 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -125,8 +125,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
125 if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { 125 if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
126 // Remove webtorrent files if not enabled 126 // Remove webtorrent files if not enabled
127 for (const file of video.VideoFiles) { 127 for (const file of video.VideoFiles) {
128 await video.removeFile(file) 128 await video.removeFileAndTorrent(file)
129 await file.removeTorrent()
130 await file.destroy() 129 await file.destroy()
131 } 130 }
132 131
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index da764e009..f106d69fb 100644
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -4,16 +4,17 @@ import { isTestInstance } from '@server/helpers/core-utils'
4import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 4import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
5import { logger, loggerTagsFactory } from '@server/helpers/logger' 5import { logger, loggerTagsFactory } from '@server/helpers/logger'
6import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' 6import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
7import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' 7import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME } from '@server/initializers/constants'
8import { UserModel } from '@server/models/user/user' 8import { UserModel } from '@server/models/user/user'
9import { VideoModel } from '@server/models/video/video' 9import { VideoModel } from '@server/models/video/video'
10import { VideoLiveModel } from '@server/models/video/video-live' 10import { VideoLiveModel } from '@server/models/video/video-live'
11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' 11import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
12import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' 12import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
13import { VideoState, VideoStreamingPlaylistType } from '@shared/models' 13import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
14import { federateVideoIfNeeded } from '../activitypub/videos' 14import { federateVideoIfNeeded } from '../activitypub/videos'
15import { JobQueue } from '../job-queue' 15import { JobQueue } from '../job-queue'
16import { PeerTubeSocket } from '../peertube-socket' 16import { PeerTubeSocket } from '../peertube-socket'
17import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths'
17import { LiveQuotaStore } from './live-quota-store' 18import { LiveQuotaStore } from './live-quota-store'
18import { LiveSegmentShaStore } from './live-segment-sha-store' 19import { LiveSegmentShaStore } from './live-segment-sha-store'
19import { cleanupLive } from './live-utils' 20import { cleanupLive } from './live-utils'
@@ -392,19 +393,18 @@ class LiveManager {
392 return resolutionsEnabled.concat([ originResolution ]) 393 return resolutionsEnabled.concat([ originResolution ])
393 } 394 }
394 395
395 private async createLivePlaylist (video: MVideo, allResolutions: number[]) { 396 private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise<MStreamingPlaylistVideo> {
396 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) 397 const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
397 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
398 videoId: video.id,
399 playlistUrl,
400 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
401 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
402 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
403 398
404 type: VideoStreamingPlaylistType.HLS 399 playlist.playlistFilename = generateHLSMasterPlaylistFilename(true)
405 }, { returning: true }) as [ MStreamingPlaylist, boolean ] 400 playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true)
406 401
407 return Object.assign(videoStreamingPlaylist, { Video: video }) 402 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
403 playlist.type = VideoStreamingPlaylistType.HLS
404
405 playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
406
407 return playlist.save()
408 } 408 }
409 409
410 static get Instance () { 410 static get Instance () {
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts
index 26467f060..709d6c615 100644
--- a/server/lib/live/shared/muxing-session.ts
+++ b/server/lib/live/shared/muxing-session.ts
@@ -112,13 +112,16 @@ class MuxingSession extends EventEmitter {
112 this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED 112 this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
113 ? await getLiveTranscodingCommand({ 113 ? await getLiveTranscodingCommand({
114 rtmpUrl: this.rtmpUrl, 114 rtmpUrl: this.rtmpUrl,
115
115 outPath, 116 outPath,
117 masterPlaylistName: this.streamingPlaylist.playlistFilename,
118
116 resolutions: this.allResolutions, 119 resolutions: this.allResolutions,
117 fps: this.fps, 120 fps: this.fps,
118 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 121 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
119 profile: CONFIG.LIVE.TRANSCODING.PROFILE 122 profile: CONFIG.LIVE.TRANSCODING.PROFILE
120 }) 123 })
121 : getLiveMuxingCommand(this.rtmpUrl, outPath) 124 : getLiveMuxingCommand(this.rtmpUrl, outPath, this.streamingPlaylist.playlistFilename)
122 125
123 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) 126 logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
124 127
@@ -182,7 +185,7 @@ class MuxingSession extends EventEmitter {
182 } 185 }
183 186
184 private watchMasterFile (outPath: string) { 187 private watchMasterFile (outPath: string) {
185 this.masterWatcher = chokidar.watch(outPath + '/master.m3u8') 188 this.masterWatcher = chokidar.watch(outPath + '/' + this.streamingPlaylist.playlistFilename)
186 189
187 this.masterWatcher.on('add', async () => { 190 this.masterWatcher.on('add', async () => {
188 this.emit('master-playlist-created', { videoId: this.videoId }) 191 this.emit('master-playlist-created', { videoId: this.videoId })
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index b5a5eb697..103ab1fab 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -267,7 +267,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
267 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) 267 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
268 268
269 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 269 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
270 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) 270 const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
271 await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
271 272
272 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ 273 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
273 expiresOn, 274 expiresOn,
@@ -282,7 +283,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
282 283
283 await sendCreateCacheFile(serverActor, video, createdModel) 284 await sendCreateCacheFile(serverActor, video, createdModel)
284 285
285 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) 286 logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url)
286 } 287 }
287 288
288 private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { 289 private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
@@ -330,7 +331,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
330 private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { 331 private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) {
331 if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 332 if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
332 333
333 return `${object.VideoStreamingPlaylist.playlistUrl}` 334 return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
334 } 335 }
335 336
336 private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { 337 private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {
diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts
index d70f7f474..d2a556360 100644
--- a/server/lib/transcoding/video-transcoding.ts
+++ b/server/lib/transcoding/video-transcoding.ts
@@ -10,11 +10,18 @@ import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers
10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' 10import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
11import { logger } from '../../helpers/logger' 11import { logger } from '../../helpers/logger'
12import { CONFIG } from '../../initializers/config' 12import { CONFIG } from '../../initializers/config'
13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants' 13import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
14import { VideoFileModel } from '../../models/video/video-file' 14import { VideoFileModel } from '../../models/video/video-file'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' 16import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
17import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths' 17import {
18 generateHLSMasterPlaylistFilename,
19 generateHlsSha256SegmentsFilename,
20 generateHLSVideoFilename,
21 generateWebTorrentVideoFilename,
22 getHlsResolutionPlaylistFilename,
23 getVideoFilePath
24} from '../video-paths'
18import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' 25import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
19 26
20/** 27/**
@@ -272,14 +279,14 @@ async function generateHlsPlaylistCommon (options: {
272 await ensureDir(videoTranscodedBasePath) 279 await ensureDir(videoTranscodedBasePath)
273 280
274 const videoFilename = generateHLSVideoFilename(resolution) 281 const videoFilename = generateHLSVideoFilename(resolution)
275 const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution) 282 const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
276 const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename) 283 const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
277 284
278 const transcodeOptions = { 285 const transcodeOptions = {
279 type, 286 type,
280 287
281 inputPath, 288 inputPath,
282 outputPath: playlistFileTranscodePath, 289 outputPath: resolutionPlaylistFileTranscodePath,
283 290
284 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), 291 availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
285 profile: CONFIG.TRANSCODING.PROFILE, 292 profile: CONFIG.TRANSCODING.PROFILE,
@@ -299,19 +306,23 @@ async function generateHlsPlaylistCommon (options: {
299 306
300 await transcode(transcodeOptions) 307 await transcode(transcodeOptions)
301 308
302 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
303
304 // Create or update the playlist 309 // Create or update the playlist
305 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ 310 const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
306 videoId: video.id, 311
307 playlistUrl, 312 if (!playlist.playlistFilename) {
308 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), 313 playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
309 p2pMediaLoaderInfohashes: [], 314 }
310 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, 315
316 if (!playlist.segmentsSha256Filename) {
317 playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
318 }
319
320 playlist.p2pMediaLoaderInfohashes = []
321 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
311 322
312 type: VideoStreamingPlaylistType.HLS 323 playlist.type = VideoStreamingPlaylistType.HLS
313 }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ] 324
314 videoStreamingPlaylist.Video = video 325 await playlist.save()
315 326
316 // Build the new playlist file 327 // Build the new playlist file
317 const extname = extnameUtil(videoFilename) 328 const extname = extnameUtil(videoFilename)
@@ -321,18 +332,18 @@ async function generateHlsPlaylistCommon (options: {
321 size: 0, 332 size: 0,
322 filename: videoFilename, 333 filename: videoFilename,
323 fps: -1, 334 fps: -1,
324 videoStreamingPlaylistId: videoStreamingPlaylist.id 335 videoStreamingPlaylistId: playlist.id
325 }) 336 })
326 337
327 const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile) 338 const videoFilePath = getVideoFilePath(playlist, newVideoFile)
328 339
329 // Move files from tmp transcoded directory to the appropriate place 340 // Move files from tmp transcoded directory to the appropriate place
330 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 341 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
331 await ensureDir(baseHlsDirectory) 342 await ensureDir(baseHlsDirectory)
332 343
333 // Move playlist file 344 // Move playlist file
334 const playlistPath = join(baseHlsDirectory, playlistFilename) 345 const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename)
335 await move(playlistFileTranscodePath, playlistPath, { overwrite: true }) 346 await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
336 // Move video file 347 // Move video file
337 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) 348 await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
338 349
@@ -342,20 +353,20 @@ async function generateHlsPlaylistCommon (options: {
342 newVideoFile.fps = await getVideoFileFPS(videoFilePath) 353 newVideoFile.fps = await getVideoFileFPS(videoFilePath)
343 newVideoFile.metadata = await getMetadataFromFile(videoFilePath) 354 newVideoFile.metadata = await getMetadataFromFile(videoFilePath)
344 355
345 await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) 356 await createTorrentAndSetInfoHash(playlist, newVideoFile)
346 357
347 await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) 358 await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
348 videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles')
349 359
350 videoStreamingPlaylist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes( 360 const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
351 playlistUrl, videoStreamingPlaylist.VideoFiles 361 playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
352 ) 362 playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
353 await videoStreamingPlaylist.save() 363
364 await playlist.save()
354 365
355 video.setHLSPlaylist(videoStreamingPlaylist) 366 video.setHLSPlaylist(playlist)
356 367
357 await updateMasterHLSPlaylist(video) 368 await updateMasterHLSPlaylist(video, playlistWithFiles)
358 await updateSha256VODSegments(video) 369 await updateSha256VODSegments(video, playlistWithFiles)
359 370
360 return playlistPath 371 return resolutionPlaylistPath
361} 372}
diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts
index b7068190c..1e4382108 100644
--- a/server/lib/video-paths.ts
+++ b/server/lib/video-paths.ts
@@ -4,19 +4,16 @@ import { CONFIG } from '@server/initializers/config'
4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' 4import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
5import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' 5import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
6import { buildUUID } from '@server/helpers/uuid' 6import { buildUUID } from '@server/helpers/uuid'
7import { removeFragmentedMP4Ext } from '@shared/core-utils'
7 8
8// ################## Video file name ################## 9// ################## Video file name ##################
9 10
10function generateWebTorrentVideoFilename (resolution: number, extname: string) { 11function generateWebTorrentVideoFilename (resolution: number, extname: string) {
11 const uuid = buildUUID() 12 return buildUUID() + '-' + resolution + extname
12
13 return uuid + '-' + resolution + extname
14} 13}
15 14
16function generateHLSVideoFilename (resolution: number) { 15function generateHLSVideoFilename (resolution: number) {
17 const uuid = buildUUID() 16 return `${buildUUID()}-${resolution}-fragmented.mp4`
18
19 return `${uuid}-${resolution}-fragmented.mp4`
20} 17}
21 18
22function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { 19function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) {
@@ -54,6 +51,23 @@ function getHLSDirectory (video: MVideoUUID, isRedundancy = false) {
54 return join(baseDir, video.uuid) 51 return join(baseDir, video.uuid)
55} 52}
56 53
54function getHlsResolutionPlaylistFilename (videoFilename: string) {
55 // Video file name already contain resolution
56 return removeFragmentedMP4Ext(videoFilename) + '.m3u8'
57}
58
59function generateHLSMasterPlaylistFilename (isLive = false) {
60 if (isLive) return 'master.m3u8'
61
62 return buildUUID() + '-master.m3u8'
63}
64
65function generateHlsSha256SegmentsFilename (isLive = false) {
66 if (isLive) return 'segments-sha256.json'
67
68 return buildUUID() + '-segments-sha256.json'
69}
70
57// ################## Torrents ################## 71// ################## Torrents ##################
58 72
59function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { 73function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) {
@@ -91,6 +105,9 @@ export {
91 getTorrentFilePath, 105 getTorrentFilePath,
92 106
93 getHLSDirectory, 107 getHLSDirectory,
108 generateHLSMasterPlaylistFilename,
109 generateHlsSha256SegmentsFilename,
110 getHlsResolutionPlaylistFilename,
94 111
95 getLocalVideoFileMetadataUrl, 112 getLocalVideoFileMetadataUrl,
96 113
diff --git a/server/lib/video.ts b/server/lib/video.ts
index daf998704..61fee4949 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database'
5import { TagModel } from '@server/models/video/tag' 5import { TagModel } from '@server/models/video/tag'
6import { VideoModel } from '@server/models/video/video' 6import { VideoModel } from '@server/models/video/video'
7import { FilteredModelAttributes } from '@server/types' 7import { FilteredModelAttributes } from '@server/types'
8import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' 8import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' 9import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
10import { federateVideoIfNeeded } from './activitypub/videos' 10import { federateVideoIfNeeded } from './activitypub/videos'
11import { JobQueue } from './job-queue/job-queue' 11import { JobQueue } from './job-queue/job-queue'
@@ -105,7 +105,7 @@ async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
105 } 105 }
106} 106}
107 107
108async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) { 108async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
109 let dataInput: VideoTranscodingPayload 109 let dataInput: VideoTranscodingPayload
110 110
111 if (videoFile.isAudio()) { 111 if (videoFile.isAudio()) {
diff --git a/server/models/actor/actor-follow.ts b/server/models/actor/actor-follow.ts
index 83c00a22d..3080e02a6 100644
--- a/server/models/actor/actor-follow.ts
+++ b/server/models/actor/actor-follow.ts
@@ -19,8 +19,8 @@ import {
19 UpdatedAt 19 UpdatedAt
20} from 'sequelize-typescript' 20} from 'sequelize-typescript'
21import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' 21import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
22import { doesExist } from '@server/helpers/database-utils'
22import { getServerActor } from '@server/models/application/application' 23import { getServerActor } from '@server/models/application/application'
23import { VideoModel } from '@server/models/video/video'
24import { 24import {
25 MActorFollowActorsDefault, 25 MActorFollowActorsDefault,
26 MActorFollowActorsDefaultSubscription, 26 MActorFollowActorsDefaultSubscription,
@@ -166,14 +166,8 @@ export class ActorFollowModel extends Model<Partial<AttributesOnly<ActorFollowMo
166 166
167 static isFollowedBy (actorId: number, followerActorId: number) { 167 static isFollowedBy (actorId: number, followerActorId: number) {
168 const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1' 168 const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
169 const options = {
170 type: QueryTypes.SELECT as QueryTypes.SELECT,
171 bind: { actorId, followerActorId },
172 raw: true
173 }
174 169
175 return VideoModel.sequelize.query(query, options) 170 return doesExist(query, { actorId, followerActorId })
176 .then(results => results.length === 1)
177 } 171 }
178 172
179 static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise<MActorFollowActorsDefault> { 173 static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise<MActorFollowActorsDefault> {
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index ccda023e0..d645be248 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -160,8 +160,8 @@ export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedu
160 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` 160 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
161 logger.info('Removing duplicated video file %s.', logIdentifier) 161 logger.info('Removing duplicated video file %s.', logIdentifier)
162 162
163 videoFile.Video.removeFile(videoFile, true) 163 videoFile.Video.removeFileAndTorrent(videoFile, true)
164 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) 164 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
165 } 165 }
166 166
167 if (instance.videoStreamingPlaylistId) { 167 if (instance.videoStreamingPlaylistId) {
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts
index 6b1e59063..3310b3b46 100644
--- a/server/models/video/formatter/video-format-utils.ts
+++ b/server/models/video/formatter/video-format-utils.ts
@@ -182,8 +182,8 @@ function streamingPlaylistsModelToFormattedJSON (
182 return { 182 return {
183 id: playlist.id, 183 id: playlist.id,
184 type: playlist.type, 184 type: playlist.type,
185 playlistUrl: playlist.playlistUrl, 185 playlistUrl: playlist.getMasterPlaylistUrl(video),
186 segmentsSha256Url: playlist.segmentsSha256Url, 186 segmentsSha256Url: playlist.getSha256SegmentsUrl(video),
187 redundancies, 187 redundancies,
188 files 188 files
189 } 189 }
@@ -331,7 +331,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
331 type: 'Link', 331 type: 'Link',
332 name: 'sha256', 332 name: 'sha256',
333 mediaType: 'application/json' as 'application/json', 333 mediaType: 'application/json' as 'application/json',
334 href: playlist.segmentsSha256Url 334 href: playlist.getSha256SegmentsUrl(video)
335 }) 335 })
336 336
337 addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) 337 addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
@@ -339,7 +339,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
339 url.push({ 339 url.push({
340 type: 'Link', 340 type: 'Link',
341 mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', 341 mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
342 href: playlist.playlistUrl, 342 href: playlist.getMasterPlaylistUrl(video),
343 tag 343 tag
344 }) 344 })
345 } 345 }
diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts
index abdd22188..742d19099 100644
--- a/server/models/video/sql/shared/video-tables.ts
+++ b/server/models/video/sql/shared/video-tables.ts
@@ -92,12 +92,13 @@ export class VideoTables {
92 } 92 }
93 93
94 getStreamingPlaylistAttributes () { 94 getStreamingPlaylistAttributes () {
95 let playlistKeys = [ 'id', 'playlistUrl', 'type' ] 95 let playlistKeys = [ 'id', 'playlistUrl', 'playlistFilename', 'type' ]
96 96
97 if (this.mode === 'get') { 97 if (this.mode === 'get') {
98 playlistKeys = playlistKeys.concat([ 98 playlistKeys = playlistKeys.concat([
99 'p2pMediaLoaderInfohashes', 99 'p2pMediaLoaderInfohashes',
100 'p2pMediaLoaderPeerVersion', 100 'p2pMediaLoaderPeerVersion',
101 'segmentsSha256Filename',
101 'segmentsSha256Url', 102 'segmentsSha256Url',
102 'videoId', 103 'videoId',
103 'createdAt', 104 'createdAt',
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 22cf63804..797a85a4e 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -1,7 +1,7 @@
1import { remove } from 'fs-extra' 1import { remove } from 'fs-extra'
2import * as memoizee from 'memoizee' 2import * as memoizee from 'memoizee'
3import { join } from 'path' 3import { join } from 'path'
4import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' 4import { FindOptions, Op, Transaction } from 'sequelize'
5import { 5import {
6 AllowNull, 6 AllowNull,
7 BelongsTo, 7 BelongsTo,
@@ -21,6 +21,7 @@ import {
21import { Where } from 'sequelize/types/lib/utils' 21import { Where } from 'sequelize/types/lib/utils'
22import validator from 'validator' 22import validator from 'validator'
23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' 23import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
24import { doesExist } from '@server/helpers/database-utils'
24import { logger } from '@server/helpers/logger' 25import { logger } from '@server/helpers/logger'
25import { extractVideo } from '@server/helpers/video' 26import { extractVideo } from '@server/helpers/video'
26import { getTorrentFilePath } from '@server/lib/video-paths' 27import { getTorrentFilePath } from '@server/lib/video-paths'
@@ -250,14 +251,8 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
250 251
251 static doesInfohashExist (infoHash: string) { 252 static doesInfohashExist (infoHash: string) {
252 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' 253 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
253 const options = {
254 type: QueryTypes.SELECT as QueryTypes.SELECT,
255 bind: { infoHash },
256 raw: true
257 }
258 254
259 return VideoModel.sequelize.query(query, options) 255 return doesExist(query, { infoHash })
260 .then(results => results.length === 1)
261 } 256 }
262 257
263 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { 258 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
@@ -266,6 +261,33 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
266 return !!videoFile 261 return !!videoFile
267 } 262 }
268 263
264 static async doesOwnedTorrentFileExist (filename: string) {
265 const query = 'SELECT 1 FROM "videoFile" ' +
266 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
267 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
268 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
269 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
270
271 return doesExist(query, { filename })
272 }
273
274 static async doesOwnedWebTorrentVideoFileExist (filename: string) {
275 const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
276 'WHERE "filename" = $filename LIMIT 1'
277
278 return doesExist(query, { filename })
279 }
280
281 static loadByFilename (filename: string) {
282 const query = {
283 where: {
284 filename
285 }
286 }
287
288 return VideoFileModel.findOne(query)
289 }
290
269 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { 291 static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
270 const query = { 292 const query = {
271 where: { 293 where: {
@@ -443,10 +465,9 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
443 } 465 }
444 466
445 getFileDownloadUrl (video: MVideoWithHost) { 467 getFileDownloadUrl (video: MVideoWithHost) {
446 const basePath = this.isHLS() 468 const path = this.isHLS()
447 ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS 469 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
448 : STATIC_DOWNLOAD_PATHS.VIDEOS 470 : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
449 const path = join(basePath, this.filename)
450 471
451 if (video.isOwned()) return WEBSERVER.URL + path 472 if (video.isOwned()) return WEBSERVER.URL + path
452 473
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index d627e8c9d..b15d20cf9 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -1,19 +1,27 @@
1import * as memoizee from 'memoizee' 1import * as memoizee from 'memoizee'
2import { join } from 'path' 2import { join } from 'path'
3import { Op, QueryTypes } from 'sequelize' 3import { Op } from 'sequelize'
4import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' 4import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
5import { doesExist } from '@server/helpers/database-utils'
5import { VideoFileModel } from '@server/models/video/video-file' 6import { VideoFileModel } from '@server/models/video/video-file'
6import { MStreamingPlaylist } from '@server/types/models' 7import { MStreamingPlaylist, MVideo } from '@server/types/models'
8import { AttributesOnly } from '@shared/core-utils'
7import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 9import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
8import { sha1 } from '../../helpers/core-utils' 10import { sha1 } from '../../helpers/core-utils'
9import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 11import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
10import { isArrayOf } from '../../helpers/custom-validators/misc' 12import { isArrayOf } from '../../helpers/custom-validators/misc'
11import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 13import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
12import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants' 14import {
15 CONSTRAINTS_FIELDS,
16 MEMOIZE_LENGTH,
17 MEMOIZE_TTL,
18 P2P_MEDIA_LOADER_PEER_VERSION,
19 STATIC_PATHS,
20 WEBSERVER
21} from '../../initializers/constants'
13import { VideoRedundancyModel } from '../redundancy/video-redundancy' 22import { VideoRedundancyModel } from '../redundancy/video-redundancy'
14import { throwIfNotValid } from '../utils' 23import { throwIfNotValid } from '../utils'
15import { VideoModel } from './video' 24import { VideoModel } from './video'
16import { AttributesOnly } from '@shared/core-utils'
17 25
18@Table({ 26@Table({
19 tableName: 'videoStreamingPlaylist', 27 tableName: 'videoStreamingPlaylist',
@@ -43,7 +51,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
43 type: VideoStreamingPlaylistType 51 type: VideoStreamingPlaylistType
44 52
45 @AllowNull(false) 53 @AllowNull(false)
46 @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url')) 54 @Column
55 playlistFilename: string
56
57 @AllowNull(true)
58 @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true))
47 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) 59 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
48 playlistUrl: string 60 playlistUrl: string
49 61
@@ -57,7 +69,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
57 p2pMediaLoaderPeerVersion: number 69 p2pMediaLoaderPeerVersion: number
58 70
59 @AllowNull(false) 71 @AllowNull(false)
60 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) 72 @Column
73 segmentsSha256Filename: string
74
75 @AllowNull(true)
76 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true))
61 @Column 77 @Column
62 segmentsSha256Url: string 78 segmentsSha256Url: string
63 79
@@ -98,14 +114,8 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
98 114
99 static doesInfohashExist (infoHash: string) { 115 static doesInfohashExist (infoHash: string) {
100 const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' 116 const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
101 const options = {
102 type: QueryTypes.SELECT as QueryTypes.SELECT,
103 bind: { infoHash },
104 raw: true
105 }
106 117
107 return VideoModel.sequelize.query<object>(query, options) 118 return doesExist(query, { infoHash })
108 .then(results => results.length === 1)
109 } 119 }
110 120
111 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) { 121 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) {
@@ -125,7 +135,13 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
125 p2pMediaLoaderPeerVersion: { 135 p2pMediaLoaderPeerVersion: {
126 [Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION 136 [Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
127 } 137 }
128 } 138 },
139 include: [
140 {
141 model: VideoModel.unscoped(),
142 required: true
143 }
144 ]
129 } 145 }
130 146
131 return VideoStreamingPlaylistModel.findAll(query) 147 return VideoStreamingPlaylistModel.findAll(query)
@@ -144,7 +160,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
144 return VideoStreamingPlaylistModel.findByPk(id, options) 160 return VideoStreamingPlaylistModel.findByPk(id, options)
145 } 161 }
146 162
147 static loadHLSPlaylistByVideo (videoId: number) { 163 static loadHLSPlaylistByVideo (videoId: number): Promise<MStreamingPlaylist> {
148 const options = { 164 const options = {
149 where: { 165 where: {
150 type: VideoStreamingPlaylistType.HLS, 166 type: VideoStreamingPlaylistType.HLS,
@@ -155,30 +171,29 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
155 return VideoStreamingPlaylistModel.findOne(options) 171 return VideoStreamingPlaylistModel.findOne(options)
156 } 172 }
157 173
158 static getHlsPlaylistFilename (resolution: number) { 174 static async loadOrGenerate (video: MVideo) {
159 return resolution + '.m3u8' 175 let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
160 } 176 if (!playlist) playlist = new VideoStreamingPlaylistModel()
161 177
162 static getMasterHlsPlaylistFilename () { 178 return Object.assign(playlist, { videoId: video.id, Video: video })
163 return 'master.m3u8'
164 } 179 }
165 180
166 static getHlsSha256SegmentsFilename () { 181 assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
167 return 'segments-sha256.json' 182 const masterPlaylistUrl = this.getMasterPlaylistUrl(video)
168 }
169 183
170 static getHlsMasterPlaylistStaticPath (videoUUID: string) { 184 this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
171 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
172 } 185 }
173 186
174 static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) { 187 getMasterPlaylistUrl (video: MVideo) {
175 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) 188 if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
189
190 return this.playlistUrl
176 } 191 }
177 192
178 static getHlsSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) { 193 getSha256SegmentsUrl (video: MVideo) {
179 if (isLive) return join('/live', 'segments-sha256', videoUUID) 194 if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
180 195
181 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) 196 return this.segmentsSha256Url
182 } 197 }
183 198
184 getStringType () { 199 getStringType () {
@@ -195,4 +210,14 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
195 return this.type === other.type && 210 return this.type === other.type &&
196 this.videoId === other.videoId 211 this.videoId === other.videoId
197 } 212 }
213
214 private getMasterPlaylistStaticPath (videoUUID: string) {
215 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
216 }
217
218 private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
219 if (isLive) return join('/live', 'segments-sha256', videoUUID)
220
221 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
222 }
198} 223}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 1e5648a36..0f0f894e4 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -762,8 +762,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
762 762
763 // Remove physical files and torrents 763 // Remove physical files and torrents
764 instance.VideoFiles.forEach(file => { 764 instance.VideoFiles.forEach(file => {
765 tasks.push(instance.removeFile(file)) 765 tasks.push(instance.removeFileAndTorrent(file))
766 tasks.push(file.removeTorrent())
767 }) 766 })
768 767
769 // Remove playlists file 768 // Remove playlists file
@@ -1670,10 +1669,13 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1670 .concat(toAdd) 1669 .concat(toAdd)
1671 } 1670 }
1672 1671
1673 removeFile (videoFile: MVideoFile, isRedundancy = false) { 1672 removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
1674 const filePath = getVideoFilePath(this, videoFile, isRedundancy) 1673 const filePath = getVideoFilePath(this, videoFile, isRedundancy)
1675 return remove(filePath) 1674
1676 .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) 1675 const promises: Promise<any>[] = [ remove(filePath) ]
1676 if (!isRedundancy) promises.push(videoFile.removeTorrent())
1677
1678 return Promise.all(promises)
1677 } 1679 }
1678 1680
1679 async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { 1681 async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts
index 20346113d..4acde3cc5 100644
--- a/server/tests/api/live/live-constraints.ts
+++ b/server/tests/api/live/live-constraints.ts
@@ -4,7 +4,7 @@ import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { VideoPrivacy } from '@shared/models' 5import { VideoPrivacy } from '@shared/models'
6import { 6import {
7 checkLiveCleanup, 7 checkLiveCleanupAfterSave,
8 cleanupTests, 8 cleanupTests,
9 ConfigCommand, 9 ConfigCommand,
10 createMultipleServers, 10 createMultipleServers,
@@ -43,7 +43,7 @@ describe('Test live constraints', function () {
43 expect(video.duration).to.be.greaterThan(0) 43 expect(video.duration).to.be.greaterThan(0)
44 } 44 }
45 45
46 await checkLiveCleanup(servers[0], videoId, resolutions) 46 await checkLiveCleanupAfterSave(servers[0], videoId, resolutions)
47 } 47 }
48 48
49 async function waitUntilLivePublishedOnAllServers (videoId: string) { 49 async function waitUntilLivePublishedOnAllServers (videoId: string) {
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index bd15396ec..8f1fb78a5 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -4,7 +4,7 @@ import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { FfmpegCommand } from 'fluent-ffmpeg' 5import { FfmpegCommand } from 'fluent-ffmpeg'
6import { 6import {
7 checkLiveCleanup, 7 checkLiveCleanupAfterSave,
8 cleanupTests, 8 cleanupTests,
9 ConfigCommand, 9 ConfigCommand,
10 createMultipleServers, 10 createMultipleServers,
@@ -150,7 +150,7 @@ describe('Save replay setting', function () {
150 await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) 150 await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
151 151
152 // No resolutions saved since we did not save replay 152 // No resolutions saved since we did not save replay
153 await checkLiveCleanup(servers[0], liveVideoUUID, []) 153 await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
154 }) 154 })
155 155
156 it('Should correctly terminate the stream on blacklist and delete the live', async function () { 156 it('Should correctly terminate the stream on blacklist and delete the live', async function () {
@@ -179,7 +179,7 @@ describe('Save replay setting', function () {
179 179
180 await wait(5000) 180 await wait(5000)
181 await waitJobs(servers) 181 await waitJobs(servers)
182 await checkLiveCleanup(servers[0], liveVideoUUID, []) 182 await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
183 }) 183 })
184 184
185 it('Should correctly terminate the stream on delete and delete the video', async function () { 185 it('Should correctly terminate the stream on delete and delete the video', async function () {
@@ -203,7 +203,7 @@ describe('Save replay setting', function () {
203 await waitJobs(servers) 203 await waitJobs(servers)
204 204
205 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 205 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
206 await checkLiveCleanup(servers[0], liveVideoUUID, []) 206 await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
207 }) 207 })
208 }) 208 })
209 209
@@ -259,7 +259,7 @@ describe('Save replay setting', function () {
259 }) 259 })
260 260
261 it('Should have cleaned up the live files', async function () { 261 it('Should have cleaned up the live files', async function () {
262 await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) 262 await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
263 }) 263 })
264 264
265 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { 265 it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@@ -287,7 +287,7 @@ describe('Save replay setting', function () {
287 287
288 await wait(5000) 288 await wait(5000)
289 await waitJobs(servers) 289 await waitJobs(servers)
290 await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) 290 await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
291 }) 291 })
292 292
293 it('Should correctly terminate the stream on delete and delete the video', async function () { 293 it('Should correctly terminate the stream on delete and delete the video', async function () {
@@ -310,7 +310,7 @@ describe('Save replay setting', function () {
310 await waitJobs(servers) 310 await waitJobs(servers)
311 311
312 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) 312 await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
313 await checkLiveCleanup(servers[0], liveVideoUUID, []) 313 await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
314 }) 314 })
315 }) 315 })
316 316
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index 4676a840a..d555cff19 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -2,10 +2,10 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { join } from 'path' 5import { basename, join } from 'path'
6import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' 6import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
7import { 7import {
8 checkLiveCleanup, 8 checkLiveCleanupAfterSave,
9 checkLiveSegmentHash, 9 checkLiveSegmentHash,
10 checkResolutionsInMasterPlaylist, 10 checkResolutionsInMasterPlaylist,
11 cleanupTests, 11 cleanupTests,
@@ -506,6 +506,10 @@ describe('Test live', function () {
506 await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) 506 await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
507 await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) 507 await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
508 508
509 // We should have generated random filenames
510 expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
511 expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
512
509 expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length) 513 expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
510 514
511 for (const resolution of resolutions) { 515 for (const resolution of resolutions) {
@@ -520,7 +524,9 @@ describe('Test live', function () {
520 expect(file.fps).to.be.approximately(30, 2) 524 expect(file.fps).to.be.approximately(30, 2)
521 } 525 }
522 526
523 const filename = `${video.uuid}-${resolution}-fragmented.mp4` 527 const filename = basename(file.fileUrl)
528 expect(filename).to.not.contain(video.uuid)
529
524 const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename)) 530 const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
525 531
526 const probe = await ffprobePromise(segmentPath) 532 const probe = await ffprobePromise(segmentPath)
@@ -537,7 +543,7 @@ describe('Test live', function () {
537 it('Should correctly have cleaned up the live files', async function () { 543 it('Should correctly have cleaned up the live files', async function () {
538 this.timeout(30000) 544 this.timeout(30000)
539 545
540 await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ]) 546 await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
541 }) 547 })
542 }) 548 })
543 549
diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts
index 16372b039..d0ca82b07 100644
--- a/server/tests/api/users/users-multiple-servers.ts
+++ b/server/tests/api/users/users-multiple-servers.ts
@@ -58,10 +58,10 @@ describe('Test users with multiple servers', function () {
58 const { uuid } = await servers[0].videos.upload({ token: userAccessToken }) 58 const { uuid } = await servers[0].videos.upload({ token: userAccessToken })
59 videoUUID = uuid 59 videoUUID = uuid
60 60
61 await waitJobs(servers)
62
61 await saveVideoInServers(servers, videoUUID) 63 await saveVideoInServers(servers, videoUUID)
62 } 64 }
63
64 await waitJobs(servers)
65 }) 65 })
66 66
67 it('Should be able to update my display name', async function () { 67 it('Should be able to update my display name', async function () {
diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts
index c94d92cf2..857859fd3 100644
--- a/server/tests/api/videos/resumable-upload.ts
+++ b/server/tests/api/videos/resumable-upload.ts
@@ -170,8 +170,13 @@ describe('Test resumable upload', function () {
170 170
171 const size = 1000 171 const size = 1000
172 172
173 // Content length check seems to have changed in v16
174 const expectedStatus = process.version.startsWith('v16')
175 ? HttpStatusCode.CONFLICT_409
176 : HttpStatusCode.BAD_REQUEST_400
177
173 const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}` 178 const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}`
174 await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size }) 179 await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size })
175 await checkFileSize(uploadId, 0) 180 await checkFileSize(uploadId, 0)
176 }) 181 })
177 }) 182 })
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
index 921d7ce64..961f0e617 100644
--- a/server/tests/api/videos/video-hls.ts
+++ b/server/tests/api/videos/video-hls.ts
@@ -2,7 +2,8 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { join } from 'path' 5import { basename, join } from 'path'
6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
6import { 7import {
7 checkDirectoryIsEmpty, 8 checkDirectoryIsEmpty,
8 checkResolutionsInMasterPlaylist, 9 checkResolutionsInMasterPlaylist,
@@ -19,8 +20,6 @@ import {
19} from '@shared/extra-utils' 20} from '@shared/extra-utils'
20import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' 21import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
21import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' 22import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
22import { uuidRegex } from '@shared/core-utils'
23import { basename } from 'path/posix'
24 23
25const expect = chai.expect 24const expect = chai.expect
26 25
@@ -78,11 +77,13 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
78 // Check resolution playlists 77 // Check resolution playlists
79 { 78 {
80 for (const resolution of resolutions) { 79 for (const resolution of resolutions) {
80 const file = hlsFiles.find(f => f.resolution.id === resolution)
81 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
82
81 const subPlaylist = await server.streamingPlaylists.get({ 83 const subPlaylist = await server.streamingPlaylists.get({
82 url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8` 84 url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
83 }) 85 })
84 86
85 const file = hlsFiles.find(f => f.resolution.id === resolution)
86 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) 87 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
87 expect(subPlaylist).to.contain(basename(file.fileUrl)) 88 expect(subPlaylist).to.contain(basename(file.fileUrl))
88 } 89 }
diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts
index 685b3b7b8..579b2e7d8 100644
--- a/server/tests/cli/optimize-old-videos.ts
+++ b/server/tests/cli/optimize-old-videos.ts
@@ -2,7 +2,6 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { join } from 'path'
6import { 5import {
7 cleanupTests, 6 cleanupTests,
8 createMultipleServers, 7 createMultipleServers,
@@ -86,7 +85,7 @@ describe('Test optimize old videos', function () {
86 85
87 expect(file.size).to.be.below(8000000) 86 expect(file.size).to.be.below(8000000)
88 87
89 const path = servers[0].servers.buildDirectory(join('videos', video.uuid + '-' + file.resolution.id + '.mp4')) 88 const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl)
90 const bitrate = await getVideoFileBitrate(path) 89 const bitrate = await getVideoFileBitrate(path)
91 const fps = await getVideoFileFPS(path) 90 const fps = await getVideoFileFPS(path)
92 const resolution = await getVideoFileResolution(path) 91 const resolution = await getVideoFileResolution(path)
diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts
index 954a87833..2d4c02da7 100644
--- a/server/tests/cli/prune-storage.ts
+++ b/server/tests/cli/prune-storage.ts
@@ -36,7 +36,7 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
36 } 36 }
37} 37}
38 38
39async function assertCountAreOkay (servers: PeerTubeServer[]) { 39async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) {
40 for (const server of servers) { 40 for (const server of servers) {
41 const videosCount = await countFiles(server, 'videos') 41 const videosCount = await countFiles(server, 'videos')
42 expect(videosCount).to.equal(8) 42 expect(videosCount).to.equal(8)
@@ -53,12 +53,21 @@ async function assertCountAreOkay (servers: PeerTubeServer[]) {
53 const avatarsCount = await countFiles(server, 'avatars') 53 const avatarsCount = await countFiles(server, 'avatars')
54 expect(avatarsCount).to.equal(2) 54 expect(avatarsCount).to.equal(2)
55 } 55 }
56
57 // When we'll prune HLS directories too
58 // const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/')
59 // expect(hlsRootCount).to.equal(2)
60
61 // const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID)
62 // expect(hlsCount).to.equal(10)
56} 63}
57 64
58describe('Test prune storage scripts', function () { 65describe('Test prune storage scripts', function () {
59 let servers: PeerTubeServer[] 66 let servers: PeerTubeServer[]
60 const badNames: { [directory: string]: string[] } = {} 67 const badNames: { [directory: string]: string[] } = {}
61 68
69 let videoServer2UUID: string
70
62 before(async function () { 71 before(async function () {
63 this.timeout(120000) 72 this.timeout(120000)
64 73
@@ -68,7 +77,9 @@ describe('Test prune storage scripts', function () {
68 77
69 for (const server of servers) { 78 for (const server of servers) {
70 await server.videos.upload({ attributes: { name: 'video 1' } }) 79 await server.videos.upload({ attributes: { name: 'video 1' } })
71 await server.videos.upload({ attributes: { name: 'video 2' } }) 80
81 const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
82 if (server.serverNumber === 2) videoServer2UUID = uuid
72 83
73 await server.users.updateMyAvatar({ fixture: 'avatar.png' }) 84 await server.users.updateMyAvatar({ fixture: 'avatar.png' })
74 85
@@ -112,7 +123,7 @@ describe('Test prune storage scripts', function () {
112 }) 123 })
113 124
114 it('Should have the files on the disk', async function () { 125 it('Should have the files on the disk', async function () {
115 await assertCountAreOkay(servers) 126 await assertCountAreOkay(servers, videoServer2UUID)
116 }) 127 })
117 128
118 it('Should create some dirty files', async function () { 129 it('Should create some dirty files', async function () {
@@ -176,6 +187,28 @@ describe('Test prune storage scripts', function () {
176 187
177 badNames['avatars'] = [ n1, n2 ] 188 badNames['avatars'] = [ n1, n2 ]
178 } 189 }
190
191 // When we'll prune HLS directories too
192 // {
193 // const directory = join('streaming-playlists', 'hls')
194 // const base = servers[1].servers.buildDirectory(directory)
195
196 // const n1 = buildUUID()
197 // await createFile(join(base, n1))
198 // badNames[directory] = [ n1 ]
199 // }
200
201 // {
202 // const directory = join('streaming-playlists', 'hls', videoServer2UUID)
203 // const base = servers[1].servers.buildDirectory(directory)
204 // const n1 = buildUUID() + '-240-fragmented-.mp4'
205 // const n2 = buildUUID() + '-master.m3u8'
206
207 // await createFile(join(base, n1))
208 // await createFile(join(base, n2))
209
210 // badNames[directory] = [ n1, n2 ]
211 // }
179 } 212 }
180 }) 213 })
181 214
@@ -187,7 +220,7 @@ describe('Test prune storage scripts', function () {
187 }) 220 })
188 221
189 it('Should have removed files', async function () { 222 it('Should have removed files', async function () {
190 await assertCountAreOkay(servers) 223 await assertCountAreOkay(servers, videoServer2UUID)
191 224
192 for (const directory of Object.keys(badNames)) { 225 for (const directory of Object.keys(badNames)) {
193 for (const name of badNames[directory]) { 226 for (const name of badNames[directory]) {
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts
index fcbcb55ba..43fbaec30 100644
--- a/server/tests/cli/update-host.ts
+++ b/server/tests/cli/update-host.ts
@@ -108,21 +108,22 @@ describe('Test update host scripts', function () {
108 108
109 for (const video of data) { 109 for (const video of data) {
110 const videoDetails = await server.videos.get({ id: video.id }) 110 const videoDetails = await server.videos.get({ id: video.id })
111 const files = videoDetails.files.concat(videoDetails.streamingPlaylists[0].files)
111 112
112 expect(videoDetails.files).to.have.lengthOf(4) 113 expect(files).to.have.lengthOf(8)
113 114
114 for (const file of videoDetails.files) { 115 for (const file of files) {
115 expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket') 116 expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket')
116 expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F') 117 expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2F')
117 118
118 const torrent = await parseTorrentVideo(server, videoDetails.uuid, file.resolution.id) 119 const torrent = await parseTorrentVideo(server, file)
119 const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket') 120 const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket')
120 expect(announceWS).to.not.be.undefined 121 expect(announceWS).to.not.be.undefined
121 122
122 const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce') 123 const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce')
123 expect(announceHttp).to.not.be.undefined 124 expect(announceHttp).to.not.be.undefined
124 125
125 expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed') 126 expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/')
126 } 127 }
127 } 128 }
128 }) 129 })
diff --git a/server/tests/plugins/plugin-transcoding.ts b/server/tests/plugins/plugin-transcoding.ts
index c14c34c7e..93637e3ce 100644
--- a/server/tests/plugins/plugin-transcoding.ts
+++ b/server/tests/plugins/plugin-transcoding.ts
@@ -2,7 +2,6 @@
2 2
3import 'mocha' 3import 'mocha'
4import { expect } from 'chai' 4import { expect } from 'chai'
5import { join } from 'path'
6import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' 5import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
7import { 6import {
8 cleanupTests, 7 cleanupTests,
@@ -247,7 +246,9 @@ describe('Test transcoding plugins', function () {
247 const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid 246 const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid
248 await waitJobs([ server ]) 247 await waitJobs([ server ])
249 248
250 const path = server.servers.buildDirectory(join('videos', videoUUID + '-240.mp4')) 249 const video = await server.videos.get({ id: videoUUID })
250
251 const path = server.servers.buildWebTorrentFilePath(video.files[0].fileUrl)
251 const audioProbe = await getAudioStream(path) 252 const audioProbe = await getAudioStream(path)
252 expect(audioProbe.audioStream.codec_name).to.equal('opus') 253 expect(audioProbe.audioStream.codec_name).to.equal('opus')
253 254
diff --git a/server/types/models/video/video-streaming-playlist.ts b/server/types/models/video/video-streaming-playlist.ts
index 8b3ef51fc..1e4dccb8e 100644
--- a/server/types/models/video/video-streaming-playlist.ts
+++ b/server/types/models/video/video-streaming-playlist.ts
@@ -39,5 +39,5 @@ export type MStreamingPlaylistRedundanciesOpt =
39 PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]> 39 PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
40 40
41export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo { 41export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo {
42 return !!(value as MStreamingPlaylist).playlistUrl 42 return !!(value as MStreamingPlaylist).videoId
43} 43}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 10:41:17 +0000