diff options
Diffstat (limited to 'server/lib')
-rw-r--r-- | server/lib/activitypub/videos/shared/abstract-builder.ts | 25 | ||||
-rw-r--r-- | server/lib/hls.ts | 147 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-file-import.ts | 2 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-studio-edition.ts | 18 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-transcoding.ts | 2 | ||||
-rw-r--r-- | server/lib/transcoding/transcoding.ts | 61 | ||||
-rw-r--r-- | server/lib/video-file.ts | 69 |
7 files changed, 202 insertions, 122 deletions
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index f299ba4fd..c0b92c93d 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { Transaction } from 'sequelize/types' | 1 | import { CreationAttributes, Transaction } from 'sequelize/types' |
2 | import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' | 2 | import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' |
3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | 3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' |
4 | import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' | 4 | import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' |
@@ -7,7 +7,15 @@ import { VideoCaptionModel } from '@server/models/video/video-caption' | |||
7 | import { VideoFileModel } from '@server/models/video/video-file' | 7 | import { VideoFileModel } from '@server/models/video/video-file' |
8 | import { VideoLiveModel } from '@server/models/video/video-live' | 8 | import { VideoLiveModel } from '@server/models/video/video-live' |
9 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 9 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
10 | import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models' | 10 | import { |
11 | MStreamingPlaylistFiles, | ||
12 | MStreamingPlaylistFilesVideo, | ||
13 | MThumbnail, | ||
14 | MVideoCaption, | ||
15 | MVideoFile, | ||
16 | MVideoFullLight, | ||
17 | MVideoThumbnail | ||
18 | } from '@server/types/models' | ||
11 | import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' | 19 | import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' |
12 | import { getOrCreateAPActor } from '../../actors' | 20 | import { getOrCreateAPActor } from '../../actors' |
13 | import { checkUrlsSameHost } from '../../url' | 21 | import { checkUrlsSameHost } from '../../url' |
@@ -125,38 +133,39 @@ export abstract class APVideoAbstractBuilder { | |||
125 | // Remove video playlists that do not exist anymore | 133 | // Remove video playlists that do not exist anymore |
126 | await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) | 134 | await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) |
127 | 135 | ||
136 | const oldPlaylists = video.VideoStreamingPlaylists | ||
128 | video.VideoStreamingPlaylists = [] | 137 | video.VideoStreamingPlaylists = [] |
129 | 138 | ||
130 | for (const playlistAttributes of streamingPlaylistAttributes) { | 139 | for (const playlistAttributes of streamingPlaylistAttributes) { |
131 | const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) | 140 | const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) |
132 | streamingPlaylistModel.Video = video | 141 | streamingPlaylistModel.Video = video |
133 | 142 | ||
134 | await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t) | 143 | await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t) |
135 | 144 | ||
136 | video.VideoStreamingPlaylists.push(streamingPlaylistModel) | 145 | video.VideoStreamingPlaylists.push(streamingPlaylistModel) |
137 | } | 146 | } |
138 | } | 147 | } |
139 | 148 | ||
140 | private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) { | 149 | private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes<VideoStreamingPlaylistModel>, t: Transaction) { |
141 | const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) | 150 | const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) |
142 | 151 | ||
143 | return streamingPlaylist as MStreamingPlaylistFilesVideo | 152 | return streamingPlaylist as MStreamingPlaylistFilesVideo |
144 | } | 153 | } |
145 | 154 | ||
146 | private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) { | 155 | private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType) { |
147 | const playlist = video.VideoStreamingPlaylists.find(s => s.type === type) | 156 | const playlist = oldPlaylists.find(s => s.type === type) |
148 | if (!playlist) return [] | 157 | if (!playlist) return [] |
149 | 158 | ||
150 | return playlist.VideoFiles | 159 | return playlist.VideoFiles |
151 | } | 160 | } |
152 | 161 | ||
153 | private async setStreamingPlaylistFiles ( | 162 | private async setStreamingPlaylistFiles ( |
154 | video: MVideoFullLight, | 163 | oldPlaylists: MStreamingPlaylistFiles[], |
155 | playlistModel: MStreamingPlaylistFilesVideo, | 164 | playlistModel: MStreamingPlaylistFilesVideo, |
156 | tagObjects: ActivityTagObject[], | 165 | tagObjects: ActivityTagObject[], |
157 | t: Transaction | 166 | t: Transaction |
158 | ) { | 167 | ) { |
159 | const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type) | 168 | const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type) |
160 | 169 | ||
161 | const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) | 170 | const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) |
162 | 171 | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 43043315b..20754219f 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -1,7 +1,8 @@ | |||
1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' | 1 | import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' |
2 | import { flatten, uniq } from 'lodash' | 2 | import { flatten, uniq } from 'lodash' |
3 | import PQueue from 'p-queue' | ||
3 | import { basename, dirname, join } from 'path' | 4 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' | 5 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' |
5 | import { sha256 } from '@shared/extra-utils' | 6 | import { sha256 } from '@shared/extra-utils' |
6 | import { VideoStorage } from '@shared/models' | 7 | import { VideoStorage } from '@shared/models' |
7 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' | 8 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' |
@@ -14,7 +15,7 @@ import { sequelizeTypescript } from '../initializers/database' | |||
14 | import { VideoFileModel } from '../models/video/video-file' | 15 | import { VideoFileModel } from '../models/video/video-file' |
15 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 16 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
16 | import { storeHLSFile } from './object-storage' | 17 | import { storeHLSFile } from './object-storage' |
17 | import { getHlsResolutionPlaylistFilename } from './paths' | 18 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' |
18 | import { VideoPathManager } from './video-path-manager' | 19 | import { VideoPathManager } from './video-path-manager' |
19 | 20 | ||
20 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | 21 | async function updateStreamingPlaylistsInfohashesIfNeeded () { |
@@ -33,80 +34,123 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { | |||
33 | } | 34 | } |
34 | } | 35 | } |
35 | 36 | ||
36 | async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { | 37 | async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { |
37 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] | 38 | let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) |
39 | playlistWithFiles = await updateSha256VODSegments(video, playlist) | ||
38 | 40 | ||
39 | for (const file of playlist.VideoFiles) { | 41 | // Refresh playlist, operations can take some time |
40 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) | 42 | playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) |
43 | playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) | ||
44 | await playlistWithFiles.save() | ||
41 | 45 | ||
42 | await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { | 46 | video.setHLSPlaylist(playlistWithFiles) |
43 | const size = await getVideoStreamDimensionsInfo(videoFilePath) | 47 | } |
44 | 48 | ||
45 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | 49 | // --------------------------------------------------------------------------- |
46 | const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` | ||
47 | 50 | ||
48 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | 51 | // Avoid concurrency issues when updating streaming playlist files |
49 | if (file.fps) line += ',FRAME-RATE=' + file.fps | 52 | const playlistFilesQueue = new PQueue({ concurrency: 1 }) |
50 | 53 | ||
51 | const codecs = await Promise.all([ | 54 | function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> { |
52 | getVideoStreamCodec(videoFilePath), | 55 | return playlistFilesQueue.add(async () => { |
53 | getAudioStreamCodec(videoFilePath) | 56 | const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) |
54 | ]) | ||
55 | 57 | ||
56 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` | 58 | const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] |
57 | 59 | ||
58 | masterPlaylists.push(line) | 60 | for (const file of playlist.VideoFiles) { |
59 | masterPlaylists.push(playlistFilename) | 61 | const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) |
60 | }) | 62 | |
61 | } | 63 | await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { |
64 | const size = await getVideoStreamDimensionsInfo(videoFilePath) | ||
65 | |||
66 | const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) | ||
67 | const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` | ||
68 | |||
69 | let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` | ||
70 | if (file.fps) line += ',FRAME-RATE=' + file.fps | ||
71 | |||
72 | const codecs = await Promise.all([ | ||
73 | getVideoStreamCodec(videoFilePath), | ||
74 | getAudioStreamCodec(videoFilePath) | ||
75 | ]) | ||
62 | 76 | ||
63 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => { | 77 | line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` |
78 | |||
79 | masterPlaylists.push(line) | ||
80 | masterPlaylists.push(playlistFilename) | ||
81 | }) | ||
82 | } | ||
83 | |||
84 | if (playlist.playlistFilename) { | ||
85 | await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) | ||
86 | } | ||
87 | playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) | ||
88 | |||
89 | const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) | ||
64 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 90 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') |
65 | 91 | ||
66 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 92 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
67 | await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath) | 93 | playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) |
94 | await remove(masterPlaylistPath) | ||
68 | } | 95 | } |
96 | |||
97 | return playlist.save() | ||
69 | }) | 98 | }) |
70 | } | 99 | } |
71 | 100 | ||
72 | async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { | 101 | // --------------------------------------------------------------------------- |
73 | const json: { [filename: string]: { [range: string]: string } } = {} | 102 | |
103 | async function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> { | ||
104 | return playlistFilesQueue.add(async () => { | ||
105 | const json: { [filename: string]: { [range: string]: string } } = {} | ||
74 | 106 | ||
75 | // For all the resolutions available for this video | 107 | const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) |
76 | for (const file of playlist.VideoFiles) { | ||
77 | const rangeHashes: { [range: string]: string } = {} | ||
78 | const fileWithPlaylist = file.withVideoOrPlaylist(playlist) | ||
79 | 108 | ||
80 | await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { | 109 | // For all the resolutions available for this video |
110 | for (const file of playlist.VideoFiles) { | ||
111 | const rangeHashes: { [range: string]: string } = {} | ||
112 | const fileWithPlaylist = file.withVideoOrPlaylist(playlist) | ||
81 | 113 | ||
82 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { | 114 | await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { |
83 | const playlistContent = await readFile(resolutionPlaylistPath) | ||
84 | const ranges = getRangesFromPlaylist(playlistContent.toString()) | ||
85 | 115 | ||
86 | const fd = await open(videoPath, 'r') | 116 | return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { |
87 | for (const range of ranges) { | 117 | const playlistContent = await readFile(resolutionPlaylistPath) |
88 | const buf = Buffer.alloc(range.length) | 118 | const ranges = getRangesFromPlaylist(playlistContent.toString()) |
89 | await read(fd, buf, 0, range.length, range.offset) | ||
90 | 119 | ||
91 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) | 120 | const fd = await open(videoPath, 'r') |
92 | } | 121 | for (const range of ranges) { |
93 | await close(fd) | 122 | const buf = Buffer.alloc(range.length) |
123 | await read(fd, buf, 0, range.length, range.offset) | ||
94 | 124 | ||
95 | const videoFilename = file.filename | 125 | rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) |
96 | json[videoFilename] = rangeHashes | 126 | } |
127 | await close(fd) | ||
128 | |||
129 | const videoFilename = file.filename | ||
130 | json[videoFilename] = rangeHashes | ||
131 | }) | ||
97 | }) | 132 | }) |
98 | }) | 133 | } |
99 | } | ||
100 | 134 | ||
101 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) | 135 | if (playlist.segmentsSha256Filename) { |
102 | await outputJSON(outputPath, json) | 136 | await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) |
137 | } | ||
138 | playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) | ||
103 | 139 | ||
104 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 140 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
105 | await storeHLSFile(playlist, playlist.segmentsSha256Filename) | 141 | await outputJSON(outputPath, json) |
106 | await remove(outputPath) | 142 | |
107 | } | 143 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
144 | playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) | ||
145 | await remove(outputPath) | ||
146 | } | ||
147 | |||
148 | return playlist.save() | ||
149 | }) | ||
108 | } | 150 | } |
109 | 151 | ||
152 | // --------------------------------------------------------------------------- | ||
153 | |||
110 | async function buildSha256Segment (segmentPath: string) { | 154 | async function buildSha256Segment (segmentPath: string) { |
111 | const buf = await readFile(segmentPath) | 155 | const buf = await readFile(segmentPath) |
112 | return sha256(buf) | 156 | return sha256(buf) |
@@ -190,7 +234,8 @@ export { | |||
190 | updateSha256VODSegments, | 234 | updateSha256VODSegments, |
191 | buildSha256Segment, | 235 | buildSha256Segment, |
192 | downloadPlaylistSegments, | 236 | downloadPlaylistSegments, |
193 | updateStreamingPlaylistsInfohashesIfNeeded | 237 | updateStreamingPlaylistsInfohashesIfNeeded, |
238 | updatePlaylistAfterFileChange | ||
194 | } | 239 | } |
195 | 240 | ||
196 | // --------------------------------------------------------------------------- | 241 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 1c600e2a7..71c5444af 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -55,7 +55,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
55 | 55 | ||
56 | if (currentVideoFile) { | 56 | if (currentVideoFile) { |
57 | // Remove old file and old torrent | 57 | // Remove old file and old torrent |
58 | await video.removeWebTorrentFileAndTorrent(currentVideoFile) | 58 | await video.removeWebTorrentFile(currentVideoFile) |
59 | // Remove the old video file from the array | 59 | // Remove the old video file from the array |
60 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) | 60 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) |
61 | 61 | ||
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index 434d0ffe8..735150d57 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts | |||
@@ -9,6 +9,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths' | |||
9 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' | 9 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' |
10 | import { isAbleToUploadVideo } from '@server/lib/user' | 10 | import { isAbleToUploadVideo } from '@server/lib/user' |
11 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' | 11 | import { addOptimizeOrMergeAudioJob } from '@server/lib/video' |
12 | import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' | ||
12 | import { VideoPathManager } from '@server/lib/video-path-manager' | 13 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' | 14 | import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' |
14 | import { UserModel } from '@server/models/user/user' | 15 | import { UserModel } from '@server/models/user/user' |
@@ -27,12 +28,12 @@ import { | |||
27 | } from '@shared/extra-utils' | 28 | } from '@shared/extra-utils' |
28 | import { | 29 | import { |
29 | VideoStudioEditionPayload, | 30 | VideoStudioEditionPayload, |
30 | VideoStudioTaskPayload, | 31 | VideoStudioTask, |
31 | VideoStudioTaskCutPayload, | 32 | VideoStudioTaskCutPayload, |
32 | VideoStudioTaskIntroPayload, | 33 | VideoStudioTaskIntroPayload, |
33 | VideoStudioTaskOutroPayload, | 34 | VideoStudioTaskOutroPayload, |
34 | VideoStudioTaskWatermarkPayload, | 35 | VideoStudioTaskPayload, |
35 | VideoStudioTask | 36 | VideoStudioTaskWatermarkPayload |
36 | } from '@shared/models' | 37 | } from '@shared/models' |
37 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 38 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
38 | 39 | ||
@@ -89,7 +90,6 @@ async function processVideoStudioEdition (job: Job) { | |||
89 | await move(editionResultPath, outputPath) | 90 | await move(editionResultPath, outputPath) |
90 | 91 | ||
91 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) | 92 | await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) |
92 | |||
93 | await removeAllFiles(video, newFile) | 93 | await removeAllFiles(video, newFile) |
94 | 94 | ||
95 | await newFile.save() | 95 | await newFile.save() |
@@ -197,18 +197,12 @@ async function buildNewFile (video: MVideoId, path: string) { | |||
197 | } | 197 | } |
198 | 198 | ||
199 | async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { | 199 | async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { |
200 | const hls = video.getHLSPlaylist() | 200 | await removeHLSPlaylist(video) |
201 | |||
202 | if (hls) { | ||
203 | await video.removeStreamingPlaylistFiles(hls) | ||
204 | await hls.destroy() | ||
205 | } | ||
206 | 201 | ||
207 | for (const file of video.VideoFiles) { | 202 | for (const file of video.VideoFiles) { |
208 | if (file.id === webTorrentFileException.id) continue | 203 | if (file.id === webTorrentFileException.id) continue |
209 | 204 | ||
210 | await video.removeWebTorrentFileAndTorrent(file) | 205 | await removeWebTorrentFile(video, file.id) |
211 | await file.destroy() | ||
212 | } | 206 | } |
213 | } | 207 | } |
214 | 208 | ||
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 5afca65ca..1b34ced14 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -149,7 +149,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay | |||
149 | if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { | 149 | if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { |
150 | // Remove webtorrent files if not enabled | 150 | // Remove webtorrent files if not enabled |
151 | for (const file of video.VideoFiles) { | 151 | for (const file of video.VideoFiles) { |
152 | await video.removeWebTorrentFileAndTorrent(file) | 152 | await video.removeWebTorrentFile(file) |
153 | await file.destroy() | 153 | await file.destroy() |
154 | } | 154 | } |
155 | 155 | ||
diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts index 69a973fbd..924141d1c 100644 --- a/server/lib/transcoding/transcoding.ts +++ b/server/lib/transcoding/transcoding.ts | |||
@@ -5,9 +5,8 @@ import { toEven } from '@server/helpers/core-utils' | |||
5 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | 5 | import { retryTransactionWrapper } from '@server/helpers/database-utils' |
6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
7 | import { sequelizeTypescript } from '@server/initializers/database' | 7 | import { sequelizeTypescript } from '@server/initializers/database' |
8 | import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 8 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
9 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' | 9 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' |
10 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | ||
11 | import { | 10 | import { |
12 | buildFileMetadata, | 11 | buildFileMetadata, |
13 | canDoQuickTranscode, | 12 | canDoQuickTranscode, |
@@ -18,17 +17,10 @@ import { | |||
18 | TranscodeVODOptionsType | 17 | TranscodeVODOptionsType |
19 | } from '../../helpers/ffmpeg' | 18 | } from '../../helpers/ffmpeg' |
20 | import { CONFIG } from '../../initializers/config' | 19 | import { CONFIG } from '../../initializers/config' |
21 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' | ||
22 | import { VideoFileModel } from '../../models/video/video-file' | 20 | import { VideoFileModel } from '../../models/video/video-file' |
23 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | 21 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' |
24 | import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' | 22 | import { updatePlaylistAfterFileChange } from '../hls' |
25 | import { | 23 | import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' |
26 | generateHLSMasterPlaylistFilename, | ||
27 | generateHlsSha256SegmentsFilename, | ||
28 | generateHLSVideoFilename, | ||
29 | generateWebTorrentVideoFilename, | ||
30 | getHlsResolutionPlaylistFilename | ||
31 | } from '../paths' | ||
32 | import { VideoPathManager } from '../video-path-manager' | 24 | import { VideoPathManager } from '../video-path-manager' |
33 | import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' | 25 | import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' |
34 | 26 | ||
@@ -260,7 +252,7 @@ async function onWebTorrentVideoFileTranscoding ( | |||
260 | await createTorrentAndSetInfoHash(video, videoFile) | 252 | await createTorrentAndSetInfoHash(video, videoFile) |
261 | 253 | ||
262 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | 254 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) |
263 | if (oldFile) await video.removeWebTorrentFileAndTorrent(oldFile) | 255 | if (oldFile) await video.removeWebTorrentFile(oldFile) |
264 | 256 | ||
265 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | 257 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) |
266 | video.VideoFiles = await video.$get('VideoFiles') | 258 | video.VideoFiles = await video.$get('VideoFiles') |
@@ -314,35 +306,15 @@ async function generateHlsPlaylistCommon (options: { | |||
314 | await transcodeVOD(transcodeOptions) | 306 | await transcodeVOD(transcodeOptions) |
315 | 307 | ||
316 | // Create or update the playlist | 308 | // Create or update the playlist |
317 | const { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } = await retryTransactionWrapper(() => { | 309 | const playlist = await retryTransactionWrapper(() => { |
318 | return sequelizeTypescript.transaction(async transaction => { | 310 | return sequelizeTypescript.transaction(async transaction => { |
319 | const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) | 311 | return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) |
320 | |||
321 | const oldPlaylistFilename = playlist.playlistFilename | ||
322 | const oldSegmentsSha256Filename = playlist.segmentsSha256Filename | ||
323 | |||
324 | playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) | ||
325 | playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) | ||
326 | |||
327 | playlist.p2pMediaLoaderInfohashes = [] | ||
328 | playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION | ||
329 | |||
330 | playlist.type = VideoStreamingPlaylistType.HLS | ||
331 | |||
332 | await playlist.save({ transaction }) | ||
333 | |||
334 | return { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } | ||
335 | }) | 312 | }) |
336 | }) | 313 | }) |
337 | 314 | ||
338 | if (oldPlaylistFilename) await video.removeStreamingPlaylistFile(playlist, oldPlaylistFilename) | ||
339 | if (oldSegmentsSha256Filename) await video.removeStreamingPlaylistFile(playlist, oldSegmentsSha256Filename) | ||
340 | |||
341 | // Build the new playlist file | ||
342 | const extname = extnameUtil(videoFilename) | ||
343 | const newVideoFile = new VideoFileModel({ | 315 | const newVideoFile = new VideoFileModel({ |
344 | resolution, | 316 | resolution, |
345 | extname, | 317 | extname: extnameUtil(videoFilename), |
346 | size: 0, | 318 | size: 0, |
347 | filename: videoFilename, | 319 | filename: videoFilename, |
348 | fps: -1, | 320 | fps: -1, |
@@ -350,8 +322,6 @@ async function generateHlsPlaylistCommon (options: { | |||
350 | }) | 322 | }) |
351 | 323 | ||
352 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) | 324 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) |
353 | |||
354 | // Move files from tmp transcoded directory to the appropriate place | ||
355 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | 325 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) |
356 | 326 | ||
357 | // Move playlist file | 327 | // Move playlist file |
@@ -369,21 +339,14 @@ async function generateHlsPlaylistCommon (options: { | |||
369 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | 339 | await createTorrentAndSetInfoHash(playlist, newVideoFile) |
370 | 340 | ||
371 | const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) | 341 | const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) |
372 | if (oldFile) await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | 342 | if (oldFile) { |
343 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | ||
344 | await oldFile.destroy() | ||
345 | } | ||
373 | 346 | ||
374 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | 347 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) |
375 | 348 | ||
376 | const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo | 349 | await updatePlaylistAfterFileChange(video, playlist) |
377 | playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') | ||
378 | playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) | ||
379 | playlist.storage = VideoStorage.FILE_SYSTEM | ||
380 | |||
381 | await playlist.save() | ||
382 | |||
383 | video.setHLSPlaylist(playlist) | ||
384 | |||
385 | await updateMasterHLSPlaylist(video, playlistWithFiles) | ||
386 | await updateSha256VODSegments(video, playlistWithFiles) | ||
387 | 350 | ||
388 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | 351 | return { resolutionPlaylistPath, videoFile: savedVideoFile } |
389 | } | 352 | } |
diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts new file mode 100644 index 000000000..2ab7190f1 --- /dev/null +++ b/server/lib/video-file.ts | |||
@@ -0,0 +1,69 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { MVideoWithAllFiles } from '@server/types/models' | ||
3 | import { lTags } from './object-storage/shared' | ||
4 | |||
5 | async function removeHLSPlaylist (video: MVideoWithAllFiles) { | ||
6 | const hls = video.getHLSPlaylist() | ||
7 | if (!hls) return | ||
8 | |||
9 | await video.removeStreamingPlaylistFiles(hls) | ||
10 | await hls.destroy() | ||
11 | |||
12 | video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) | ||
13 | } | ||
14 | |||
15 | async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { | ||
16 | logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) | ||
17 | |||
18 | const hls = video.getHLSPlaylist() | ||
19 | const files = hls.VideoFiles | ||
20 | |||
21 | if (files.length === 1) { | ||
22 | await removeHLSPlaylist(video) | ||
23 | return undefined | ||
24 | } | ||
25 | |||
26 | const toDelete = files.find(f => f.id === fileToDeleteId) | ||
27 | await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete) | ||
28 | await toDelete.destroy() | ||
29 | |||
30 | hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id) | ||
31 | |||
32 | return hls | ||
33 | } | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | async function removeAllWebTorrentFiles (video: MVideoWithAllFiles) { | ||
38 | for (const file of video.VideoFiles) { | ||
39 | await video.removeWebTorrentFile(file) | ||
40 | await file.destroy() | ||
41 | } | ||
42 | |||
43 | video.VideoFiles = [] | ||
44 | |||
45 | return video | ||
46 | } | ||
47 | |||
48 | async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId: number) { | ||
49 | const files = video.VideoFiles | ||
50 | |||
51 | if (files.length === 1) { | ||
52 | return removeAllWebTorrentFiles(video) | ||
53 | } | ||
54 | |||
55 | const toDelete = files.find(f => f.id === fileToDeleteId) | ||
56 | await video.removeWebTorrentFile(toDelete) | ||
57 | await toDelete.destroy() | ||
58 | |||
59 | video.VideoFiles = files.filter(f => f.id !== toDelete.id) | ||
60 | |||
61 | return video | ||
62 | } | ||
63 | |||
64 | export { | ||
65 | removeHLSPlaylist, | ||
66 | removeHLSFile, | ||
67 | removeAllWebTorrentFiles, | ||
68 | removeWebTorrentFile | ||
69 | } | ||