diff options
Diffstat (limited to 'server/lib')
-rw-r--r-- | server/lib/hls.ts | 15 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/move-to-object-storage.ts | 17 | ||||
-rw-r--r-- | server/lib/object-storage/videos.ts | 6 |
3 files changed, 28 insertions, 10 deletions
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 1574ff27b..985f50587 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -3,6 +3,7 @@ import { flatten, uniq } from 'lodash' | |||
3 | import { basename, dirname, join } from 'path' | 3 | import { basename, dirname, join } from 'path' |
4 | import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' | 4 | import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' |
5 | import { sha256 } from '@shared/extra-utils' | 5 | import { sha256 } from '@shared/extra-utils' |
6 | import { VideoStorage } from '@shared/models' | ||
6 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' | 7 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' |
7 | import { logger } from '../helpers/logger' | 8 | import { logger } from '../helpers/logger' |
8 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | 9 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
@@ -12,6 +13,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers | |||
12 | import { sequelizeTypescript } from '../initializers/database' | 13 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { VideoFileModel } from '../models/video/video-file' | 14 | import { VideoFileModel } from '../models/video/video-file' |
14 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 15 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
16 | import { storeHLSFile } from './object-storage' | ||
15 | import { getHlsResolutionPlaylistFilename } from './paths' | 17 | import { getHlsResolutionPlaylistFilename } from './paths' |
16 | import { VideoPathManager } from './video-path-manager' | 18 | import { VideoPathManager } from './video-path-manager' |
17 | 19 | ||
@@ -58,8 +60,12 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl | |||
58 | }) | 60 | }) |
59 | } | 61 | } |
60 | 62 | ||
61 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { | 63 | await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => { |
62 | return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 64 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') |
65 | |||
66 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | ||
67 | await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath) | ||
68 | } | ||
63 | }) | 69 | }) |
64 | } | 70 | } |
65 | 71 | ||
@@ -94,6 +100,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP | |||
94 | 100 | ||
95 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) | 101 | const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) |
96 | await outputJSON(outputPath, json) | 102 | await outputJSON(outputPath, json) |
103 | |||
104 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | ||
105 | await storeHLSFile(playlist, playlist.segmentsSha256Filename) | ||
106 | await remove(outputPath) | ||
107 | } | ||
97 | } | 108 | } |
98 | 109 | ||
99 | async function buildSha256Segment (segmentPath: string) { | 110 | async function buildSha256Segment (segmentPath: string) { |
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index 9e39322a8..69b441176 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { Job } from 'bull' | 1 | import { Job } from 'bull' |
2 | import { remove } from 'fs-extra' | 2 | import { remove } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { logger } from '@server/helpers/logger' | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' | 5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' |
6 | import { CONFIG } from '@server/initializers/config' | 6 | import { CONFIG } from '@server/initializers/config' |
7 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' | 7 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' |
@@ -13,6 +13,8 @@ import { VideoJobInfoModel } from '@server/models/video/video-job-info' | |||
13 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' | 13 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' |
14 | import { MoveObjectStoragePayload, VideoStorage } from '@shared/models' | 14 | import { MoveObjectStoragePayload, VideoStorage } from '@shared/models' |
15 | 15 | ||
16 | const lTagsBase = loggerTagsFactory('move-object-storage') | ||
17 | |||
16 | export async function processMoveToObjectStorage (job: Job) { | 18 | export async function processMoveToObjectStorage (job: Job) { |
17 | const payload = job.data as MoveObjectStoragePayload | 19 | const payload = job.data as MoveObjectStoragePayload |
18 | logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) | 20 | logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) |
@@ -20,26 +22,33 @@ export async function processMoveToObjectStorage (job: Job) { | |||
20 | const video = await VideoModel.loadWithFiles(payload.videoUUID) | 22 | const video = await VideoModel.loadWithFiles(payload.videoUUID) |
21 | // No video, maybe deleted? | 23 | // No video, maybe deleted? |
22 | if (!video) { | 24 | if (!video) { |
23 | logger.info('Can\'t process job %d, video does not exist.', job.id) | 25 | logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID)) |
24 | return undefined | 26 | return undefined |
25 | } | 27 | } |
26 | 28 | ||
29 | const lTags = lTagsBase(video.uuid, video.url) | ||
30 | |||
27 | try { | 31 | try { |
28 | if (video.VideoFiles) { | 32 | if (video.VideoFiles) { |
33 | logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags) | ||
34 | |||
29 | await moveWebTorrentFiles(video) | 35 | await moveWebTorrentFiles(video) |
30 | } | 36 | } |
31 | 37 | ||
32 | if (video.VideoStreamingPlaylists) { | 38 | if (video.VideoStreamingPlaylists) { |
39 | logger.debug('Moving HLS playlist of %s.', video.uuid) | ||
40 | |||
33 | await moveHLSFiles(video) | 41 | await moveHLSFiles(video) |
34 | } | 42 | } |
35 | 43 | ||
36 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') | 44 | const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') |
37 | if (pendingMove === 0) { | 45 | if (pendingMove === 0) { |
38 | logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) | 46 | logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id, lTags) |
47 | |||
39 | await doAfterLastJob(video, payload.isNewVideo) | 48 | await doAfterLastJob(video, payload.isNewVideo) |
40 | } | 49 | } |
41 | } catch (err) { | 50 | } catch (err) { |
42 | logger.error('Cannot move video %s to object storage.', video.url, { err }) | 51 | logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags }) |
43 | 52 | ||
44 | await moveToFailedMoveToObjectStorageState(video) | 53 | await moveToFailedMoveToObjectStorageState(video) |
45 | await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') | 54 | await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') |
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index 8988f3e2a..066b48ab0 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts | |||
@@ -6,11 +6,9 @@ import { getHLSDirectory } from '../paths' | |||
6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | 6 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' |
7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | 7 | import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' |
8 | 8 | ||
9 | function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) { | 9 | function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) { |
10 | const baseHlsDirectory = getHLSDirectory(playlist.Video) | ||
11 | |||
12 | return storeObject({ | 10 | return storeObject({ |
13 | inputPath: join(baseHlsDirectory, filename), | 11 | inputPath: path ?? join(getHLSDirectory(playlist.Video), filename), |
14 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), | 12 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), |
15 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | 13 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS |
16 | }) | 14 | }) |