aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-01-25 11:33:26 +0100
committerChocobozzz <chocobozzz@cpy.re>2021-01-25 14:38:52 +0100
commit6939cbac48e0a9823b34577836480ae3c28186be (patch)
tree9ee44d4d26931ee0550121acd4f19e83b4c5caf3
parent77d7e851dccf17dcc89e8fcc2db3f655d1e63f95 (diff)
downloadPeerTube-6939cbac48e0a9823b34577836480ae3c28186be.tar.gz
PeerTube-6939cbac48e0a9823b34577836480ae3c28186be.tar.zst
PeerTube-6939cbac48e0a9823b34577836480ae3c28186be.zip
Safely remove webtorrent files
Only remove them on max quality HLS playlist generation
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.ts6
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts10
-rw-r--r--server/models/video/video.ts4
-rw-r--r--server/tests/api/videos/video-transcoder.ts44
4 files changed, 54 insertions, 10 deletions
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index d08079f7e..925450286 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -73,11 +73,11 @@ export class JobsComponent extends RestTable implements OnInit {
73 } 73 }
74 74
75 getColspan () { 75 getColspan () {
76 if (this.jobState === 'all' && this.hasProgress()) return 6 76 if (this.jobState === 'all' && this.hasProgress()) return 7
77 77
78 if (this.jobState === 'all' || this.hasProgress()) return 5 78 if (this.jobState === 'all' || this.hasProgress()) return 6
79 79
80 return 4 80 return 5
81 } 81 }
82 82
83 onJobStateOrTypeChanged () { 83 onJobStateOrTypeChanged () {
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index ee241ad03..9c0b1d1f1 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -93,7 +93,7 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide
93 job 93 job
94 }) 94 })
95 95
96 await retryTransactionWrapper(onHlsPlaylistGeneration, video) 96 await retryTransactionWrapper(onHlsPlaylistGeneration, video, payload.resolution)
97} 97}
98 98
99async function handleNewWebTorrentResolutionJob ( 99async function handleNewWebTorrentResolutionJob (
@@ -121,11 +121,13 @@ async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTran
121 121
122// --------------------------------------------------------------------------- 122// ---------------------------------------------------------------------------
123 123
124async function onHlsPlaylistGeneration (video: MVideoFullLight) { 124async function onHlsPlaylistGeneration (video: MVideoFullLight, resolution: number) {
125 if (video === undefined) return undefined 125 if (video === undefined) return undefined
126 126
127 // We generated the HLS playlist, we don't need the webtorrent files anymore if the admin disabled it 127 const maxQualityFile = video.getMaxQualityFile()
128 if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { 128
129 // We generated the max quality HLS playlist, we don't need the webtorrent files anymore if the admin disabled it
130 if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false && video.hasWebTorrentFiles() && maxQualityFile.resolution === resolution) {
129 for (const file of video.VideoFiles) { 131 for (const file of video.VideoFiles) {
130 await video.removeFile(file) 132 await video.removeFile(file)
131 await file.destroy() 133 await file.destroy()
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 720bfd829..343abde44 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1804,6 +1804,10 @@ export class VideoModel extends Model {
1804 return Object.assign(file, { Video: this }) 1804 return Object.assign(file, { Video: this })
1805 } 1805 }
1806 1806
1807 hasWebTorrentFiles () {
1808 return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0
1809 }
1810
1807 async addAndSaveThumbnail (thumbnail: MThumbnail, transaction: Transaction) { 1811 async addAndSaveThumbnail (thumbnail: MThumbnail, transaction: Transaction) {
1808 thumbnail.videoId = this.id 1812 thumbnail.videoId = this.id
1809 1813
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index 32f566506..631230f26 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -5,7 +5,9 @@ import * as chai from 'chai'
5import { FfprobeData } from 'fluent-ffmpeg' 5import { FfprobeData } from 'fluent-ffmpeg'
6import { omit } from 'lodash' 6import { omit } from 'lodash'
7import { join } from 'path' 7import { join } from 'path'
8import { Job } from '@shared/models'
8import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants' 9import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants'
10import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
9import { 11import {
10 buildAbsoluteFixturePath, 12 buildAbsoluteFixturePath,
11 buildServerDirectory, 13 buildServerDirectory,
@@ -14,6 +16,7 @@ import {
14 flushAndRunMultipleServers, 16 flushAndRunMultipleServers,
15 generateHighBitrateVideo, 17 generateHighBitrateVideo,
16 generateVideoWithFramerate, 18 generateVideoWithFramerate,
19 getJobsListPaginationAndSort,
17 getMyVideos, 20 getMyVideos,
18 getServerFileSize, 21 getServerFileSize,
19 getVideo, 22 getVideo,
@@ -37,12 +40,12 @@ import {
37 getVideoFileFPS, 40 getVideoFileFPS,
38 getVideoFileResolution 41 getVideoFileResolution
39} from '../../../helpers/ffprobe-utils' 42} from '../../../helpers/ffprobe-utils'
40import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
41 43
42const expect = chai.expect 44const expect = chai.expect
43 45
44describe('Test video transcoding', function () { 46describe('Test video transcoding', function () {
45 let servers: ServerInfo[] = [] 47 let servers: ServerInfo[] = []
48 let video4k: string
46 49
47 before(async function () { 50 before(async function () {
48 this.timeout(30_000) 51 this.timeout(30_000)
@@ -578,14 +581,14 @@ describe('Test video transcoding', function () {
578 } 581 }
579 582
580 const resUpload = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) 583 const resUpload = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
581 const videoUUID = resUpload.body.video.uuid 584 video4k = resUpload.body.video.uuid
582 585
583 await waitJobs(servers) 586 await waitJobs(servers)
584 587
585 const resolutions = [ 240, 360, 480, 720, 1080, 1440, 2160 ] 588 const resolutions = [ 240, 360, 480, 720, 1080, 1440, 2160 ]
586 589
587 for (const server of servers) { 590 for (const server of servers) {
588 const res = await getVideo(server.url, videoUUID) 591 const res = await getVideo(server.url, video4k)
589 const videoDetails: VideoDetails = res.body 592 const videoDetails: VideoDetails = res.body
590 593
591 expect(videoDetails.files).to.have.lengthOf(resolutions.length) 594 expect(videoDetails.files).to.have.lengthOf(resolutions.length)
@@ -597,6 +600,41 @@ describe('Test video transcoding', function () {
597 } 600 }
598 }) 601 })
599 602
603 it('Should have the appropriate priorities for transcoding jobs', async function () {
604 const res = await getJobsListPaginationAndSort({
605 url: servers[1].url,
606 accessToken: servers[1].accessToken,
607 start: 0,
608 count: 100,
609 sort: '-createdAt',
610 jobType: 'video-transcoding'
611 })
612
613 const jobs = res.body.data as Job[]
614
615 const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k)
616
617 expect(transcodingJobs).to.have.lengthOf(14)
618
619 const hlsJobs = transcodingJobs.filter(j => j.data.type === 'new-resolution-to-hls')
620 const webtorrentJobs = transcodingJobs.filter(j => j.data.type === 'new-resolution-to-webtorrent')
621 const optimizeJobs = transcodingJobs.filter(j => j.data.type === 'optimize-to-webtorrent')
622
623 expect(hlsJobs).to.have.lengthOf(7)
624 expect(webtorrentJobs).to.have.lengthOf(6)
625 expect(optimizeJobs).to.have.lengthOf(1)
626
627 for (const j of optimizeJobs) {
628 expect(j.priority).to.be.greaterThan(11)
629 expect(j.priority).to.be.lessThan(50)
630 }
631
632 for (const j of hlsJobs.concat(webtorrentJobs)) {
633 expect(j.priority).to.be.greaterThan(100)
634 expect(j.priority).to.be.lessThan(150)
635 }
636 })
637
600 after(async function () { 638 after(async function () {
601 await cleanupTests(servers) 639 await cleanupTests(servers)
602 }) 640 })