From 764b1a14fc494f2cfd7ea590d2f07b01df65c7ad Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 23 Jul 2021 11:20:00 +0200 Subject: Use random names for VOD HLS playlists --- server/tests/api/live/live-constraints.ts | 4 +-- server/tests/api/live/live-save-replay.ts | 14 ++++---- server/tests/api/live/live.ts | 14 +++++--- server/tests/api/users/users-multiple-servers.ts | 4 +-- server/tests/api/videos/resumable-upload.ts | 7 +++- server/tests/api/videos/video-hls.ts | 11 ++++--- server/tests/cli/optimize-old-videos.ts | 3 +- server/tests/cli/prune-storage.ts | 41 +++++++++++++++++++++--- server/tests/cli/update-host.ts | 11 ++++--- server/tests/plugins/plugin-transcoding.ts | 5 +-- 10 files changed, 80 insertions(+), 34 deletions(-) (limited to 'server/tests') diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts index 20346113d..4acde3cc5 100644 --- a/server/tests/api/live/live-constraints.ts +++ b/server/tests/api/live/live-constraints.ts @@ -4,7 +4,7 @@ import 'mocha' import * as chai from 'chai' import { VideoPrivacy } from '@shared/models' import { - checkLiveCleanup, + checkLiveCleanupAfterSave, cleanupTests, ConfigCommand, createMultipleServers, @@ -43,7 +43,7 @@ describe('Test live constraints', function () { expect(video.duration).to.be.greaterThan(0) } - await checkLiveCleanup(servers[0], videoId, resolutions) + await checkLiveCleanupAfterSave(servers[0], videoId, resolutions) } async function waitUntilLivePublishedOnAllServers (videoId: string) { diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index bd15396ec..8f1fb78a5 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts @@ -4,7 +4,7 @@ import 'mocha' import * as chai from 'chai' import { FfmpegCommand } from 'fluent-ffmpeg' import { - checkLiveCleanup, + checkLiveCleanupAfterSave, cleanupTests, ConfigCommand, createMultipleServers, @@ -150,7 +150,7 @@ describe('Save replay setting', function () { await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) // No resolutions saved since we did not save replay - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) it('Should correctly terminate the stream on blacklist and delete the live', async function () { @@ -179,7 +179,7 @@ describe('Save replay setting', function () { await wait(5000) await waitJobs(servers) - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) it('Should correctly terminate the stream on delete and delete the video', async function () { @@ -203,7 +203,7 @@ describe('Save replay setting', function () { await waitJobs(servers) await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) }) @@ -259,7 +259,7 @@ describe('Save replay setting', function () { }) it('Should have cleaned up the live files', async function () { - await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ]) }) it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { @@ -287,7 +287,7 @@ describe('Save replay setting', function () { await wait(5000) await waitJobs(servers) - await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ]) }) it('Should correctly terminate the stream on delete and delete the video', async function () { @@ -310,7 +310,7 @@ describe('Save replay setting', function () { await waitJobs(servers) await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) }) diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index 4676a840a..d555cff19 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts @@ -2,10 +2,10 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' +import { basename, join } from 'path' import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' import { - checkLiveCleanup, + checkLiveCleanupAfterSave, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, cleanupTests, @@ -506,6 +506,10 @@ describe('Test live', function () { await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) + // We should have generated random filenames + expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') + expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json') + expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length) for (const resolution of resolutions) { @@ -520,7 +524,9 @@ describe('Test live', function () { expect(file.fps).to.be.approximately(30, 2) } - const filename = `${video.uuid}-${resolution}-fragmented.mp4` + const filename = basename(file.fileUrl) + expect(filename).to.not.contain(video.uuid) + const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename)) const probe = await ffprobePromise(segmentPath) @@ -537,7 +543,7 @@ describe('Test live', function () { it('Should correctly have cleaned up the live files', async function () { this.timeout(30000) - await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ]) + await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ]) }) }) diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts index 16372b039..d0ca82b07 100644 --- a/server/tests/api/users/users-multiple-servers.ts +++ b/server/tests/api/users/users-multiple-servers.ts @@ -58,10 +58,10 @@ describe('Test users with multiple servers', function () { const { uuid } = await servers[0].videos.upload({ token: userAccessToken }) videoUUID = uuid + await waitJobs(servers) + await saveVideoInServers(servers, videoUUID) } - - await waitJobs(servers) }) it('Should be able to update my display name', async function () { diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts index c94d92cf2..857859fd3 100644 --- a/server/tests/api/videos/resumable-upload.ts +++ b/server/tests/api/videos/resumable-upload.ts @@ -170,8 +170,13 @@ describe('Test resumable upload', function () { const size = 1000 + // Content length check seems to have changed in v16 + const expectedStatus = process.version.startsWith('v16') + ? HttpStatusCode.CONFLICT_409 + : HttpStatusCode.BAD_REQUEST_400 + const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}` - await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size }) + await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size }) await checkFileSize(uploadId, 0) }) }) diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 921d7ce64..961f0e617 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -2,7 +2,8 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' +import { basename, join } from 'path' +import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' import { checkDirectoryIsEmpty, checkResolutionsInMasterPlaylist, @@ -19,8 +20,6 @@ import { } from '@shared/extra-utils' import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' -import { uuidRegex } from '@shared/core-utils' -import { basename } from 'path/posix' const expect = chai.expect @@ -78,11 +77,13 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h // Check resolution playlists { for (const resolution of resolutions) { + const file = hlsFiles.find(f => f.resolution.id === resolution) + const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' + const subPlaylist = await server.streamingPlaylists.get({ - url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8` + url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` }) - const file = hlsFiles.find(f => f.resolution.id === resolution) expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) expect(subPlaylist).to.contain(basename(file.fileUrl)) } diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts index 685b3b7b8..579b2e7d8 100644 --- a/server/tests/cli/optimize-old-videos.ts +++ b/server/tests/cli/optimize-old-videos.ts @@ -2,7 +2,6 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' import { cleanupTests, createMultipleServers, @@ -86,7 +85,7 @@ describe('Test optimize old videos', function () { expect(file.size).to.be.below(8000000) - const path = servers[0].servers.buildDirectory(join('videos', video.uuid + '-' + file.resolution.id + '.mp4')) + const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl) const bitrate = await getVideoFileBitrate(path) const fps = await getVideoFileFPS(path) const resolution = await getVideoFileResolution(path) diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 954a87833..2d4c02da7 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts @@ -36,7 +36,7 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst } } -async function assertCountAreOkay (servers: PeerTubeServer[]) { +async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) { for (const server of servers) { const videosCount = await countFiles(server, 'videos') expect(videosCount).to.equal(8) @@ -53,12 +53,21 @@ async function assertCountAreOkay (servers: PeerTubeServer[]) { const avatarsCount = await countFiles(server, 'avatars') expect(avatarsCount).to.equal(2) } + + // When we'll prune HLS directories too + // const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/') + // expect(hlsRootCount).to.equal(2) + + // const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID) + // expect(hlsCount).to.equal(10) } describe('Test prune storage scripts', function () { let servers: PeerTubeServer[] const badNames: { [directory: string]: string[] } = {} + let videoServer2UUID: string + before(async function () { this.timeout(120000) @@ -68,7 +77,9 @@ describe('Test prune storage scripts', function () { for (const server of servers) { await server.videos.upload({ attributes: { name: 'video 1' } }) - await server.videos.upload({ attributes: { name: 'video 2' } }) + + const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } }) + if (server.serverNumber === 2) videoServer2UUID = uuid await server.users.updateMyAvatar({ fixture: 'avatar.png' }) @@ -112,7 +123,7 @@ describe('Test prune storage scripts', function () { }) it('Should have the files on the disk', async function () { - await assertCountAreOkay(servers) + await assertCountAreOkay(servers, videoServer2UUID) }) it('Should create some dirty files', async function () { @@ -176,6 +187,28 @@ describe('Test prune storage scripts', function () { badNames['avatars'] = [ n1, n2 ] } + + // When we'll prune HLS directories too + // { + // const directory = join('streaming-playlists', 'hls') + // const base = servers[1].servers.buildDirectory(directory) + + // const n1 = buildUUID() + // await createFile(join(base, n1)) + // badNames[directory] = [ n1 ] + // } + + // { + // const directory = join('streaming-playlists', 'hls', videoServer2UUID) + // const base = servers[1].servers.buildDirectory(directory) + // const n1 = buildUUID() + '-240-fragmented-.mp4' + // const n2 = buildUUID() + '-master.m3u8' + + // await createFile(join(base, n1)) + // await createFile(join(base, n2)) + + // badNames[directory] = [ n1, n2 ] + // } } }) @@ -187,7 +220,7 @@ describe('Test prune storage scripts', function () { }) it('Should have removed files', async function () { - await assertCountAreOkay(servers) + await assertCountAreOkay(servers, videoServer2UUID) for (const directory of Object.keys(badNames)) { for (const name of badNames[directory]) { diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index fcbcb55ba..43fbaec30 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts @@ -108,21 +108,22 @@ describe('Test update host scripts', function () { for (const video of data) { const videoDetails = await server.videos.get({ id: video.id }) + const files = videoDetails.files.concat(videoDetails.streamingPlaylists[0].files) - expect(videoDetails.files).to.have.lengthOf(4) + expect(files).to.have.lengthOf(8) - for (const file of videoDetails.files) { + for (const file of files) { expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket') - expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F') + expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2F') - const torrent = await parseTorrentVideo(server, videoDetails.uuid, file.resolution.id) + const torrent = await parseTorrentVideo(server, file) const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket') expect(announceWS).to.not.be.undefined const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce') expect(announceHttp).to.not.be.undefined - expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed') + expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/') } } }) diff --git a/server/tests/plugins/plugin-transcoding.ts b/server/tests/plugins/plugin-transcoding.ts index c14c34c7e..93637e3ce 100644 --- a/server/tests/plugins/plugin-transcoding.ts +++ b/server/tests/plugins/plugin-transcoding.ts @@ -2,7 +2,6 @@ import 'mocha' import { expect } from 'chai' -import { join } from 'path' import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' import { cleanupTests, @@ -247,7 +246,9 @@ describe('Test transcoding plugins', function () { const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid await waitJobs([ server ]) - const path = server.servers.buildDirectory(join('videos', videoUUID + '-240.mp4')) + const video = await server.videos.get({ id: videoUUID }) + + const path = server.servers.buildWebTorrentFilePath(video.files[0].fileUrl) const audioProbe = await getAudioStream(path) expect(audioProbe.audioStream.codec_name).to.equal('opus') -- cgit v1.2.3