From 7448551fe5c111fcc7b7448ad864fb15a5c8d87f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 1 Feb 2021 11:18:50 +0100 Subject: Fix redundancy with HLS only files --- server/models/redundancy/video-redundancy.ts | 99 +++++++++++++++------------- server/tests/api/redundancy/redundancy.ts | 90 +++++++++++++++++++++---- shared/extra-utils/videos/videos.ts | 2 +- 3 files changed, 132 insertions(+), 59 deletions(-) diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index c536c288b..d3b839cfe 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -1,5 +1,5 @@ import { sample } from 'lodash' -import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' +import { col, FindOptions, fn, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize' import { AllowNull, BeforeDestroy, @@ -15,7 +15,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { getServerActor } from '@server/models/application/application' -import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' +import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' import { FileRedundancyInformation, @@ -36,6 +36,7 @@ import { VideoModel } from '../video/video' import { VideoChannelModel } from '../video/video-channel' import { VideoFileModel } from '../video/video-file' import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' +import { forEachSeries } from 'async' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -261,6 +262,8 @@ export class VideoRedundancyModel extends Model { } static async findMostViewToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'views' ], @@ -268,10 +271,10 @@ export class VideoRedundancyModel extends Model { order: getVideoSort('-views'), where: { privacy: VideoPrivacy.PUBLIC, - isLive: false + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude() ] } @@ -280,6 +283,8 @@ export class VideoRedundancyModel extends Model { } static async findTrendingToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'views' ], @@ -289,10 +294,10 @@ export class VideoRedundancyModel extends Model { order: getVideoSort('-trending'), where: { privacy: VideoPrivacy.PUBLIC, - isLive: false + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude(), VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) @@ -303,6 +308,8 @@ export class VideoRedundancyModel extends Model { } static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'publishedAt' ], @@ -313,10 +320,10 @@ export class VideoRedundancyModel extends Model { isLive: false, views: { [Op.gte]: minViews - } + }, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude() ] } @@ -573,32 +580,35 @@ export class VideoRedundancyModel extends Model { static async getStats (strategy: VideoRedundancyStrategyWithManual) { const actor = await getServerActor() - const query: FindOptions = { - raw: true, - attributes: [ - [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ], - [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ], - [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ] - ], - where: { - strategy, - actorId: actor.id - }, - include: [ - { - attributes: [], - model: VideoFileModel, - required: true - } - ] - } - - return VideoRedundancyModel.findOne(query) - .then((r: any) => ({ - totalUsed: parseAggregateResult(r.totalUsed), - totalVideos: r.totalVideos, - totalVideoFiles: r.totalVideoFiles - })) + const sql = `WITH "tmp" AS ` + + `(` + + `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` + + `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + + `FROM "videoRedundancy" AS "videoRedundancy" ` + + `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` + + `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` + + `LEFT JOIN "videoFile" AS "videoStreamingFile" ` + + `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` + + `), ` + + `"videoIds" AS (` + + `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` + + `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` + + `) ` + + `SELECT ` + + `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + + `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` + + `COUNT(*) AS "totalVideoFiles" ` + + `FROM "tmp"` + + return VideoRedundancyModel.sequelize.query(sql, { + replacements: { strategy, actorId: actor.id }, + type: QueryTypes.SELECT + }).then(([ row ]) => ({ + totalUsed: parseAggregateResult(row.totalUsed), + totalVideos: row.totalVideos, + totalVideoFiles: row.totalVideoFiles + })) } static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { @@ -692,23 +702,22 @@ export class VideoRedundancyModel extends Model { } // Don't include video files we already duplicated - private static async buildVideoFileForDuplication () { - const actor = await getServerActor() - + private static buildVideoIdsForDuplication (peertubeActor: MActor) { const notIn = literal( '(' + - `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + + `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + + `UNION ` + + `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + ')' ) return { - attributes: [], - model: VideoFileModel, - required: true, - where: { - id: { - [Op.notIn]: notIn - } + id: { + [Op.notIn]: notIn } } } diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 7cfcf70e1..8da0ba72a 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -21,6 +21,8 @@ import { ServerInfo, setAccessTokensToServers, unfollow, + updateCustomConfig, + updateCustomSubConfig, uploadVideo, viewVideo, wait, @@ -60,7 +62,7 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) } -async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) { +async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) { const strategies: any[] = [] if (strategy !== null) { @@ -75,6 +77,9 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, add const config = { transcoding: { + webtorrent: { + enabled: withWebtorrent + }, hls: { enabled: true } @@ -253,7 +258,7 @@ async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { return stat } -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) { +async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual) { const stat = await checkStatsGlobal(strategy) expect(stat.totalUsed).to.be.at.least(1).and.below(409601) @@ -261,7 +266,7 @@ async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManu expect(stat.totalVideos).to.equal(1) } -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) { +async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) { const stat = await checkStatsGlobal(strategy) expect(stat.totalUsed).to.equal(0) @@ -313,7 +318,7 @@ describe('Test videos redundancy', function () { it('Should have 1 webseed on the first video', async function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkStatsWith1Webseed(strategy) + await checkStatsWithoutRedundancy(strategy) }) it('Should enable redundancy on server 1', function () { @@ -329,7 +334,7 @@ describe('Test videos redundancy', function () { await check2Webseeds() await check1PlaylistRedundancies() - await checkStatsWith2Webseed(strategy) + await checkStatsWith1Redundancy(strategy) }) it('Should undo redundancy on server 1 and remove duplicated videos', async function () { @@ -363,7 +368,7 @@ describe('Test videos redundancy', function () { it('Should have 1 webseed on the first video', async function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkStatsWith1Webseed(strategy) + await checkStatsWithoutRedundancy(strategy) }) it('Should enable redundancy on server 1', function () { @@ -379,7 +384,7 @@ describe('Test videos redundancy', function () { await check2Webseeds() await check1PlaylistRedundancies() - await checkStatsWith2Webseed(strategy) + await checkStatsWith1Redundancy(strategy) }) it('Should unfollow on server 1 and remove duplicated videos', async function () { @@ -413,7 +418,7 @@ describe('Test videos redundancy', function () { it('Should have 1 webseed on the first video', async function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkStatsWith1Webseed(strategy) + await checkStatsWithoutRedundancy(strategy) }) it('Should enable redundancy on server 1', function () { @@ -429,7 +434,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkStatsWith1Webseed(strategy) + await checkStatsWithoutRedundancy(strategy) }) it('Should view 2 times the first video to have > min_views config', async function () { @@ -451,7 +456,7 @@ describe('Test videos redundancy', function () { await check2Webseeds() await check1PlaylistRedundancies() - await checkStatsWith2Webseed(strategy) + await checkStatsWith1Redundancy(strategy) }) it('Should remove the video and the redundancy files', async function () { @@ -471,6 +476,65 @@ describe('Test videos redundancy', function () { }) }) + describe('With only HLS files', function () { + const strategy = 'recently-added' + + before(async function () { + this.timeout(120000) + + await flushAndRunServers(strategy, { min_views: 3 }, false) + }) + + it('Should have 0 playlist redundancy on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should still have 0 redundancy on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy(strategy) + }) + + it('Should have 1 redundancy on the first video', async function () { + this.timeout(160000) + + await viewVideo(servers[0].url, video1Server2UUID) + await viewVideo(servers[2].url, video1Server2UUID) + + await wait(10000) + await waitJobs(servers) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 1) + await waitJobs(servers) + + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + }) + + it('Should remove the video and the redundancy files', async function () { + this.timeout(20000) + + await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber) + } + }) + }) + describe('With manual strategy', function () { before(function () { this.timeout(120000) @@ -481,7 +545,7 @@ describe('Test videos redundancy', function () { it('Should have 1 webseed on the first video', async function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkStatsWith1Webseed('manual') + await checkStatsWithoutRedundancy('manual') }) it('Should create a redundancy on first video', async function () { @@ -501,7 +565,7 @@ describe('Test videos redundancy', function () { await check2Webseeds() await check1PlaylistRedundancies() - await checkStatsWith2Webseed('manual') + await checkStatsWith1Redundancy('manual') }) it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { @@ -619,7 +683,7 @@ describe('Test videos redundancy', function () { await check2Webseeds() await check1PlaylistRedundancies() - await checkStatsWith2Webseed(strategy) + await checkStatsWith1Redundancy(strategy) const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' }) video2Server2UUID = res.body.video.uuid diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 110552c77..f94fa233c 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -338,7 +338,7 @@ async function checkVideoFilesWereRemoved ( const files = await readdir(directoryPath) for (const file of files) { - expect(file).to.not.contain(videoUUID) + expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID) } } } -- cgit v1.2.3