X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fredundancy%2Fvideo-redundancy.ts;h=c2a72b71f42d554bae801ca1e106f88a741d2bd8;hb=5e47f6ab984a7d00782e4c7030afffa1ba480add;hp=1c8b2cf782ea123a25bbf382e5f00bdee2da02c4;hpb=26d6bf6533023326fa017812cf31bbe20c752d36;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 1c8b2cf78..c2a72b71f 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -1,3 +1,5 @@ +import { sample } from 'lodash' +import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize' import { AllowNull, BeforeDestroy, @@ -12,31 +14,32 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { ActorModel } from '../activitypub/actor' -import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' +import { getServerActor } from '@server/models/application/application' +import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' +import { + CacheFileObject, + FileRedundancyInformation, + StreamingPlaylistRedundancyInformation, + VideoPrivacy, + VideoRedundanciesTarget, + VideoRedundancy, + VideoRedundancyStrategy, + VideoRedundancyStrategyWithManual +} from '@shared/models' +import { AttributesOnly } from '@shared/typescript-utils' +import { isTestInstance } from '../../helpers/core-utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' +import { logger } from '../../helpers/logger' +import { CONFIG } from '../../initializers/config' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' -import { VideoFileModel } from '../video/video-file' +import { ActorModel } from '../actor/actor' +import { ServerModel } from '../server/server' +import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared' +import { ScheduleVideoUpdateModel } from '../video/schedule-video-update' import { VideoModel } from '../video/video' -import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy' -import { logger } from '../../helpers/logger' -import { CacheFileObject, VideoPrivacy } from '../../../shared' import { VideoChannelModel } from '../video/video-channel' -import { ServerModel } from '../server/server' -import { sample } from 'lodash' -import { isTestInstance } from '../../helpers/core-utils' -import * as Bluebird from 'bluebird' -import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' +import { VideoFileModel } from '../video/video-file' import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' -import { CONFIG } from '../../initializers/config' -import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' -import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' -import { - FileRedundancyInformation, - StreamingPlaylistRedundancyInformation, - VideoRedundancy -} from '@shared/models/redundancy/video-redundancy.model' -import { getServerActor } from '@server/models/application/application' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -78,13 +81,16 @@ export enum ScopeNames { { fields: [ 'actorId' ] }, + { + fields: [ 'expiresOn' ] + }, { fields: [ 'url' ], unique: true } ] }) -export class VideoRedundancyModel extends Model { +export class VideoRedundancyModel extends Model>> { @CreatedAt createdAt: Date @@ -156,8 +162,8 @@ export class VideoRedundancyModel extends Model { const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` logger.info('Removing duplicated video file %s.', logIdentifier) - videoFile.Video.removeFile(videoFile, true) - .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) + videoFile.Video.removeWebTorrentFile(videoFile, true) + .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) } if (instance.videoStreamingPlaylistId) { @@ -186,6 +192,57 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } + static async listLocalByVideoId (videoId: number): Promise { + const actor = await getServerActor() + + const queryStreamingPlaylist = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + required: true, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: { + id: videoId + } + } + ] + } + ] + } + + const queryFiles = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoFileModel, + required: true, + include: [ + { + model: VideoModel, + required: true, + where: { + id: videoId + } + } + ] + } + ] + } + + return Promise.all([ + VideoRedundancyModel.findAll(queryStreamingPlaylist), + VideoRedundancyModel.findAll(queryFiles) + ]).then(([ r1, r2 ]) => r1.concat(r2)) + } + static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise { const actor = await getServerActor() @@ -199,7 +256,7 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird { + static loadByIdWithVideo (id: number, transaction?: Transaction): Promise { const query = { where: { id }, transaction @@ -208,7 +265,7 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static loadByUrl (url: string, transaction?: Transaction): Bluebird { + static loadByUrl (url: string, transaction?: Transaction): Promise { const query = { where: { url @@ -251,7 +308,7 @@ export class VideoRedundancyModel extends Model { .then(r => !!r) } - static async getVideoSample (p: Bluebird) { + static async getVideoSample (p: Promise) { const rows = await p if (rows.length === 0) return undefined @@ -262,16 +319,19 @@ export class VideoRedundancyModel extends Model { } static async findMostViewToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'views' ], limit: randomizedFactor, order: getVideoSort('-views'), where: { - privacy: VideoPrivacy.PUBLIC + privacy: VideoPrivacy.PUBLIC, + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude() ] } @@ -280,6 +340,8 @@ export class VideoRedundancyModel extends Model { } static async findTrendingToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'views' ], @@ -288,10 +350,11 @@ export class VideoRedundancyModel extends Model { limit: randomizedFactor, order: getVideoSort('-trending'), where: { - privacy: VideoPrivacy.PUBLIC + privacy: VideoPrivacy.PUBLIC, + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude(), VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) @@ -302,6 +365,8 @@ export class VideoRedundancyModel extends Model { } static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'publishedAt' ], @@ -309,13 +374,20 @@ export class VideoRedundancyModel extends Model { order: getVideoSort('-publishedAt'), where: { privacy: VideoPrivacy.PUBLIC, + isLive: false, views: { [Op.gte]: minViews - } + }, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), - VideoRedundancyModel.buildServerRedundancyInclude() + VideoRedundancyModel.buildServerRedundancyInclude(), + + // Required by publishedAt sort + { + model: ScheduleVideoUpdateModel.unscoped(), + required: false + } ] } @@ -341,51 +413,7 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) } - static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { - const actor = await getServerActor() - const redundancyInclude = { - attributes: [], - model: VideoRedundancyModel, - required: true, - where: { - actorId: actor.id, - strategy - } - } - - const queryFiles: FindOptions = { - include: [ redundancyInclude ] - } - - const queryStreamingPlaylists: FindOptions = { - include: [ - { - attributes: [], - model: VideoModel.unscoped(), - required: true, - include: [ - { - required: true, - attributes: [], - model: VideoStreamingPlaylistModel.unscoped(), - include: [ - redundancyInclude - ] - } - ] - } - ] - } - - return Promise.all([ - VideoFileModel.aggregate('size', 'SUM', queryFiles), - VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists) - ]).then(([ r1, r2 ]) => { - return parseAggregateResult(r1) + parseAggregateResult(r2) - }) - } - - static async listLocalExpired () { + static async listLocalExpired (): Promise { const actor = await getServerActor() const query = { @@ -444,16 +472,34 @@ export class VideoRedundancyModel extends Model { const query = { where: { - actorId: actor.id + [Op.and]: [ + { + actorId: actor.id + }, + { + [Op.or]: [ + { + '$VideoStreamingPlaylist.id$': { + [Op.ne]: null + } + }, + { + '$VideoFile.id$': { + [Op.ne]: null + } + } + ] + } + ] }, include: [ { - model: VideoFileModel, + model: VideoFileModel.unscoped(), required: false, include: [ buildVideoInclude() ] }, { - model: VideoStreamingPlaylistModel, + model: VideoStreamingPlaylistModel.unscoped(), required: false, include: [ buildVideoInclude() ] } @@ -484,7 +530,7 @@ export class VideoRedundancyModel extends Model { } if (strategy) { - Object.assign(redundancyWhere, { strategy: strategy }) + Object.assign(redundancyWhere, { strategy }) } const videoFilterWhere = { @@ -571,32 +617,35 @@ export class VideoRedundancyModel extends Model { static async getStats (strategy: VideoRedundancyStrategyWithManual) { const actor = await getServerActor() - const query: FindOptions = { - raw: true, - attributes: [ - [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ], - [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ], - [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ] - ], - where: { - strategy, - actorId: actor.id - }, - include: [ - { - attributes: [], - model: VideoFileModel, - required: true - } - ] - } - - return VideoRedundancyModel.findOne(query) - .then((r: any) => ({ - totalUsed: parseAggregateResult(r.totalUsed), - totalVideos: r.totalVideos, - totalVideoFiles: r.totalVideoFiles - })) + const sql = `WITH "tmp" AS ` + + `(` + + `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` + + `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + + `FROM "videoRedundancy" AS "videoRedundancy" ` + + `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` + + `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` + + `LEFT JOIN "videoFile" AS "videoStreamingFile" ` + + `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` + + `), ` + + `"videoIds" AS (` + + `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` + + `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` + + `) ` + + `SELECT ` + + `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + + `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` + + `COUNT(*) AS "totalVideoFiles" ` + + `FROM "tmp"` + + return VideoRedundancyModel.sequelize.query(sql, { + replacements: { strategy, actorId: actor.id }, + type: QueryTypes.SELECT + }).then(([ row ]) => ({ + totalUsed: parseAggregateResult(row.totalUsed), + totalVideos: row.totalVideos, + totalVideoFiles: row.totalVideoFiles + })) } static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { @@ -647,9 +696,18 @@ export class VideoRedundancyModel extends Model { } getVideo () { - if (this.VideoFile) return this.VideoFile.Video + if (this.VideoFile?.Video) return this.VideoFile.Video + + if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video - return this.VideoStreamingPlaylist.Video + return undefined + } + + getVideoUUID () { + const video = this.getVideo() + if (!video) return undefined + + return video.uuid } isOwned () { @@ -688,23 +746,22 @@ export class VideoRedundancyModel extends Model { } // Don't include video files we already duplicated - private static async buildVideoFileForDuplication () { - const actor = await getServerActor() - + private static buildVideoIdsForDuplication (peertubeActor: MActor) { const notIn = literal( '(' + - `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + + `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + + `UNION ` + + `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + ')' ) return { - attributes: [], - model: VideoFileModel, - required: true, - where: { - id: { - [Op.notIn]: notIn - } + id: { + [Op.notIn]: notIn } } }