X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fredundancy%2Fvideo-redundancy.ts;h=ca56a57dc67bb1cba0c8ce2afc23401fd914a09c;hb=5ec3cbdf22fc88ebe57f370fc0bc0e3df7453458;hp=39c50be5c1de3a923a5f612619023860257a889d;hpb=6dd9de95dfa39bd5c1faed00d1dbd52cd112bae0;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 39c50be5c..ca56a57dc 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -1,3 +1,5 @@ +import { sample } from 'lodash' +import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize' import { AllowNull, BeforeDestroy, @@ -12,55 +14,61 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { ActorModel } from '../activitypub/actor' -import { getVideoSort, throwIfNotValid } from '../utils' +import { getServerActor } from '@server/models/application/application' +import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' +import { AttributesOnly } from '@shared/core-utils' +import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' +import { + FileRedundancyInformation, + StreamingPlaylistRedundancyInformation, + VideoRedundancy +} from '@shared/models/redundancy/video-redundancy.model' +import { CacheFileObject, VideoPrivacy } from '../../../shared' +import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy' +import { isTestInstance } from '../../helpers/core-utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers' -import { VideoFileModel } from '../video/video-file' -import { getServerActor } from '../../helpers/utils' -import { VideoModel } from '../video/video' -import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' -import { CacheFileObject, VideoPrivacy } from '../../../shared' -import { VideoChannelModel } from '../video/video-channel' +import { CONFIG } from '../../initializers/config' +import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' +import { ActorModel } from '../actor/actor' import { ServerModel } from '../server/server' -import { sample } from 'lodash' -import { isTestInstance } from '../../helpers/core-utils' -import * as Bluebird from 'bluebird' -import * as Sequelize from 'sequelize' +import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' +import { ScheduleVideoUpdateModel } from '../video/schedule-video-update' +import { VideoModel } from '../video/video' +import { VideoChannelModel } from '../video/video-channel' +import { VideoFileModel } from '../video/video-file' import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' -import { CONFIG } from '../../initializers/config' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' } -@Scopes({ - [ ScopeNames.WITH_VIDEO ]: { +@Scopes(() => ({ + [ScopeNames.WITH_VIDEO]: { include: [ { - model: () => VideoFileModel, + model: VideoFileModel, required: false, include: [ { - model: () => VideoModel, + model: VideoModel, required: true } ] }, { - model: () => VideoStreamingPlaylistModel, + model: VideoStreamingPlaylistModel, required: false, include: [ { - model: () => VideoModel, + model: VideoModel, required: true } ] } ] } -}) +})) @Table({ tableName: 'videoRedundancy', @@ -77,7 +85,7 @@ export enum ScopeNames { } ] }) -export class VideoRedundancyModel extends Model { +export class VideoRedundancyModel extends Model>> { @CreatedAt createdAt: Date @@ -85,7 +93,7 @@ export class VideoRedundancyModel extends Model { @UpdatedAt updatedAt: Date - @AllowNull(false) + @AllowNull(true) @Column expiresOn: Date @@ -159,14 +167,14 @@ export class VideoRedundancyModel extends Model { const videoUUID = videoStreamingPlaylist.Video.uuid logger.info('Removing duplicated video streaming playlist %s.', videoUUID) - videoStreamingPlaylist.Video.removeStreamingPlaylist(true) - .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) + videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true) + .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) } return undefined } - static async loadLocalByFileId (videoFileId: number) { + static async loadLocalByFileId (videoFileId: number): Promise { const actor = await getServerActor() const query = { @@ -179,7 +187,58 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) { + static async listLocalByVideoId (videoId: number): Promise { + const actor = await getServerActor() + + const queryStreamingPlaylist = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + required: true, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: { + id: videoId + } + } + ] + } + ] + } + + const queryFiles = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoFileModel, + required: true, + include: [ + { + model: VideoModel, + required: true, + where: { + id: videoId + } + } + ] + } + ] + } + + return Promise.all([ + VideoRedundancyModel.findAll(queryStreamingPlaylist), + VideoRedundancyModel.findAll(queryFiles) + ]).then(([ r1, r2 ]) => r1.concat(r2)) + } + + static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise { const actor = await getServerActor() const query = { @@ -192,7 +251,16 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static loadByUrl (url: string, transaction?: Sequelize.Transaction) { + static loadByIdWithVideo (id: number, transaction?: Transaction): Promise { + const query = { + where: { id }, + transaction + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + + static loadByUrl (url: string, transaction?: Transaction): Promise { const query = { where: { url @@ -214,12 +282,12 @@ export class VideoRedundancyModel extends Model { }, include: [ { - attributes: [ ], + attributes: [], model: VideoFileModel, required: true, include: [ { - attributes: [ ], + attributes: [], model: VideoModel, required: true, where: { @@ -232,11 +300,13 @@ export class VideoRedundancyModel extends Model { } return VideoRedundancyModel.findOne(query) - .then(r => !!r) + .then(r => !!r) } - static async getVideoSample (p: Bluebird) { + static async getVideoSample (p: Promise) { const rows = await p + if (rows.length === 0) return undefined + const ids = rows.map(r => r.id) const id = sample(ids) @@ -244,16 +314,19 @@ export class VideoRedundancyModel extends Model { } static async findMostViewToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'views' ], limit: randomizedFactor, order: getVideoSort('-views'), where: { - privacy: VideoPrivacy.PUBLIC + privacy: VideoPrivacy.PUBLIC, + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude() ] } @@ -262,6 +335,8 @@ export class VideoRedundancyModel extends Model { } static async findTrendingToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'views' ], @@ -270,10 +345,11 @@ export class VideoRedundancyModel extends Model { limit: randomizedFactor, order: getVideoSort('-trending'), where: { - privacy: VideoPrivacy.PUBLIC + privacy: VideoPrivacy.PUBLIC, + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude(), VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) @@ -284,6 +360,8 @@ export class VideoRedundancyModel extends Model { } static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + const peertubeActor = await getServerActor() + // On VideoModel! const query = { attributes: [ 'id', 'publishedAt' ], @@ -291,20 +369,27 @@ export class VideoRedundancyModel extends Model { order: getVideoSort('-publishedAt'), where: { privacy: VideoPrivacy.PUBLIC, + isLive: false, views: { - [ Sequelize.Op.gte ]: minViews - } + [Op.gte]: minViews + }, + ...this.buildVideoIdsForDuplication(peertubeActor) }, include: [ - await VideoRedundancyModel.buildVideoFileForDuplication(), - VideoRedundancyModel.buildServerRedundancyInclude() + VideoRedundancyModel.buildServerRedundancyInclude(), + + // Required by publishedAt sort + { + model: ScheduleVideoUpdateModel.unscoped(), + required: false + } ] } return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } - static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) { + static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise { const expiredDate = new Date() expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) @@ -315,7 +400,7 @@ export class VideoRedundancyModel extends Model { actorId: actor.id, strategy, createdAt: { - [ Sequelize.Op.lt ]: expiredDate + [Op.lt]: expiredDate } } } @@ -323,31 +408,6 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) } - static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { - const actor = await getServerActor() - - const options = { - include: [ - { - attributes: [], - model: VideoRedundancyModel, - required: true, - where: { - actorId: actor.id, - strategy - } - } - ] - } - - return VideoFileModel.sum('size', options as any) // FIXME: typings - .then(v => { - if (!v || isNaN(v)) return 0 - - return v - }) - } - static async listLocalExpired () { const actor = await getServerActor() @@ -355,7 +415,7 @@ export class VideoRedundancyModel extends Model { where: { actorId: actor.id, expiresOn: { - [ Sequelize.Op.lt ]: new Date() + [Op.lt]: new Date() } } } @@ -369,10 +429,11 @@ export class VideoRedundancyModel extends Model { const query = { where: { actorId: { - [Sequelize.Op.ne]: actor.id + [Op.ne]: actor.id }, expiresOn: { - [ Sequelize.Op.lt ]: new Date() + [Op.lt]: new Date(), + [Op.ne]: null } } } @@ -425,57 +486,213 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.findAll(query) } - static async getStats (strategy: VideoRedundancyStrategy) { - const actor = await getServerActor() + static listForApi (options: { + start: number + count: number + sort: string + target: VideoRedundanciesTarget + strategy?: string + }) { + const { start, count, sort, target, strategy } = options + const redundancyWhere: WhereOptions = {} + const videosWhere: WhereOptions = {} + let redundancySqlSuffix = '' + + if (target === 'my-videos') { + Object.assign(videosWhere, { remote: false }) + } else if (target === 'remote-videos') { + Object.assign(videosWhere, { remote: true }) + Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) + redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' + } - const query = { - raw: true, - attributes: [ - [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ], - [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ], - [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ] - ], - where: { - strategy, - actorId: actor.id - }, + if (strategy) { + Object.assign(redundancyWhere, { strategy: strategy }) + } + + const videoFilterWhere = { + [Op.and]: [ + { + [Op.or]: [ + { + id: { + [Op.in]: literal( + '(' + + 'SELECT "videoId" FROM "videoFile" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + + redundancySqlSuffix + + ')' + ) + } + }, + { + id: { + [Op.in]: literal( + '(' + + 'select "videoId" FROM "videoStreamingPlaylist" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + + redundancySqlSuffix + + ')' + ) + } + } + ] + }, + + videosWhere + ] + } + + // /!\ On video model /!\ + const findOptions = { + offset: start, + limit: count, + order: getSort(sort), include: [ { - attributes: [], + required: false, model: VideoFileModel, - required: true + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + } + ] + }, + { + required: false, + model: VideoStreamingPlaylistModel.unscoped(), + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + }, + { + model: VideoFileModel, + required: false + } + ] } - ] + ], + where: videoFilterWhere + } + + // /!\ On video model /!\ + const countOptions = { + where: videoFilterWhere } - return VideoRedundancyModel.findOne(query as any) // FIXME: typings - .then((r: any) => ({ - totalUsed: parseInt(r.totalUsed.toString(), 10), - totalVideos: r.totalVideos, - totalVideoFiles: r.totalVideoFiles - })) + return Promise.all([ + VideoModel.findAll(findOptions), + + VideoModel.count(countOptions) + ]).then(([ data, total ]) => ({ total, data })) + } + + static async getStats (strategy: VideoRedundancyStrategyWithManual) { + const actor = await getServerActor() + + const sql = `WITH "tmp" AS ` + + `(` + + `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` + + `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + + `FROM "videoRedundancy" AS "videoRedundancy" ` + + `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` + + `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` + + `LEFT JOIN "videoFile" AS "videoStreamingFile" ` + + `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` + + `), ` + + `"videoIds" AS (` + + `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` + + `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` + + `) ` + + `SELECT ` + + `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + + `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` + + `COUNT(*) AS "totalVideoFiles" ` + + `FROM "tmp"` + + return VideoRedundancyModel.sequelize.query(sql, { + replacements: { strategy, actorId: actor.id }, + type: QueryTypes.SELECT + }).then(([ row ]) => ({ + totalUsed: parseAggregateResult(row.totalUsed), + totalVideos: row.totalVideos, + totalVideoFiles: row.totalVideoFiles + })) + } + + static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { + const filesRedundancies: FileRedundancyInformation[] = [] + const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] + + for (const file of video.VideoFiles) { + for (const redundancy of file.RedundancyVideos) { + filesRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size: file.size + }) + } + } + + for (const playlist of video.VideoStreamingPlaylists) { + const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) + + for (const redundancy of playlist.RedundancyVideos) { + streamingPlaylistsRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size + }) + } + } + + return { + id: video.id, + name: video.name, + url: video.url, + uuid: video.uuid, + + redundancies: { + files: filesRedundancies, + streamingPlaylists: streamingPlaylistsRedundancies + } + } } getVideo () { - if (this.VideoFile) return this.VideoFile.Video + if (this.VideoFile?.Video) return this.VideoFile.Video + + if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video - return this.VideoStreamingPlaylist.Video + return undefined } isOwned () { return !!this.strategy } - toActivityPubObject (): CacheFileObject { + toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject { if (this.VideoStreamingPlaylist) { return { id: this.url, type: 'CacheFile' as 'CacheFile', object: this.VideoStreamingPlaylist.Video.url, - expires: this.expiresOn.toISOString(), + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', - mimeType: 'application/x-mpegURL', mediaType: 'application/x-mpegURL', href: this.fileUrl } @@ -486,11 +703,10 @@ export class VideoRedundancyModel extends Model { id: this.url, type: 'CacheFile' as 'CacheFile', object: this.VideoFile.Video.url, - expires: this.expiresOn.toISOString(), + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', - mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, - mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any, href: this.fileUrl, height: this.VideoFile.resolution, size: this.VideoFile.size, @@ -500,23 +716,22 @@ export class VideoRedundancyModel extends Model { } // Don't include video files we already duplicated - private static async buildVideoFileForDuplication () { - const actor = await getServerActor() - - const notIn = Sequelize.literal( + private static buildVideoIdsForDuplication (peertubeActor: MActor) { + const notIn = literal( '(' + - `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + + `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + + `UNION ` + + `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + ')' ) return { - attributes: [], - model: VideoFileModel.unscoped(), - required: true, - where: { - id: { - [ Sequelize.Op.notIn ]: notIn - } + id: { + [Op.notIn]: notIn } } }