X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fredundancy%2Fvideo-redundancy.ts;h=cbfc7f7fa085c5ef75e3049cd070c8de73f37348;hb=e27ff5da6ed7bc1f56f50f862b80fb0c7d8a6d98;hp=b13ade0f4328c7e4ab3f3a8f339d22bd98b8bf13;hpb=b36f41ca09e92ecb30d367d91d1089a23d10d585;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index b13ade0f4..cbfc7f7fa 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -1,6 +1,6 @@ import { - AfterDestroy, AllowNull, + BeforeDestroy, BelongsTo, Column, CreatedAt, @@ -9,7 +9,6 @@ import { Is, Model, Scopes, - Sequelize, Table, UpdatedAt } from 'sequelize-typescript' @@ -22,11 +21,13 @@ import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../video/video' import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' -import { CacheFileObject } from '../../../shared' +import { CacheFileObject, VideoPrivacy } from '../../../shared' import { VideoChannelModel } from '../video/video-channel' import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' +import * as Bluebird from 'bluebird' +import * as Sequelize from 'sequelize' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -114,19 +115,28 @@ export class VideoRedundancyModel extends Model { }) Actor: ActorModel - @AfterDestroy - static removeFilesAndSendDelete (instance: VideoRedundancyModel) { + @BeforeDestroy + static async removeFile (instance: VideoRedundancyModel) { // Not us if (!instance.strategy) return - logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) + const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) - return instance.VideoFile.Video.removeFile(instance.VideoFile) + const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` + logger.info('Removing duplicated video file %s.', logIdentifier) + + videoFile.Video.removeFile(videoFile) + .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) + + return undefined } - static loadByFileId (videoFileId: number) { + static async loadLocalByFileId (videoFileId: number) { + const actor = await getServerActor() + const query = { where: { + actorId: actor.id, videoFileId } } @@ -134,17 +144,51 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static loadByUrl (url: string) { + static loadByUrl (url: string, transaction?: Sequelize.Transaction) { const query = { where: { url - } + }, + transaction + } + + return VideoRedundancyModel.findOne(query) + } + + static async isLocalByVideoUUIDExists (uuid: string) { + const actor = await getServerActor() + + const query = { + raw: true, + attributes: [ 'id' ], + where: { + actorId: actor.id + }, + include: [ + { + attributes: [ ], + model: VideoFileModel, + required: true, + include: [ + { + attributes: [ ], + model: VideoModel, + required: true, + where: { + uuid + } + } + ] + } + ] } return VideoRedundancyModel.findOne(query) + .then(r => !!r) } - static getVideoSample (rows: { id: number }[]) { + static async getVideoSample (p: Bluebird) { + const rows = await p const ids = rows.map(r => r.id) const id = sample(ids) @@ -155,18 +199,18 @@ export class VideoRedundancyModel extends Model { // On VideoModel! const query = { attributes: [ 'id', 'views' ], - logging: !isTestInstance(), limit: randomizedFactor, order: getVideoSort('-views'), + where: { + privacy: VideoPrivacy.PUBLIC + }, include: [ await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude() ] } - const rows = await VideoModel.unscoped().findAll(query) - - return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } static async findTrendingToDuplicate (randomizedFactor: number) { @@ -174,10 +218,12 @@ export class VideoRedundancyModel extends Model { const query = { attributes: [ 'id', 'views' ], subQuery: false, - logging: !isTestInstance(), group: 'VideoModel.id', limit: randomizedFactor, order: getVideoSort('-trending'), + where: { + privacy: VideoPrivacy.PUBLIC + }, include: [ await VideoRedundancyModel.buildVideoFileForDuplication(), VideoRedundancyModel.buildServerRedundancyInclude(), @@ -186,38 +232,171 @@ export class VideoRedundancyModel extends Model { ] } - const rows = await VideoModel.unscoped().findAll(query) + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + // On VideoModel! + const query = { + attributes: [ 'id', 'publishedAt' ], + limit: randomizedFactor, + order: getVideoSort('-publishedAt'), + where: { + privacy: VideoPrivacy.PUBLIC, + views: { + [ Sequelize.Op.gte ]: minViews + } + }, + include: [ + await VideoRedundancyModel.buildVideoFileForDuplication(), + VideoRedundancyModel.buildServerRedundancyInclude() + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) { + const expiredDate = new Date() + expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) + + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + strategy, + createdAt: { + [ Sequelize.Op.lt ]: expiredDate + } + } + } + + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) + } + + static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const options = { + include: [ + { + attributes: [], + model: VideoRedundancyModel, + required: true, + where: { + actorId: actor.id, + strategy + } + } + ] + } - return VideoRedundancyModel.getVideoSample(rows as { id: number }[]) + return VideoFileModel.sum('size', options as any) // FIXME: typings } - static async getVideoFiles (strategy: VideoRedundancyStrategy) { + static async listLocalExpired () { const actor = await getServerActor() - const queryVideoFiles = { - logging: !isTestInstance(), + const query = { where: { actorId: actor.id, - strategy + expiresOn: { + [ Sequelize.Op.lt ]: new Date() + } } } - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) - .findAll(queryVideoFiles) + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) } - static listAllExpired () { + static async listRemoteExpired () { + const actor = await getServerActor() + const query = { - logging: !isTestInstance(), where: { + actorId: { + [Sequelize.Op.ne]: actor.id + }, expiresOn: { [ Sequelize.Op.lt ]: new Date() } } } - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) - .findAll(query) + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) + } + + static async listLocalOfServer (serverId: number) { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoFileModel, + required: true, + include: [ + { + model: VideoModel, + required: true, + include: [ + { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + where: { + serverId + } + } + ] + } + ] + } + ] + } + ] + } + + return VideoRedundancyModel.findAll(query) + } + + static async getStats (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const query = { + raw: true, + attributes: [ + [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ], + [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ], + [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ] + ], + where: { + strategy, + actorId: actor.id + }, + include: [ + { + attributes: [], + model: VideoFileModel, + required: true + } + ] + } + + return VideoRedundancyModel.find(query as any) // FIXME: typings + .then((r: any) => ({ + totalUsed: parseInt(r.totalUsed.toString(), 10), + totalVideos: r.totalVideos, + totalVideoFiles: r.totalVideoFiles + })) } toActivityPubObject (): CacheFileObject { @@ -229,6 +408,7 @@ export class VideoRedundancyModel extends Model { url: { type: 'Link', mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, + mediaType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, href: this.fileUrl, height: this.VideoFile.resolution, size: this.VideoFile.size, @@ -243,7 +423,7 @@ export class VideoRedundancyModel extends Model { const notIn = Sequelize.literal( '(' + - `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + ')' )