From e5565833f62b97f62ea75eba5b479963ae78b873 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 24 Sep 2018 13:07:33 +0200 Subject: Improve redundancy: add 'min_lifetime' configuration --- server/models/activitypub/actor.ts | 31 ++++++++++++++ server/models/redundancy/video-redundancy.ts | 64 +++++++++++++++++++++------- server/models/video/video-file.ts | 6 +++ 3 files changed, 86 insertions(+), 15 deletions(-) (limited to 'server/models') diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index f8bb59323..12b83916e 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -37,6 +37,7 @@ import { ServerModel } from '../server/server' import { throwIfNotValid } from '../utils' import { VideoChannelModel } from '../video/video-channel' import { ActorFollowModel } from './actor-follow' +import { VideoModel } from '../video/video' enum ScopeNames { FULL = 'FULL' @@ -266,6 +267,36 @@ export class ActorModel extends Model { return ActorModel.unscoped().findById(id) } + static loadAccountActorByVideoId (videoId: number, transaction: Sequelize.Transaction) { + const query = { + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + required: true, + include: { + attributes: [ 'id' ], + model: VideoModel.unscoped(), + required: true, + where: { + id: videoId + } + } + } + ] + } + ], + transaction + } + + return ActorModel.unscoped().findOne(query as any) // FIXME: typings + } + static isActorUrlExist (url: string) { const query = { raw: true, diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index fb07287a8..970d2fe06 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -9,7 +9,6 @@ import { Is, Model, Scopes, - Sequelize, Table, UpdatedAt } from 'sequelize-typescript' @@ -28,6 +27,7 @@ import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' import * as Bluebird from 'bluebird' +import * as Sequelize from 'sequelize' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' @@ -116,11 +116,11 @@ export class VideoRedundancyModel extends Model { Actor: ActorModel @AfterDestroy - static removeFilesAndSendDelete (instance: VideoRedundancyModel) { + static removeFile (instance: VideoRedundancyModel) { // Not us if (!instance.strategy) return - logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) + logger.info('Removing duplicated video file %s-%s.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) return instance.VideoFile.Video.removeFile(instance.VideoFile) } @@ -135,11 +135,12 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static loadByUrl (url: string) { + static loadByUrl (url: string, transaction?: Sequelize.Transaction) { const query = { where: { url - } + }, + transaction } return VideoRedundancyModel.findOne(query) @@ -157,7 +158,6 @@ export class VideoRedundancyModel extends Model { // On VideoModel! const query = { attributes: [ 'id', 'views' ], - logging: !isTestInstance(), limit: randomizedFactor, order: getVideoSort('-views'), include: [ @@ -174,7 +174,6 @@ export class VideoRedundancyModel extends Model { const query = { attributes: [ 'id', 'views' ], subQuery: false, - logging: !isTestInstance(), group: 'VideoModel.id', limit: randomizedFactor, order: getVideoSort('-trending'), @@ -193,7 +192,6 @@ export class VideoRedundancyModel extends Model { // On VideoModel! const query = { attributes: [ 'id', 'publishedAt' ], - logging: !isTestInstance(), limit: randomizedFactor, order: getVideoSort('-publishedAt'), where: { @@ -210,11 +208,29 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } + static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) { + const expiredDate = new Date() + expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) + + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + strategy, + createdAt: { + [ Sequelize.Op.lt ]: expiredDate + } + } + } + + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) + } + static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { const actor = await getServerActor() const options = { - logging: !isTestInstance(), include: [ { attributes: [], @@ -228,21 +244,39 @@ export class VideoRedundancyModel extends Model { ] } - return VideoFileModel.sum('size', options) + return VideoFileModel.sum('size', options as any) // FIXME: typings } - static listAllExpired () { + static async listLocalExpired () { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + expiresOn: { + [ Sequelize.Op.lt ]: new Date() + } + } + } + + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) + } + + static async listRemoteExpired () { + const actor = await getServerActor() + const query = { - logging: !isTestInstance(), where: { + actorId: { + [Sequelize.Op.ne]: actor.id + }, expiresOn: { [ Sequelize.Op.lt ]: new Date() } } } - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) - .findAll(query) + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) } static async getStats (strategy: VideoRedundancyStrategy) { @@ -299,7 +333,7 @@ export class VideoRedundancyModel extends Model { const notIn = Sequelize.literal( '(' + - `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + ')' ) diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 0907ea569..0887a3738 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -106,4 +106,10 @@ export class VideoFileModel extends Model { return results.length === 1 }) } + + hasSameUniqueKeysThan (other: VideoFileModel) { + return this.fps === other.fps && + this.resolution === other.resolution && + this.videoId === other.videoId + } } -- cgit v1.2.3