import {
- AfterDestroy,
AllowNull,
+ BeforeDestroy,
BelongsTo,
Column,
CreatedAt,
Is,
Model,
Scopes,
- Sequelize,
Table,
UpdatedAt
} from 'sequelize-typescript'
import { ActorModel } from '../activitypub/actor'
-import { throwIfNotValid } from '../utils'
+import { getVideoSort, throwIfNotValid } from '../utils'
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { CONFIG, CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers'
import { VideoFileModel } from '../video/video-file'
-import { isDateValid } from '../../helpers/custom-validators/misc'
import { getServerActor } from '../../helpers/utils'
import { VideoModel } from '../video/video'
import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
import { logger } from '../../helpers/logger'
-import { CacheFileObject } from '../../../shared'
+import { CacheFileObject, VideoPrivacy } from '../../../shared'
import { VideoChannelModel } from '../video/video-channel'
import { ServerModel } from '../server/server'
import { sample } from 'lodash'
import { isTestInstance } from '../../helpers/core-utils'
+import * as Bluebird from 'bluebird'
+import * as Sequelize from 'sequelize'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
})
Actor: ActorModel
- @AfterDestroy
- static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
- // Not us
- if (!instance.strategy) return
+ @BeforeDestroy
+ static async removeFile (instance: VideoRedundancyModel) {
+ if (!instance.isOwned()) return
- logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
+ const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
- return instance.VideoFile.Video.removeFile(instance.VideoFile)
+ const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
+ logger.info('Removing duplicated video file %s.', logIdentifier)
+
+ videoFile.Video.removeFile(videoFile, true)
+ .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
+
+ return undefined
}
- static loadByFileId (videoFileId: number) {
+ static async loadLocalByFileId (videoFileId: number) {
+ const actor = await getServerActor()
+
const query = {
where: {
+ actorId: actor.id,
videoFileId
}
}
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}
- static loadByUrl (url: string) {
+ static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
const query = {
where: {
url
- }
+ },
+ transaction
}
return VideoRedundancyModel.findOne(query)
}
+ static async isLocalByVideoUUIDExists (uuid: string) {
+ const actor = await getServerActor()
+
+ const query = {
+ raw: true,
+ attributes: [ 'id' ],
+ where: {
+ actorId: actor.id
+ },
+ include: [
+ {
+ attributes: [ ],
+ model: VideoFileModel,
+ required: true,
+ include: [
+ {
+ attributes: [ ],
+ model: VideoModel,
+ required: true,
+ where: {
+ uuid
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ return VideoRedundancyModel.findOne(query)
+ .then(r => !!r)
+ }
+
+ static async getVideoSample (p: Bluebird<VideoModel[]>) {
+ const rows = await p
+ const ids = rows.map(r => r.id)
+ const id = sample(ids)
+
+ return VideoModel.loadWithFile(id, undefined, !isTestInstance())
+ }
+
static async findMostViewToDuplicate (randomizedFactor: number) {
// On VideoModel!
const query = {
- logging: !isTestInstance(),
+ attributes: [ 'id', 'views' ],
+ limit: randomizedFactor,
+ order: getVideoSort('-views'),
+ where: {
+ privacy: VideoPrivacy.PUBLIC
+ },
+ include: [
+ await VideoRedundancyModel.buildVideoFileForDuplication(),
+ VideoRedundancyModel.buildServerRedundancyInclude()
+ ]
+ }
+
+ return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
+ }
+
+ static async findTrendingToDuplicate (randomizedFactor: number) {
+ // On VideoModel!
+ const query = {
+ attributes: [ 'id', 'views' ],
+ subQuery: false,
+ group: 'VideoModel.id',
limit: randomizedFactor,
- order: [ [ 'views', 'DESC' ] ],
+ order: getVideoSort('-trending'),
+ where: {
+ privacy: VideoPrivacy.PUBLIC
+ },
+ include: [
+ await VideoRedundancyModel.buildVideoFileForDuplication(),
+ VideoRedundancyModel.buildServerRedundancyInclude(),
+
+ VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
+ ]
+ }
+
+ return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
+ }
+
+ static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
+ // On VideoModel!
+ const query = {
+ attributes: [ 'id', 'publishedAt' ],
+ limit: randomizedFactor,
+ order: getVideoSort('-publishedAt'),
+ where: {
+ privacy: VideoPrivacy.PUBLIC,
+ views: {
+ [ Sequelize.Op.gte ]: minViews
+ }
+ },
+ include: [
+ await VideoRedundancyModel.buildVideoFileForDuplication(),
+ VideoRedundancyModel.buildServerRedundancyInclude()
+ ]
+ }
+
+ return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
+ }
+
+ static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
+ const expiredDate = new Date()
+ expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
+
+ const actor = await getServerActor()
+
+ const query = {
+ where: {
+ actorId: actor.id,
+ strategy,
+ createdAt: {
+ [ Sequelize.Op.lt ]: expiredDate
+ }
+ }
+ }
+
+ return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
+ }
+
+ static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
+ const actor = await getServerActor()
+
+ const options = {
include: [
{
- model: VideoFileModel.unscoped(),
+ attributes: [],
+ model: VideoRedundancyModel,
required: true,
where: {
- id: {
- [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
- }
+ actorId: actor.id,
+ strategy
}
+ }
+ ]
+ }
+
+ return VideoFileModel.sum('size', options as any) // FIXME: typings
+ .then(v => {
+ if (!v || isNaN(v)) return 0
+
+ return v
+ })
+ }
+
+ static async listLocalExpired () {
+ const actor = await getServerActor()
+
+ const query = {
+ where: {
+ actorId: actor.id,
+ expiresOn: {
+ [ Sequelize.Op.lt ]: new Date()
+ }
+ }
+ }
+
+ return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
+ }
+
+ static async listRemoteExpired () {
+ const actor = await getServerActor()
+
+ const query = {
+ where: {
+ actorId: {
+ [Sequelize.Op.ne]: actor.id
},
+ expiresOn: {
+ [ Sequelize.Op.lt ]: new Date()
+ }
+ }
+ }
+
+ return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
+ }
+
+ static async listLocalOfServer (serverId: number) {
+ const actor = await getServerActor()
+
+ const query = {
+ where: {
+ actorId: actor.id
+ },
+ include: [
{
- attributes: [],
- model: VideoChannelModel.unscoped(),
+ model: VideoFileModel,
required: true,
include: [
{
- attributes: [],
- model: ActorModel.unscoped(),
+ model: VideoModel,
required: true,
include: [
{
attributes: [],
- model: ServerModel.unscoped(),
+ model: VideoChannelModel.unscoped(),
required: true,
- where: {
- redundancyAllowed: true
- }
+ include: [
+ {
+ attributes: [],
+ model: ActorModel.unscoped(),
+ required: true,
+ where: {
+ serverId
+ }
+ }
+ ]
}
]
}
]
}
- const rows = await VideoModel.unscoped().findAll(query)
-
- return sample(rows)
+ return VideoRedundancyModel.findAll(query)
}
- static async getVideoFiles (strategy: VideoRedundancyStrategy) {
+ static async getStats (strategy: VideoRedundancyStrategy) {
const actor = await getServerActor()
- const queryVideoFiles = {
- logging: !isTestInstance(),
- where: {
- actorId: actor.id,
- strategy
- }
- }
-
- return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
- .findAll(queryVideoFiles)
- }
-
- static listAllExpired () {
const query = {
- logging: !isTestInstance(),
+ raw: true,
+ attributes: [
+ [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
+ [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ],
+ [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ]
+ ],
where: {
- expiresOn: {
- [Sequelize.Op.lt]: new Date()
+ strategy,
+ actorId: actor.id
+ },
+ include: [
+ {
+ attributes: [],
+ model: VideoFileModel,
+ required: true
}
- }
+ ]
}
- return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
- .findAll(query)
+ return VideoRedundancyModel.findOne(query as any) // FIXME: typings
+ .then((r: any) => ({
+ totalUsed: parseInt(r.totalUsed.toString(), 10),
+ totalVideos: r.totalVideos,
+ totalVideoFiles: r.totalVideoFiles
+ }))
+ }
+
+ isOwned () {
+ return !!this.strategy
}
toActivityPubObject (): CacheFileObject {
expires: this.expiresOn.toISOString(),
url: {
type: 'Link',
- mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
+ mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
+ mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
href: this.fileUrl,
height: this.VideoFile.resolution,
size: this.VideoFile.size,
}
}
- private static async buildExcludeIn () {
+ // Don't include video files we already duplicated
+ private static async buildVideoFileForDuplication () {
const actor = await getServerActor()
- return Sequelize.literal(
+ const notIn = Sequelize.literal(
'(' +
- `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
+ `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
')'
)
+
+ return {
+ attributes: [],
+ model: VideoFileModel.unscoped(),
+ required: true,
+ where: {
+ id: {
+ [ Sequelize.Op.notIn ]: notIn
+ }
+ }
+ }
+ }
+
+ private static buildServerRedundancyInclude () {
+ return {
+ attributes: [],
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ include: [
+ {
+ attributes: [],
+ model: ActorModel.unscoped(),
+ required: true,
+ include: [
+ {
+ attributes: [],
+ model: ServerModel.unscoped(),
+ required: true,
+ where: {
+ redundancyAllowed: true
+ }
+ }
+ ]
+ }
+ ]
+ }
}
}
projects / github / Chocobozzz / PeerTube.git / blobdiff