14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getVideoSort, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
19 import { VideoFileModel } from '../video/video-file'
20 import { getServerActor } from '../../helpers/utils'
21 import { VideoModel } from '../video/video'
22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
23 import { logger } from '../../helpers/logger'
24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
25 import { VideoChannelModel } from '../video/video-channel'
26 import { ServerModel } from '../server/server'
27 import { sample } from 'lodash'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import * as Bluebird from 'bluebird'
30 import * as Sequelize from 'sequelize'
32 export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO'
37 [ ScopeNames.WITH_VIDEO ]: {
40 model: () => VideoFileModel,
44 model: () => VideoModel,
54 tableName: 'videoRedundancy',
57 fields: [ 'videoFileId' ]
68 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
81 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
82 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
86 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
87 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
92 strategy: string // Only used by us
94 @ForeignKey(() => VideoFileModel)
98 @BelongsTo(() => VideoFileModel, {
104 VideoFile: VideoFileModel
106 @ForeignKey(() => ActorModel)
110 @BelongsTo(() => ActorModel, {
119 static async removeFile (instance: VideoRedundancyModel) {
121 if (!instance.strategy) return
123 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
125 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
126 logger.info('Removing duplicated video file %s.', logIdentifier)
128 videoFile.Video.removeFile(videoFile)
129 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
134 static async loadLocalByFileId (videoFileId: number) {
135 const actor = await getServerActor()
144 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
147 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
155 return VideoRedundancyModel.findOne(query)
158 static async isLocalByVideoUUIDExists (uuid: string) {
159 const actor = await getServerActor()
163 attributes: [ 'id' ],
170 model: VideoFileModel,
186 return VideoRedundancyModel.findOne(query)
190 static async getVideoSample (p: Bluebird<VideoModel[]>) {
192 const ids = rows.map(r => r.id)
193 const id = sample(ids)
195 return VideoModel.loadWithFile(id, undefined, !isTestInstance())
198 static async findMostViewToDuplicate (randomizedFactor: number) {
201 attributes: [ 'id', 'views' ],
202 limit: randomizedFactor,
203 order: getVideoSort('-views'),
205 privacy: VideoPrivacy.PUBLIC
208 await VideoRedundancyModel.buildVideoFileForDuplication(),
209 VideoRedundancyModel.buildServerRedundancyInclude()
213 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
216 static async findTrendingToDuplicate (randomizedFactor: number) {
219 attributes: [ 'id', 'views' ],
221 group: 'VideoModel.id',
222 limit: randomizedFactor,
223 order: getVideoSort('-trending'),
225 privacy: VideoPrivacy.PUBLIC
228 await VideoRedundancyModel.buildVideoFileForDuplication(),
229 VideoRedundancyModel.buildServerRedundancyInclude(),
231 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
235 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
238 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
241 attributes: [ 'id', 'publishedAt' ],
242 limit: randomizedFactor,
243 order: getVideoSort('-publishedAt'),
245 privacy: VideoPrivacy.PUBLIC,
247 [ Sequelize.Op.gte ]: minViews
251 await VideoRedundancyModel.buildVideoFileForDuplication(),
252 VideoRedundancyModel.buildServerRedundancyInclude()
256 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
259 static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
260 const expiredDate = new Date()
261 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
263 const actor = await getServerActor()
270 [ Sequelize.Op.lt ]: expiredDate
275 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
278 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
279 const actor = await getServerActor()
285 model: VideoRedundancyModel,
295 return VideoFileModel.sum('size', options as any) // FIXME: typings
298 static async listLocalExpired () {
299 const actor = await getServerActor()
305 [ Sequelize.Op.lt ]: new Date()
310 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
313 static async listRemoteExpired () {
314 const actor = await getServerActor()
319 [Sequelize.Op.ne]: actor.id
322 [ Sequelize.Op.lt ]: new Date()
327 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
330 static async listLocalOfServer (serverId: number) {
331 const actor = await getServerActor()
339 model: VideoFileModel,
348 model: VideoChannelModel.unscoped(),
353 model: ActorModel.unscoped(),
368 return VideoRedundancyModel.findAll(query)
371 static async getStats (strategy: VideoRedundancyStrategy) {
372 const actor = await getServerActor()
377 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
378 [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ],
379 [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ]
388 model: VideoFileModel,
394 return VideoRedundancyModel.find(query as any) // FIXME: typings
396 totalUsed: parseInt(r.totalUsed.toString(), 10),
397 totalVideos: r.totalVideos,
398 totalVideoFiles: r.totalVideoFiles
402 toActivityPubObject (): CacheFileObject {
405 type: 'CacheFile' as 'CacheFile',
406 object: this.VideoFile.Video.url,
407 expires: this.expiresOn.toISOString(),
410 mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
412 height: this.VideoFile.resolution,
413 size: this.VideoFile.size,
414 fps: this.VideoFile.fps
419 // Don't include video files we already duplicated
420 private static async buildVideoFileForDuplication () {
421 const actor = await getServerActor()
423 const notIn = Sequelize.literal(
425 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
431 model: VideoFileModel.unscoped(),
435 [ Sequelize.Op.notIn ]: notIn
441 private static buildServerRedundancyInclude () {
444 model: VideoChannelModel.unscoped(),
449 model: ActorModel.unscoped(),
454 model: ServerModel.unscoped(),
457 redundancyAllowed: true