14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
19 import { VideoFileModel } from '../video/video-file'
20 import { getServerActor } from '../../helpers/utils'
21 import { VideoModel } from '../video/video'
22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
23 import { logger } from '../../helpers/logger'
24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
25 import { VideoChannelModel } from '../video/video-channel'
26 import { ServerModel } from '../server/server'
27 import { sample } from 'lodash'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import * as Bluebird from 'bluebird'
30 import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
31 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
32 import { CONFIG } from '../../initializers/config'
33 import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
35 export enum ScopeNames {
36 WITH_VIDEO = 'WITH_VIDEO'
40 [ ScopeNames.WITH_VIDEO ]: {
43 model: VideoFileModel,
53 model: VideoStreamingPlaylistModel,
67 tableName: 'videoRedundancy',
70 fields: [ 'videoFileId' ]
81 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
94 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
95 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
99 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
100 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
105 strategy: string // Only used by us
107 @ForeignKey(() => VideoFileModel)
111 @BelongsTo(() => VideoFileModel, {
117 VideoFile: VideoFileModel
119 @ForeignKey(() => VideoStreamingPlaylistModel)
121 videoStreamingPlaylistId: number
123 @BelongsTo(() => VideoStreamingPlaylistModel, {
129 VideoStreamingPlaylist: VideoStreamingPlaylistModel
131 @ForeignKey(() => ActorModel)
135 @BelongsTo(() => ActorModel, {
144 static async removeFile (instance: VideoRedundancyModel) {
145 if (!instance.isOwned()) return
147 if (instance.videoFileId) {
148 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
150 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
151 logger.info('Removing duplicated video file %s.', logIdentifier)
153 videoFile.Video.removeFile(videoFile, true)
154 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
157 if (instance.videoStreamingPlaylistId) {
158 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
160 const videoUUID = videoStreamingPlaylist.Video.uuid
161 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
163 videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
164 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
170 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
171 const actor = await getServerActor()
180 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
183 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
184 const actor = await getServerActor()
189 videoStreamingPlaylistId
193 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
196 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
204 return VideoRedundancyModel.findOne(query)
207 static async isLocalByVideoUUIDExists (uuid: string) {
208 const actor = await getServerActor()
212 attributes: [ 'id' ],
219 model: VideoFileModel,
235 return VideoRedundancyModel.findOne(query)
239 static async getVideoSample (p: Bluebird<VideoModel[]>) {
241 if (rows.length === 0) return undefined
243 const ids = rows.map(r => r.id)
244 const id = sample(ids)
246 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
249 static async findMostViewToDuplicate (randomizedFactor: number) {
252 attributes: [ 'id', 'views' ],
253 limit: randomizedFactor,
254 order: getVideoSort('-views'),
256 privacy: VideoPrivacy.PUBLIC
259 await VideoRedundancyModel.buildVideoFileForDuplication(),
260 VideoRedundancyModel.buildServerRedundancyInclude()
264 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
267 static async findTrendingToDuplicate (randomizedFactor: number) {
270 attributes: [ 'id', 'views' ],
272 group: 'VideoModel.id',
273 limit: randomizedFactor,
274 order: getVideoSort('-trending'),
276 privacy: VideoPrivacy.PUBLIC
279 await VideoRedundancyModel.buildVideoFileForDuplication(),
280 VideoRedundancyModel.buildServerRedundancyInclude(),
282 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
286 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
289 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
292 attributes: [ 'id', 'publishedAt' ],
293 limit: randomizedFactor,
294 order: getVideoSort('-publishedAt'),
296 privacy: VideoPrivacy.PUBLIC,
302 await VideoRedundancyModel.buildVideoFileForDuplication(),
303 VideoRedundancyModel.buildServerRedundancyInclude()
307 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
310 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
311 const expiredDate = new Date()
312 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
314 const actor = await getServerActor()
321 [ Op.lt ]: expiredDate
326 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
329 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
330 const actor = await getServerActor()
331 const redundancyInclude = {
333 model: VideoRedundancyModel,
341 const queryFiles: FindOptions = {
342 include: [ redundancyInclude ]
345 const queryStreamingPlaylists: FindOptions = {
349 model: VideoModel.unscoped(),
355 model: VideoStreamingPlaylistModel.unscoped(),
366 VideoFileModel.aggregate('size', 'SUM', queryFiles),
367 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
368 ]).then(([ r1, r2 ]) => {
369 return parseAggregateResult(r1) + parseAggregateResult(r2)
373 static async listLocalExpired () {
374 const actor = await getServerActor()
380 [ Op.lt ]: new Date()
385 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
388 static async listRemoteExpired () {
389 const actor = await getServerActor()
397 [ Op.lt ]: new Date()
402 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
405 static async listLocalOfServer (serverId: number) {
406 const actor = await getServerActor()
407 const buildVideoInclude = () => ({
413 model: VideoChannelModel.unscoped(),
418 model: ActorModel.unscoped(),
435 model: VideoFileModel,
437 include: [ buildVideoInclude() ]
440 model: VideoStreamingPlaylistModel,
442 include: [ buildVideoInclude() ]
447 return VideoRedundancyModel.findAll(query)
450 static async getStats (strategy: VideoRedundancyStrategy) {
451 const actor = await getServerActor()
453 const query: FindOptions = {
456 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
457 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
458 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
467 model: VideoFileModel,
473 return VideoRedundancyModel.findOne(query)
475 totalUsed: parseAggregateResult(r.totalUsed),
476 totalVideos: r.totalVideos,
477 totalVideoFiles: r.totalVideoFiles
482 if (this.VideoFile) return this.VideoFile.Video
484 return this.VideoStreamingPlaylist.Video
488 return !!this.strategy
491 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
492 if (this.VideoStreamingPlaylist) {
495 type: 'CacheFile' as 'CacheFile',
496 object: this.VideoStreamingPlaylist.Video.url,
497 expires: this.expiresOn.toISOString(),
500 mimeType: 'application/x-mpegURL',
501 mediaType: 'application/x-mpegURL',
509 type: 'CacheFile' as 'CacheFile',
510 object: this.VideoFile.Video.url,
511 expires: this.expiresOn.toISOString(),
514 mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
515 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
517 height: this.VideoFile.resolution,
518 size: this.VideoFile.size,
519 fps: this.VideoFile.fps
524 // Don't include video files we already duplicated
525 private static async buildVideoFileForDuplication () {
526 const actor = await getServerActor()
528 const notIn = literal(
530 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
536 model: VideoFileModel.unscoped(),
546 private static buildServerRedundancyInclude () {
549 model: VideoChannelModel.unscoped(),
554 model: ActorModel.unscoped(),
559 model: ServerModel.unscoped(),
562 redundancyAllowed: true