14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
19 import { VideoFileModel } from '../video/video-file'
20 import { getServerActor } from '../../helpers/utils'
21 import { VideoModel } from '../video/video'
22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
23 import { logger } from '../../helpers/logger'
24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
25 import { VideoChannelModel } from '../video/video-channel'
26 import { ServerModel } from '../server/server'
27 import { sample } from 'lodash'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import * as Bluebird from 'bluebird'
30 import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
31 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
32 import { CONFIG } from '../../initializers/config'
34 export enum ScopeNames {
35 WITH_VIDEO = 'WITH_VIDEO'
39 [ ScopeNames.WITH_VIDEO ]: {
42 model: VideoFileModel,
52 model: VideoStreamingPlaylistModel,
66 tableName: 'videoRedundancy',
69 fields: [ 'videoFileId' ]
80 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
93 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
94 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
98 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
99 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
104 strategy: string // Only used by us
106 @ForeignKey(() => VideoFileModel)
110 @BelongsTo(() => VideoFileModel, {
116 VideoFile: VideoFileModel
118 @ForeignKey(() => VideoStreamingPlaylistModel)
120 videoStreamingPlaylistId: number
122 @BelongsTo(() => VideoStreamingPlaylistModel, {
128 VideoStreamingPlaylist: VideoStreamingPlaylistModel
130 @ForeignKey(() => ActorModel)
134 @BelongsTo(() => ActorModel, {
143 static async removeFile (instance: VideoRedundancyModel) {
144 if (!instance.isOwned()) return
146 if (instance.videoFileId) {
147 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
149 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
150 logger.info('Removing duplicated video file %s.', logIdentifier)
152 videoFile.Video.removeFile(videoFile, true)
153 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
156 if (instance.videoStreamingPlaylistId) {
157 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
159 const videoUUID = videoStreamingPlaylist.Video.uuid
160 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
162 videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
163 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
169 static async loadLocalByFileId (videoFileId: number) {
170 const actor = await getServerActor()
179 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
182 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
183 const actor = await getServerActor()
188 videoStreamingPlaylistId
192 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
195 static loadByUrl (url: string, transaction?: Transaction) {
203 return VideoRedundancyModel.findOne(query)
206 static async isLocalByVideoUUIDExists (uuid: string) {
207 const actor = await getServerActor()
211 attributes: [ 'id' ],
218 model: VideoFileModel,
234 return VideoRedundancyModel.findOne(query)
238 static async getVideoSample (p: Bluebird<VideoModel[]>) {
240 if (rows.length === 0) return undefined
242 const ids = rows.map(r => r.id)
243 const id = sample(ids)
245 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
248 static async findMostViewToDuplicate (randomizedFactor: number) {
251 attributes: [ 'id', 'views' ],
252 limit: randomizedFactor,
253 order: getVideoSort('-views'),
255 privacy: VideoPrivacy.PUBLIC
258 await VideoRedundancyModel.buildVideoFileForDuplication(),
259 VideoRedundancyModel.buildServerRedundancyInclude()
263 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
266 static async findTrendingToDuplicate (randomizedFactor: number) {
269 attributes: [ 'id', 'views' ],
271 group: 'VideoModel.id',
272 limit: randomizedFactor,
273 order: getVideoSort('-trending'),
275 privacy: VideoPrivacy.PUBLIC
278 await VideoRedundancyModel.buildVideoFileForDuplication(),
279 VideoRedundancyModel.buildServerRedundancyInclude(),
281 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
285 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
288 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
291 attributes: [ 'id', 'publishedAt' ],
292 limit: randomizedFactor,
293 order: getVideoSort('-publishedAt'),
295 privacy: VideoPrivacy.PUBLIC,
301 await VideoRedundancyModel.buildVideoFileForDuplication(),
302 VideoRedundancyModel.buildServerRedundancyInclude()
306 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
309 static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
310 const expiredDate = new Date()
311 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
313 const actor = await getServerActor()
320 [ Op.lt ]: expiredDate
325 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
328 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
329 const actor = await getServerActor()
330 const redundancyInclude = {
332 model: VideoRedundancyModel,
340 const queryFiles: FindOptions = {
341 include: [ redundancyInclude ]
344 const queryStreamingPlaylists: FindOptions = {
348 model: VideoModel.unscoped(),
354 model: VideoStreamingPlaylistModel.unscoped(),
365 VideoFileModel.aggregate('size', 'SUM', queryFiles),
366 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
367 ]).then(([ r1, r2 ]) => {
368 return parseAggregateResult(r1) + parseAggregateResult(r2)
372 static async listLocalExpired () {
373 const actor = await getServerActor()
379 [ Op.lt ]: new Date()
384 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
387 static async listRemoteExpired () {
388 const actor = await getServerActor()
396 [ Op.lt ]: new Date()
401 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
404 static async listLocalOfServer (serverId: number) {
405 const actor = await getServerActor()
406 const buildVideoInclude = () => ({
412 model: VideoChannelModel.unscoped(),
417 model: ActorModel.unscoped(),
434 model: VideoFileModel,
436 include: [ buildVideoInclude() ]
439 model: VideoStreamingPlaylistModel,
441 include: [ buildVideoInclude() ]
446 return VideoRedundancyModel.findAll(query)
449 static async getStats (strategy: VideoRedundancyStrategy) {
450 const actor = await getServerActor()
452 const query: FindOptions = {
455 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
456 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
457 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
466 model: VideoFileModel,
472 return VideoRedundancyModel.findOne(query)
474 totalUsed: parseAggregateResult(r.totalUsed),
475 totalVideos: r.totalVideos,
476 totalVideoFiles: r.totalVideoFiles
481 if (this.VideoFile) return this.VideoFile.Video
483 return this.VideoStreamingPlaylist.Video
487 return !!this.strategy
490 toActivityPubObject (): CacheFileObject {
491 if (this.VideoStreamingPlaylist) {
494 type: 'CacheFile' as 'CacheFile',
495 object: this.VideoStreamingPlaylist.Video.url,
496 expires: this.expiresOn.toISOString(),
499 mimeType: 'application/x-mpegURL',
500 mediaType: 'application/x-mpegURL',
508 type: 'CacheFile' as 'CacheFile',
509 object: this.VideoFile.Video.url,
510 expires: this.expiresOn.toISOString(),
513 mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
514 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
516 height: this.VideoFile.resolution,
517 size: this.VideoFile.size,
518 fps: this.VideoFile.fps
523 // Don't include video files we already duplicated
524 private static async buildVideoFileForDuplication () {
525 const actor = await getServerActor()
527 const notIn = literal(
529 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
535 model: VideoFileModel.unscoped(),
545 private static buildServerRedundancyInclude () {
548 model: VideoChannelModel.unscoped(),
553 model: ActorModel.unscoped(),
558 model: ServerModel.unscoped(),
561 redundancyAllowed: true