14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
19 import { VideoFileModel } from '../video/video-file'
20 import { VideoModel } from '../video/video'
21 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
22 import { logger } from '../../helpers/logger'
23 import { CacheFileObject, VideoPrivacy } from '../../../shared'
24 import { VideoChannelModel } from '../video/video-channel'
25 import { ServerModel } from '../server/server'
26 import { sample } from 'lodash'
27 import { isTestInstance } from '../../helpers/core-utils'
28 import * as Bluebird from 'bluebird'
29 import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
30 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
31 import { CONFIG } from '../../initializers/config'
32 import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
33 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
35 FileRedundancyInformation,
36 StreamingPlaylistRedundancyInformation,
38 } from '@shared/models/redundancy/video-redundancy.model'
39 import { getServerActor } from '@server/models/application/application'
41 export enum ScopeNames {
42 WITH_VIDEO = 'WITH_VIDEO'
46 [ScopeNames.WITH_VIDEO]: {
49 model: VideoFileModel,
59 model: VideoStreamingPlaylistModel,
73 tableName: 'videoRedundancy',
76 fields: [ 'videoFileId' ]
87 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
100 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
101 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
105 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
106 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
111 strategy: string // Only used by us
113 @ForeignKey(() => VideoFileModel)
117 @BelongsTo(() => VideoFileModel, {
123 VideoFile: VideoFileModel
125 @ForeignKey(() => VideoStreamingPlaylistModel)
127 videoStreamingPlaylistId: number
129 @BelongsTo(() => VideoStreamingPlaylistModel, {
135 VideoStreamingPlaylist: VideoStreamingPlaylistModel
137 @ForeignKey(() => ActorModel)
141 @BelongsTo(() => ActorModel, {
150 static async removeFile (instance: VideoRedundancyModel) {
151 if (!instance.isOwned()) return
153 if (instance.videoFileId) {
154 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
156 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
157 logger.info('Removing duplicated video file %s.', logIdentifier)
159 videoFile.Video.removeFile(videoFile, true)
160 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
163 if (instance.videoStreamingPlaylistId) {
164 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
166 const videoUUID = videoStreamingPlaylist.Video.uuid
167 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
169 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
170 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
176 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
177 const actor = await getServerActor()
186 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
189 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
190 const actor = await getServerActor()
195 videoStreamingPlaylistId
199 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
202 static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
208 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
211 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
219 return VideoRedundancyModel.findOne(query)
222 static async isLocalByVideoUUIDExists (uuid: string) {
223 const actor = await getServerActor()
227 attributes: [ 'id' ],
234 model: VideoFileModel,
250 return VideoRedundancyModel.findOne(query)
254 static async getVideoSample (p: Bluebird<VideoModel[]>) {
256 if (rows.length === 0) return undefined
258 const ids = rows.map(r => r.id)
259 const id = sample(ids)
261 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
264 static async findMostViewToDuplicate (randomizedFactor: number) {
267 attributes: [ 'id', 'views' ],
268 limit: randomizedFactor,
269 order: getVideoSort('-views'),
271 privacy: VideoPrivacy.PUBLIC
274 await VideoRedundancyModel.buildVideoFileForDuplication(),
275 VideoRedundancyModel.buildServerRedundancyInclude()
279 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
282 static async findTrendingToDuplicate (randomizedFactor: number) {
285 attributes: [ 'id', 'views' ],
287 group: 'VideoModel.id',
288 limit: randomizedFactor,
289 order: getVideoSort('-trending'),
291 privacy: VideoPrivacy.PUBLIC
294 await VideoRedundancyModel.buildVideoFileForDuplication(),
295 VideoRedundancyModel.buildServerRedundancyInclude(),
297 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
301 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
304 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
307 attributes: [ 'id', 'publishedAt' ],
308 limit: randomizedFactor,
309 order: getVideoSort('-publishedAt'),
311 privacy: VideoPrivacy.PUBLIC,
317 await VideoRedundancyModel.buildVideoFileForDuplication(),
318 VideoRedundancyModel.buildServerRedundancyInclude()
322 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
325 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
326 const expiredDate = new Date()
327 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
329 const actor = await getServerActor()
341 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
344 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
345 const actor = await getServerActor()
346 const redundancyInclude = {
348 model: VideoRedundancyModel,
356 const queryFiles: FindOptions = {
357 include: [ redundancyInclude ]
360 const queryStreamingPlaylists: FindOptions = {
364 model: VideoModel.unscoped(),
370 model: VideoStreamingPlaylistModel.unscoped(),
381 VideoFileModel.aggregate('size', 'SUM', queryFiles),
382 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
383 ]).then(([ r1, r2 ]) => {
384 return parseAggregateResult(r1) + parseAggregateResult(r2)
388 static async listLocalExpired () {
389 const actor = await getServerActor()
400 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
403 static async listRemoteExpired () {
404 const actor = await getServerActor()
418 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
421 static async listLocalOfServer (serverId: number) {
422 const actor = await getServerActor()
423 const buildVideoInclude = () => ({
429 model: VideoChannelModel.unscoped(),
434 model: ActorModel.unscoped(),
451 model: VideoFileModel,
453 include: [ buildVideoInclude() ]
456 model: VideoStreamingPlaylistModel,
458 include: [ buildVideoInclude() ]
463 return VideoRedundancyModel.findAll(query)
466 static listForApi (options: {
470 target: VideoRedundanciesTarget
473 const { start, count, sort, target, strategy } = options
474 const redundancyWhere: WhereOptions = {}
475 const videosWhere: WhereOptions = {}
476 let redundancySqlSuffix = ''
478 if (target === 'my-videos') {
479 Object.assign(videosWhere, { remote: false })
480 } else if (target === 'remote-videos') {
481 Object.assign(videosWhere, { remote: true })
482 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
483 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
487 Object.assign(redundancyWhere, { strategy: strategy })
490 const videoFilterWhere = {
498 'SELECT "videoId" FROM "videoFile" ' +
499 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
500 redundancySqlSuffix +
509 'select "videoId" FROM "videoStreamingPlaylist" ' +
510 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
511 redundancySqlSuffix +
523 // /!\ On video model /!\
524 const findOptions = {
527 order: getSort(sort),
531 model: VideoFileModel,
534 model: VideoRedundancyModel.unscoped(),
536 where: redundancyWhere
542 model: VideoStreamingPlaylistModel.unscoped(),
545 model: VideoRedundancyModel.unscoped(),
547 where: redundancyWhere
550 model: VideoFileModel,
556 where: videoFilterWhere
559 // /!\ On video model /!\
560 const countOptions = {
561 where: videoFilterWhere
565 VideoModel.findAll(findOptions),
567 VideoModel.count(countOptions)
568 ]).then(([ data, total ]) => ({ total, data }))
571 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
572 const actor = await getServerActor()
574 const query: FindOptions = {
577 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
578 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
579 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
588 model: VideoFileModel,
594 return VideoRedundancyModel.findOne(query)
596 totalUsed: parseAggregateResult(r.totalUsed),
597 totalVideos: r.totalVideos,
598 totalVideoFiles: r.totalVideoFiles
602 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
603 const filesRedundancies: FileRedundancyInformation[] = []
604 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
606 for (const file of video.VideoFiles) {
607 for (const redundancy of file.RedundancyVideos) {
608 filesRedundancies.push({
610 fileUrl: redundancy.fileUrl,
611 strategy: redundancy.strategy,
612 createdAt: redundancy.createdAt,
613 updatedAt: redundancy.updatedAt,
614 expiresOn: redundancy.expiresOn,
620 for (const playlist of video.VideoStreamingPlaylists) {
621 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
623 for (const redundancy of playlist.RedundancyVideos) {
624 streamingPlaylistsRedundancies.push({
626 fileUrl: redundancy.fileUrl,
627 strategy: redundancy.strategy,
628 createdAt: redundancy.createdAt,
629 updatedAt: redundancy.updatedAt,
630 expiresOn: redundancy.expiresOn,
643 files: filesRedundancies,
644 streamingPlaylists: streamingPlaylistsRedundancies
650 if (this.VideoFile) return this.VideoFile.Video
652 return this.VideoStreamingPlaylist.Video
656 return !!this.strategy
659 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
660 if (this.VideoStreamingPlaylist) {
663 type: 'CacheFile' as 'CacheFile',
664 object: this.VideoStreamingPlaylist.Video.url,
665 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
668 mediaType: 'application/x-mpegURL',
676 type: 'CacheFile' as 'CacheFile',
677 object: this.VideoFile.Video.url,
678 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
681 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
683 height: this.VideoFile.resolution,
684 size: this.VideoFile.size,
685 fps: this.VideoFile.fps
690 // Don't include video files we already duplicated
691 private static async buildVideoFileForDuplication () {
692 const actor = await getServerActor()
694 const notIn = literal(
696 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
702 model: VideoFileModel,
712 private static buildServerRedundancyInclude () {
715 model: VideoChannelModel.unscoped(),
720 model: ActorModel.unscoped(),
725 model: ServerModel.unscoped(),
728 redundancyAllowed: true