14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
19 import { VideoFileModel } from '../video/video-file'
20 import { VideoModel } from '../video/video'
21 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
22 import { logger } from '../../helpers/logger'
23 import { CacheFileObject, VideoPrivacy } from '../../../shared'
24 import { VideoChannelModel } from '../video/video-channel'
25 import { ServerModel } from '../server/server'
26 import { sample } from 'lodash'
27 import { isTestInstance } from '../../helpers/core-utils'
28 import * as Bluebird from 'bluebird'
29 import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
30 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
31 import { CONFIG } from '../../initializers/config'
32 import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
33 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
35 FileRedundancyInformation,
36 StreamingPlaylistRedundancyInformation,
38 } from '@shared/models/redundancy/video-redundancy.model'
39 import { getServerActor } from '@server/models/application/application'
41 export enum ScopeNames {
42 WITH_VIDEO = 'WITH_VIDEO'
46 [ScopeNames.WITH_VIDEO]: {
49 model: VideoFileModel,
59 model: VideoStreamingPlaylistModel,
73 tableName: 'videoRedundancy',
76 fields: [ 'videoFileId' ]
87 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
100 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
101 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
105 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
106 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
111 strategy: string // Only used by us
113 @ForeignKey(() => VideoFileModel)
117 @BelongsTo(() => VideoFileModel, {
123 VideoFile: VideoFileModel
125 @ForeignKey(() => VideoStreamingPlaylistModel)
127 videoStreamingPlaylistId: number
129 @BelongsTo(() => VideoStreamingPlaylistModel, {
135 VideoStreamingPlaylist: VideoStreamingPlaylistModel
137 @ForeignKey(() => ActorModel)
141 @BelongsTo(() => ActorModel, {
150 static async removeFile (instance: VideoRedundancyModel) {
151 if (!instance.isOwned()) return
153 if (instance.videoFileId) {
154 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
156 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
157 logger.info('Removing duplicated video file %s.', logIdentifier)
159 videoFile.Video.removeFile(videoFile, true)
160 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
163 if (instance.videoStreamingPlaylistId) {
164 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
166 const videoUUID = videoStreamingPlaylist.Video.uuid
167 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
169 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
170 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
176 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
177 const actor = await getServerActor()
186 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
189 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
190 const actor = await getServerActor()
195 videoStreamingPlaylistId
199 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
202 static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
208 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
211 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
219 return VideoRedundancyModel.findOne(query)
222 static async isLocalByVideoUUIDExists (uuid: string) {
223 const actor = await getServerActor()
227 attributes: [ 'id' ],
234 model: VideoFileModel,
250 return VideoRedundancyModel.findOne(query)
254 static async getVideoSample (p: Bluebird<VideoModel[]>) {
256 if (rows.length === 0) return undefined
258 const ids = rows.map(r => r.id)
259 const id = sample(ids)
261 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
264 static async findMostViewToDuplicate (randomizedFactor: number) {
267 attributes: [ 'id', 'views' ],
268 limit: randomizedFactor,
269 order: getVideoSort('-views'),
271 privacy: VideoPrivacy.PUBLIC,
275 await VideoRedundancyModel.buildVideoFileForDuplication(),
276 VideoRedundancyModel.buildServerRedundancyInclude()
280 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
283 static async findTrendingToDuplicate (randomizedFactor: number) {
286 attributes: [ 'id', 'views' ],
288 group: 'VideoModel.id',
289 limit: randomizedFactor,
290 order: getVideoSort('-trending'),
292 privacy: VideoPrivacy.PUBLIC,
296 await VideoRedundancyModel.buildVideoFileForDuplication(),
297 VideoRedundancyModel.buildServerRedundancyInclude(),
299 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
303 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
306 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
309 attributes: [ 'id', 'publishedAt' ],
310 limit: randomizedFactor,
311 order: getVideoSort('-publishedAt'),
313 privacy: VideoPrivacy.PUBLIC,
320 await VideoRedundancyModel.buildVideoFileForDuplication(),
321 VideoRedundancyModel.buildServerRedundancyInclude()
325 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
328 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
329 const expiredDate = new Date()
330 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
332 const actor = await getServerActor()
344 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
347 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
348 const actor = await getServerActor()
349 const redundancyInclude = {
351 model: VideoRedundancyModel,
359 const queryFiles: FindOptions = {
360 include: [ redundancyInclude ]
363 const queryStreamingPlaylists: FindOptions = {
367 model: VideoModel.unscoped(),
373 model: VideoStreamingPlaylistModel.unscoped(),
384 VideoFileModel.aggregate('size', 'SUM', queryFiles),
385 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
386 ]).then(([ r1, r2 ]) => {
387 return parseAggregateResult(r1) + parseAggregateResult(r2)
391 static async listLocalExpired () {
392 const actor = await getServerActor()
403 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
406 static async listRemoteExpired () {
407 const actor = await getServerActor()
421 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
424 static async listLocalOfServer (serverId: number) {
425 const actor = await getServerActor()
426 const buildVideoInclude = () => ({
432 model: VideoChannelModel.unscoped(),
437 model: ActorModel.unscoped(),
454 model: VideoFileModel,
456 include: [ buildVideoInclude() ]
459 model: VideoStreamingPlaylistModel,
461 include: [ buildVideoInclude() ]
466 return VideoRedundancyModel.findAll(query)
469 static listForApi (options: {
473 target: VideoRedundanciesTarget
476 const { start, count, sort, target, strategy } = options
477 const redundancyWhere: WhereOptions = {}
478 const videosWhere: WhereOptions = {}
479 let redundancySqlSuffix = ''
481 if (target === 'my-videos') {
482 Object.assign(videosWhere, { remote: false })
483 } else if (target === 'remote-videos') {
484 Object.assign(videosWhere, { remote: true })
485 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
486 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
490 Object.assign(redundancyWhere, { strategy: strategy })
493 const videoFilterWhere = {
501 'SELECT "videoId" FROM "videoFile" ' +
502 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
503 redundancySqlSuffix +
512 'select "videoId" FROM "videoStreamingPlaylist" ' +
513 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
514 redundancySqlSuffix +
526 // /!\ On video model /!\
527 const findOptions = {
530 order: getSort(sort),
534 model: VideoFileModel,
537 model: VideoRedundancyModel.unscoped(),
539 where: redundancyWhere
545 model: VideoStreamingPlaylistModel.unscoped(),
548 model: VideoRedundancyModel.unscoped(),
550 where: redundancyWhere
553 model: VideoFileModel,
559 where: videoFilterWhere
562 // /!\ On video model /!\
563 const countOptions = {
564 where: videoFilterWhere
568 VideoModel.findAll(findOptions),
570 VideoModel.count(countOptions)
571 ]).then(([ data, total ]) => ({ total, data }))
574 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
575 const actor = await getServerActor()
577 const query: FindOptions = {
580 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
581 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
582 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
591 model: VideoFileModel,
597 return VideoRedundancyModel.findOne(query)
599 totalUsed: parseAggregateResult(r.totalUsed),
600 totalVideos: r.totalVideos,
601 totalVideoFiles: r.totalVideoFiles
605 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
606 const filesRedundancies: FileRedundancyInformation[] = []
607 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
609 for (const file of video.VideoFiles) {
610 for (const redundancy of file.RedundancyVideos) {
611 filesRedundancies.push({
613 fileUrl: redundancy.fileUrl,
614 strategy: redundancy.strategy,
615 createdAt: redundancy.createdAt,
616 updatedAt: redundancy.updatedAt,
617 expiresOn: redundancy.expiresOn,
623 for (const playlist of video.VideoStreamingPlaylists) {
624 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
626 for (const redundancy of playlist.RedundancyVideos) {
627 streamingPlaylistsRedundancies.push({
629 fileUrl: redundancy.fileUrl,
630 strategy: redundancy.strategy,
631 createdAt: redundancy.createdAt,
632 updatedAt: redundancy.updatedAt,
633 expiresOn: redundancy.expiresOn,
646 files: filesRedundancies,
647 streamingPlaylists: streamingPlaylistsRedundancies
653 if (this.VideoFile) return this.VideoFile.Video
655 return this.VideoStreamingPlaylist.Video
659 return !!this.strategy
662 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
663 if (this.VideoStreamingPlaylist) {
666 type: 'CacheFile' as 'CacheFile',
667 object: this.VideoStreamingPlaylist.Video.url,
668 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
671 mediaType: 'application/x-mpegURL',
679 type: 'CacheFile' as 'CacheFile',
680 object: this.VideoFile.Video.url,
681 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
684 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
686 height: this.VideoFile.resolution,
687 size: this.VideoFile.size,
688 fps: this.VideoFile.fps
693 // Don't include video files we already duplicated
694 private static async buildVideoFileForDuplication () {
695 const actor = await getServerActor()
697 const notIn = literal(
699 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
705 model: VideoFileModel,
715 private static buildServerRedundancyInclude () {
718 model: VideoChannelModel.unscoped(),
723 model: ActorModel.unscoped(),
728 model: ServerModel.unscoped(),
731 redundancyAllowed: true