1 import { sample } from 'lodash'
2 import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
21 FileRedundancyInformation,
22 StreamingPlaylistRedundancyInformation,
24 } from '@shared/models/redundancy/video-redundancy.model'
25 import { CacheFileObject, VideoPrivacy } from '../../../shared'
26 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
27 import { isTestInstance } from '../../helpers/core-utils'
28 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
29 import { logger } from '../../helpers/logger'
30 import { CONFIG } from '../../initializers/config'
31 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
32 import { ActorModel } from '../activitypub/actor'
33 import { ServerModel } from '../server/server'
34 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
35 import { VideoModel } from '../video/video'
36 import { VideoChannelModel } from '../video/video-channel'
37 import { VideoFileModel } from '../video/video-file'
38 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
40 export enum ScopeNames {
41 WITH_VIDEO = 'WITH_VIDEO'
45 [ScopeNames.WITH_VIDEO]: {
48 model: VideoFileModel,
58 model: VideoStreamingPlaylistModel,
72 tableName: 'videoRedundancy',
75 fields: [ 'videoFileId' ]
86 export class VideoRedundancyModel extends Model {
99 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
100 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
104 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
105 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
110 strategy: string // Only used by us
112 @ForeignKey(() => VideoFileModel)
116 @BelongsTo(() => VideoFileModel, {
122 VideoFile: VideoFileModel
124 @ForeignKey(() => VideoStreamingPlaylistModel)
126 videoStreamingPlaylistId: number
128 @BelongsTo(() => VideoStreamingPlaylistModel, {
134 VideoStreamingPlaylist: VideoStreamingPlaylistModel
136 @ForeignKey(() => ActorModel)
140 @BelongsTo(() => ActorModel, {
149 static async removeFile (instance: VideoRedundancyModel) {
150 if (!instance.isOwned()) return
152 if (instance.videoFileId) {
153 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
155 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
156 logger.info('Removing duplicated video file %s.', logIdentifier)
158 videoFile.Video.removeFile(videoFile, true)
159 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
162 if (instance.videoStreamingPlaylistId) {
163 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
165 const videoUUID = videoStreamingPlaylist.Video.uuid
166 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
168 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
169 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
175 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
176 const actor = await getServerActor()
185 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
188 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
189 const actor = await getServerActor()
194 videoStreamingPlaylistId
198 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
201 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
207 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
210 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
218 return VideoRedundancyModel.findOne(query)
221 static async isLocalByVideoUUIDExists (uuid: string) {
222 const actor = await getServerActor()
226 attributes: [ 'id' ],
233 model: VideoFileModel,
249 return VideoRedundancyModel.findOne(query)
253 static async getVideoSample (p: Promise<VideoModel[]>) {
255 if (rows.length === 0) return undefined
257 const ids = rows.map(r => r.id)
258 const id = sample(ids)
260 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
263 static async findMostViewToDuplicate (randomizedFactor: number) {
266 attributes: [ 'id', 'views' ],
267 limit: randomizedFactor,
268 order: getVideoSort('-views'),
270 privacy: VideoPrivacy.PUBLIC,
274 await VideoRedundancyModel.buildVideoFileForDuplication(),
275 VideoRedundancyModel.buildServerRedundancyInclude()
279 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
282 static async findTrendingToDuplicate (randomizedFactor: number) {
285 attributes: [ 'id', 'views' ],
287 group: 'VideoModel.id',
288 limit: randomizedFactor,
289 order: getVideoSort('-trending'),
291 privacy: VideoPrivacy.PUBLIC,
295 await VideoRedundancyModel.buildVideoFileForDuplication(),
296 VideoRedundancyModel.buildServerRedundancyInclude(),
298 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
302 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
305 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
308 attributes: [ 'id', 'publishedAt' ],
309 limit: randomizedFactor,
310 order: getVideoSort('-publishedAt'),
312 privacy: VideoPrivacy.PUBLIC,
319 await VideoRedundancyModel.buildVideoFileForDuplication(),
320 VideoRedundancyModel.buildServerRedundancyInclude()
324 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
327 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
328 const expiredDate = new Date()
329 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
331 const actor = await getServerActor()
343 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
346 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
347 const actor = await getServerActor()
348 const redundancyInclude = {
350 model: VideoRedundancyModel,
358 const queryFiles: FindOptions = {
359 include: [ redundancyInclude ]
362 const queryStreamingPlaylists: FindOptions = {
366 model: VideoModel.unscoped(),
372 model: VideoStreamingPlaylistModel.unscoped(),
383 VideoFileModel.aggregate('size', 'SUM', queryFiles),
384 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
385 ]).then(([ r1, r2 ]) => {
386 return parseAggregateResult(r1) + parseAggregateResult(r2)
390 static async listLocalExpired () {
391 const actor = await getServerActor()
402 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
405 static async listRemoteExpired () {
406 const actor = await getServerActor()
420 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
423 static async listLocalOfServer (serverId: number) {
424 const actor = await getServerActor()
425 const buildVideoInclude = () => ({
431 model: VideoChannelModel.unscoped(),
436 model: ActorModel.unscoped(),
453 model: VideoFileModel,
455 include: [ buildVideoInclude() ]
458 model: VideoStreamingPlaylistModel,
460 include: [ buildVideoInclude() ]
465 return VideoRedundancyModel.findAll(query)
468 static listForApi (options: {
472 target: VideoRedundanciesTarget
475 const { start, count, sort, target, strategy } = options
476 const redundancyWhere: WhereOptions = {}
477 const videosWhere: WhereOptions = {}
478 let redundancySqlSuffix = ''
480 if (target === 'my-videos') {
481 Object.assign(videosWhere, { remote: false })
482 } else if (target === 'remote-videos') {
483 Object.assign(videosWhere, { remote: true })
484 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
485 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
489 Object.assign(redundancyWhere, { strategy: strategy })
492 const videoFilterWhere = {
500 'SELECT "videoId" FROM "videoFile" ' +
501 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
502 redundancySqlSuffix +
511 'select "videoId" FROM "videoStreamingPlaylist" ' +
512 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
513 redundancySqlSuffix +
525 // /!\ On video model /!\
526 const findOptions = {
529 order: getSort(sort),
533 model: VideoFileModel,
536 model: VideoRedundancyModel.unscoped(),
538 where: redundancyWhere
544 model: VideoStreamingPlaylistModel.unscoped(),
547 model: VideoRedundancyModel.unscoped(),
549 where: redundancyWhere
552 model: VideoFileModel,
558 where: videoFilterWhere
561 // /!\ On video model /!\
562 const countOptions = {
563 where: videoFilterWhere
567 VideoModel.findAll(findOptions),
569 VideoModel.count(countOptions)
570 ]).then(([ data, total ]) => ({ total, data }))
573 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
574 const actor = await getServerActor()
576 const query: FindOptions = {
579 [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
580 [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
581 [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
590 model: VideoFileModel,
596 return VideoRedundancyModel.findOne(query)
598 totalUsed: parseAggregateResult(r.totalUsed),
599 totalVideos: r.totalVideos,
600 totalVideoFiles: r.totalVideoFiles
604 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
605 const filesRedundancies: FileRedundancyInformation[] = []
606 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
608 for (const file of video.VideoFiles) {
609 for (const redundancy of file.RedundancyVideos) {
610 filesRedundancies.push({
612 fileUrl: redundancy.fileUrl,
613 strategy: redundancy.strategy,
614 createdAt: redundancy.createdAt,
615 updatedAt: redundancy.updatedAt,
616 expiresOn: redundancy.expiresOn,
622 for (const playlist of video.VideoStreamingPlaylists) {
623 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
625 for (const redundancy of playlist.RedundancyVideos) {
626 streamingPlaylistsRedundancies.push({
628 fileUrl: redundancy.fileUrl,
629 strategy: redundancy.strategy,
630 createdAt: redundancy.createdAt,
631 updatedAt: redundancy.updatedAt,
632 expiresOn: redundancy.expiresOn,
645 files: filesRedundancies,
646 streamingPlaylists: streamingPlaylistsRedundancies
652 if (this.VideoFile) return this.VideoFile.Video
654 if (this.VideoStreamingPlaylist.Video) return this.VideoStreamingPlaylist.Video
660 return !!this.strategy
663 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
664 if (this.VideoStreamingPlaylist) {
667 type: 'CacheFile' as 'CacheFile',
668 object: this.VideoStreamingPlaylist.Video.url,
669 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
672 mediaType: 'application/x-mpegURL',
680 type: 'CacheFile' as 'CacheFile',
681 object: this.VideoFile.Video.url,
682 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
685 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
687 height: this.VideoFile.resolution,
688 size: this.VideoFile.size,
689 fps: this.VideoFile.fps
694 // Don't include video files we already duplicated
695 private static async buildVideoFileForDuplication () {
696 const actor = await getServerActor()
698 const notIn = literal(
700 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
706 model: VideoFileModel,
716 private static buildServerRedundancyInclude () {
719 model: VideoChannelModel.unscoped(),
724 model: ActorModel.unscoped(),
729 model: ServerModel.unscoped(),
732 redundancyAllowed: true