1 import { sample } from 'lodash'
2 import { FindOptions, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { AttributesOnly } from '@shared/core-utils'
20 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
22 FileRedundancyInformation,
23 StreamingPlaylistRedundancyInformation,
25 } from '@shared/models/redundancy/video-redundancy.model'
26 import { CacheFileObject, VideoPrivacy } from '../../../shared'
27 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
30 import { logger } from '../../helpers/logger'
31 import { CONFIG } from '../../initializers/config'
32 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
33 import { ActorModel } from '../actor/actor'
34 import { ServerModel } from '../server/server'
35 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
36 import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
37 import { VideoModel } from '../video/video'
38 import { VideoChannelModel } from '../video/video-channel'
39 import { VideoFileModel } from '../video/video-file'
40 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
42 export enum ScopeNames {
43 WITH_VIDEO = 'WITH_VIDEO'
47 [ScopeNames.WITH_VIDEO]: {
50 model: VideoFileModel,
60 model: VideoStreamingPlaylistModel,
74 tableName: 'videoRedundancy',
77 fields: [ 'videoFileId' ]
88 export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
101 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
102 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
106 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
107 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
112 strategy: string // Only used by us
114 @ForeignKey(() => VideoFileModel)
118 @BelongsTo(() => VideoFileModel, {
124 VideoFile: VideoFileModel
126 @ForeignKey(() => VideoStreamingPlaylistModel)
128 videoStreamingPlaylistId: number
130 @BelongsTo(() => VideoStreamingPlaylistModel, {
136 VideoStreamingPlaylist: VideoStreamingPlaylistModel
138 @ForeignKey(() => ActorModel)
142 @BelongsTo(() => ActorModel, {
151 static async removeFile (instance: VideoRedundancyModel) {
152 if (!instance.isOwned()) return
154 if (instance.videoFileId) {
155 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
157 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
158 logger.info('Removing duplicated video file %s.', logIdentifier)
160 videoFile.Video.removeFile(videoFile, true)
161 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
164 if (instance.videoStreamingPlaylistId) {
165 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
167 const videoUUID = videoStreamingPlaylist.Video.uuid
168 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
170 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
171 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
177 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
178 const actor = await getServerActor()
187 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
190 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
191 const actor = await getServerActor()
193 const queryStreamingPlaylist = {
199 model: VideoStreamingPlaylistModel.unscoped(),
203 model: VideoModel.unscoped(),
220 model: VideoFileModel,
236 VideoRedundancyModel.findAll(queryStreamingPlaylist),
237 VideoRedundancyModel.findAll(queryFiles)
238 ]).then(([ r1, r2 ]) => r1.concat(r2))
241 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
242 const actor = await getServerActor()
247 videoStreamingPlaylistId
251 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
254 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
260 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
263 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
271 return VideoRedundancyModel.findOne(query)
274 static async isLocalByVideoUUIDExists (uuid: string) {
275 const actor = await getServerActor()
279 attributes: [ 'id' ],
286 model: VideoFileModel,
302 return VideoRedundancyModel.findOne(query)
306 static async getVideoSample (p: Promise<VideoModel[]>) {
308 if (rows.length === 0) return undefined
310 const ids = rows.map(r => r.id)
311 const id = sample(ids)
313 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
316 static async findMostViewToDuplicate (randomizedFactor: number) {
317 const peertubeActor = await getServerActor()
321 attributes: [ 'id', 'views' ],
322 limit: randomizedFactor,
323 order: getVideoSort('-views'),
325 privacy: VideoPrivacy.PUBLIC,
327 ...this.buildVideoIdsForDuplication(peertubeActor)
330 VideoRedundancyModel.buildServerRedundancyInclude()
334 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
337 static async findTrendingToDuplicate (randomizedFactor: number) {
338 const peertubeActor = await getServerActor()
342 attributes: [ 'id', 'views' ],
344 group: 'VideoModel.id',
345 limit: randomizedFactor,
346 order: getVideoSort('-trending'),
348 privacy: VideoPrivacy.PUBLIC,
350 ...this.buildVideoIdsForDuplication(peertubeActor)
353 VideoRedundancyModel.buildServerRedundancyInclude(),
355 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
359 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
362 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
363 const peertubeActor = await getServerActor()
367 attributes: [ 'id', 'publishedAt' ],
368 limit: randomizedFactor,
369 order: getVideoSort('-publishedAt'),
371 privacy: VideoPrivacy.PUBLIC,
376 ...this.buildVideoIdsForDuplication(peertubeActor)
379 VideoRedundancyModel.buildServerRedundancyInclude(),
381 // Required by publishedAt sort
383 model: ScheduleVideoUpdateModel.unscoped(),
389 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
392 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
393 const expiredDate = new Date()
394 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
396 const actor = await getServerActor()
408 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
411 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
412 const actor = await getServerActor()
413 const redundancyInclude = {
415 model: VideoRedundancyModel,
423 const queryFiles: FindOptions = {
424 include: [ redundancyInclude ]
427 const queryStreamingPlaylists: FindOptions = {
431 model: VideoModel.unscoped(),
437 model: VideoStreamingPlaylistModel.unscoped(),
448 VideoFileModel.aggregate('size', 'SUM', queryFiles),
449 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
450 ]).then(([ r1, r2 ]) => {
451 return parseAggregateResult(r1) + parseAggregateResult(r2)
455 static async listLocalExpired () {
456 const actor = await getServerActor()
467 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
470 static async listRemoteExpired () {
471 const actor = await getServerActor()
485 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
488 static async listLocalOfServer (serverId: number) {
489 const actor = await getServerActor()
490 const buildVideoInclude = () => ({
496 model: VideoChannelModel.unscoped(),
501 model: ActorModel.unscoped(),
518 model: VideoFileModel,
520 include: [ buildVideoInclude() ]
523 model: VideoStreamingPlaylistModel,
525 include: [ buildVideoInclude() ]
530 return VideoRedundancyModel.findAll(query)
533 static listForApi (options: {
537 target: VideoRedundanciesTarget
540 const { start, count, sort, target, strategy } = options
541 const redundancyWhere: WhereOptions = {}
542 const videosWhere: WhereOptions = {}
543 let redundancySqlSuffix = ''
545 if (target === 'my-videos') {
546 Object.assign(videosWhere, { remote: false })
547 } else if (target === 'remote-videos') {
548 Object.assign(videosWhere, { remote: true })
549 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
550 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
554 Object.assign(redundancyWhere, { strategy: strategy })
557 const videoFilterWhere = {
565 'SELECT "videoId" FROM "videoFile" ' +
566 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
567 redundancySqlSuffix +
576 'select "videoId" FROM "videoStreamingPlaylist" ' +
577 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
578 redundancySqlSuffix +
590 // /!\ On video model /!\
591 const findOptions = {
594 order: getSort(sort),
598 model: VideoFileModel,
601 model: VideoRedundancyModel.unscoped(),
603 where: redundancyWhere
609 model: VideoStreamingPlaylistModel.unscoped(),
612 model: VideoRedundancyModel.unscoped(),
614 where: redundancyWhere
617 model: VideoFileModel,
623 where: videoFilterWhere
626 // /!\ On video model /!\
627 const countOptions = {
628 where: videoFilterWhere
632 VideoModel.findAll(findOptions),
634 VideoModel.count(countOptions)
635 ]).then(([ data, total ]) => ({ total, data }))
638 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
639 const actor = await getServerActor()
641 const sql = `WITH "tmp" AS ` +
643 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
644 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
645 `FROM "videoRedundancy" AS "videoRedundancy" ` +
646 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
647 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
648 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
649 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
650 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
653 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
654 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
657 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
658 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
659 `COUNT(*) AS "totalVideoFiles" ` +
662 return VideoRedundancyModel.sequelize.query<any>(sql, {
663 replacements: { strategy, actorId: actor.id },
664 type: QueryTypes.SELECT
665 }).then(([ row ]) => ({
666 totalUsed: parseAggregateResult(row.totalUsed),
667 totalVideos: row.totalVideos,
668 totalVideoFiles: row.totalVideoFiles
672 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
673 const filesRedundancies: FileRedundancyInformation[] = []
674 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
676 for (const file of video.VideoFiles) {
677 for (const redundancy of file.RedundancyVideos) {
678 filesRedundancies.push({
680 fileUrl: redundancy.fileUrl,
681 strategy: redundancy.strategy,
682 createdAt: redundancy.createdAt,
683 updatedAt: redundancy.updatedAt,
684 expiresOn: redundancy.expiresOn,
690 for (const playlist of video.VideoStreamingPlaylists) {
691 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
693 for (const redundancy of playlist.RedundancyVideos) {
694 streamingPlaylistsRedundancies.push({
696 fileUrl: redundancy.fileUrl,
697 strategy: redundancy.strategy,
698 createdAt: redundancy.createdAt,
699 updatedAt: redundancy.updatedAt,
700 expiresOn: redundancy.expiresOn,
713 files: filesRedundancies,
714 streamingPlaylists: streamingPlaylistsRedundancies
720 if (this.VideoFile?.Video) return this.VideoFile.Video
722 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
728 return !!this.strategy
731 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
732 if (this.VideoStreamingPlaylist) {
735 type: 'CacheFile' as 'CacheFile',
736 object: this.VideoStreamingPlaylist.Video.url,
737 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
740 mediaType: 'application/x-mpegURL',
748 type: 'CacheFile' as 'CacheFile',
749 object: this.VideoFile.Video.url,
750 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
753 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
755 height: this.VideoFile.resolution,
756 size: this.VideoFile.size,
757 fps: this.VideoFile.fps
762 // Don't include video files we already duplicated
763 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
764 const notIn = literal(
766 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
767 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
768 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
770 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
771 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
772 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
783 private static buildServerRedundancyInclude () {
786 model: VideoChannelModel.unscoped(),
791 model: ActorModel.unscoped(),
796 model: ServerModel.unscoped(),
799 redundancyAllowed: true