1 import { sample } from 'lodash'
2 import { FindOptions, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
21 FileRedundancyInformation,
22 StreamingPlaylistRedundancyInformation,
24 } from '@shared/models/redundancy/video-redundancy.model'
25 import { CacheFileObject, VideoPrivacy } from '../../../shared'
26 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
27 import { isTestInstance } from '../../helpers/core-utils'
28 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
29 import { logger } from '../../helpers/logger'
30 import { CONFIG } from '../../initializers/config'
31 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
32 import { ActorModel } from '../actor/actor'
33 import { ServerModel } from '../server/server'
34 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
35 import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
36 import { VideoModel } from '../video/video'
37 import { VideoChannelModel } from '../video/video-channel'
38 import { VideoFileModel } from '../video/video-file'
39 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
41 export enum ScopeNames {
42 WITH_VIDEO = 'WITH_VIDEO'
46 [ScopeNames.WITH_VIDEO]: {
49 model: VideoFileModel,
59 model: VideoStreamingPlaylistModel,
73 tableName: 'videoRedundancy',
76 fields: [ 'videoFileId' ]
87 export class VideoRedundancyModel extends Model {
100 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
101 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
105 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
106 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
111 strategy: string // Only used by us
113 @ForeignKey(() => VideoFileModel)
117 @BelongsTo(() => VideoFileModel, {
123 VideoFile: VideoFileModel
125 @ForeignKey(() => VideoStreamingPlaylistModel)
127 videoStreamingPlaylistId: number
129 @BelongsTo(() => VideoStreamingPlaylistModel, {
135 VideoStreamingPlaylist: VideoStreamingPlaylistModel
137 @ForeignKey(() => ActorModel)
141 @BelongsTo(() => ActorModel, {
150 static async removeFile (instance: VideoRedundancyModel) {
151 if (!instance.isOwned()) return
153 if (instance.videoFileId) {
154 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
156 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
157 logger.info('Removing duplicated video file %s.', logIdentifier)
159 videoFile.Video.removeFile(videoFile, true)
160 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
163 if (instance.videoStreamingPlaylistId) {
164 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
166 const videoUUID = videoStreamingPlaylist.Video.uuid
167 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
169 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
170 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
176 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
177 const actor = await getServerActor()
186 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
189 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
190 const actor = await getServerActor()
192 const queryStreamingPlaylist = {
198 model: VideoStreamingPlaylistModel.unscoped(),
202 model: VideoModel.unscoped(),
219 model: VideoFileModel,
235 VideoRedundancyModel.findAll(queryStreamingPlaylist),
236 VideoRedundancyModel.findAll(queryFiles)
237 ]).then(([ r1, r2 ]) => r1.concat(r2))
240 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
241 const actor = await getServerActor()
246 videoStreamingPlaylistId
250 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
253 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
259 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
262 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
270 return VideoRedundancyModel.findOne(query)
273 static async isLocalByVideoUUIDExists (uuid: string) {
274 const actor = await getServerActor()
278 attributes: [ 'id' ],
285 model: VideoFileModel,
301 return VideoRedundancyModel.findOne(query)
305 static async getVideoSample (p: Promise<VideoModel[]>) {
307 if (rows.length === 0) return undefined
309 const ids = rows.map(r => r.id)
310 const id = sample(ids)
312 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
315 static async findMostViewToDuplicate (randomizedFactor: number) {
316 const peertubeActor = await getServerActor()
320 attributes: [ 'id', 'views' ],
321 limit: randomizedFactor,
322 order: getVideoSort('-views'),
324 privacy: VideoPrivacy.PUBLIC,
326 ...this.buildVideoIdsForDuplication(peertubeActor)
329 VideoRedundancyModel.buildServerRedundancyInclude()
333 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
336 static async findTrendingToDuplicate (randomizedFactor: number) {
337 const peertubeActor = await getServerActor()
341 attributes: [ 'id', 'views' ],
343 group: 'VideoModel.id',
344 limit: randomizedFactor,
345 order: getVideoSort('-trending'),
347 privacy: VideoPrivacy.PUBLIC,
349 ...this.buildVideoIdsForDuplication(peertubeActor)
352 VideoRedundancyModel.buildServerRedundancyInclude(),
354 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
358 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
361 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
362 const peertubeActor = await getServerActor()
366 attributes: [ 'id', 'publishedAt' ],
367 limit: randomizedFactor,
368 order: getVideoSort('-publishedAt'),
370 privacy: VideoPrivacy.PUBLIC,
375 ...this.buildVideoIdsForDuplication(peertubeActor)
378 VideoRedundancyModel.buildServerRedundancyInclude(),
380 // Required by publishedAt sort
382 model: ScheduleVideoUpdateModel.unscoped(),
388 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
391 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
392 const expiredDate = new Date()
393 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
395 const actor = await getServerActor()
407 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
410 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
411 const actor = await getServerActor()
412 const redundancyInclude = {
414 model: VideoRedundancyModel,
422 const queryFiles: FindOptions = {
423 include: [ redundancyInclude ]
426 const queryStreamingPlaylists: FindOptions = {
430 model: VideoModel.unscoped(),
436 model: VideoStreamingPlaylistModel.unscoped(),
447 VideoFileModel.aggregate('size', 'SUM', queryFiles),
448 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
449 ]).then(([ r1, r2 ]) => {
450 return parseAggregateResult(r1) + parseAggregateResult(r2)
454 static async listLocalExpired () {
455 const actor = await getServerActor()
466 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
469 static async listRemoteExpired () {
470 const actor = await getServerActor()
484 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
487 static async listLocalOfServer (serverId: number) {
488 const actor = await getServerActor()
489 const buildVideoInclude = () => ({
495 model: VideoChannelModel.unscoped(),
500 model: ActorModel.unscoped(),
517 model: VideoFileModel,
519 include: [ buildVideoInclude() ]
522 model: VideoStreamingPlaylistModel,
524 include: [ buildVideoInclude() ]
529 return VideoRedundancyModel.findAll(query)
532 static listForApi (options: {
536 target: VideoRedundanciesTarget
539 const { start, count, sort, target, strategy } = options
540 const redundancyWhere: WhereOptions = {}
541 const videosWhere: WhereOptions = {}
542 let redundancySqlSuffix = ''
544 if (target === 'my-videos') {
545 Object.assign(videosWhere, { remote: false })
546 } else if (target === 'remote-videos') {
547 Object.assign(videosWhere, { remote: true })
548 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
549 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
553 Object.assign(redundancyWhere, { strategy: strategy })
556 const videoFilterWhere = {
564 'SELECT "videoId" FROM "videoFile" ' +
565 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
566 redundancySqlSuffix +
575 'select "videoId" FROM "videoStreamingPlaylist" ' +
576 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
577 redundancySqlSuffix +
589 // /!\ On video model /!\
590 const findOptions = {
593 order: getSort(sort),
597 model: VideoFileModel,
600 model: VideoRedundancyModel.unscoped(),
602 where: redundancyWhere
608 model: VideoStreamingPlaylistModel.unscoped(),
611 model: VideoRedundancyModel.unscoped(),
613 where: redundancyWhere
616 model: VideoFileModel,
622 where: videoFilterWhere
625 // /!\ On video model /!\
626 const countOptions = {
627 where: videoFilterWhere
631 VideoModel.findAll(findOptions),
633 VideoModel.count(countOptions)
634 ]).then(([ data, total ]) => ({ total, data }))
637 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
638 const actor = await getServerActor()
640 const sql = `WITH "tmp" AS ` +
642 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
643 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
644 `FROM "videoRedundancy" AS "videoRedundancy" ` +
645 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
646 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
647 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
648 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
649 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
652 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
653 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
656 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
657 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
658 `COUNT(*) AS "totalVideoFiles" ` +
661 return VideoRedundancyModel.sequelize.query<any>(sql, {
662 replacements: { strategy, actorId: actor.id },
663 type: QueryTypes.SELECT
664 }).then(([ row ]) => ({
665 totalUsed: parseAggregateResult(row.totalUsed),
666 totalVideos: row.totalVideos,
667 totalVideoFiles: row.totalVideoFiles
671 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
672 const filesRedundancies: FileRedundancyInformation[] = []
673 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
675 for (const file of video.VideoFiles) {
676 for (const redundancy of file.RedundancyVideos) {
677 filesRedundancies.push({
679 fileUrl: redundancy.fileUrl,
680 strategy: redundancy.strategy,
681 createdAt: redundancy.createdAt,
682 updatedAt: redundancy.updatedAt,
683 expiresOn: redundancy.expiresOn,
689 for (const playlist of video.VideoStreamingPlaylists) {
690 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
692 for (const redundancy of playlist.RedundancyVideos) {
693 streamingPlaylistsRedundancies.push({
695 fileUrl: redundancy.fileUrl,
696 strategy: redundancy.strategy,
697 createdAt: redundancy.createdAt,
698 updatedAt: redundancy.updatedAt,
699 expiresOn: redundancy.expiresOn,
712 files: filesRedundancies,
713 streamingPlaylists: streamingPlaylistsRedundancies
719 if (this.VideoFile?.Video) return this.VideoFile.Video
721 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
727 return !!this.strategy
730 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
731 if (this.VideoStreamingPlaylist) {
734 type: 'CacheFile' as 'CacheFile',
735 object: this.VideoStreamingPlaylist.Video.url,
736 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
739 mediaType: 'application/x-mpegURL',
747 type: 'CacheFile' as 'CacheFile',
748 object: this.VideoFile.Video.url,
749 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
752 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
754 height: this.VideoFile.resolution,
755 size: this.VideoFile.size,
756 fps: this.VideoFile.fps
761 // Don't include video files we already duplicated
762 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
763 const notIn = literal(
765 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
766 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
767 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
769 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
770 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
771 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
782 private static buildServerRedundancyInclude () {
785 model: VideoChannelModel.unscoped(),
790 model: ActorModel.unscoped(),
795 model: ServerModel.unscoped(),
798 redundancyAllowed: true