1 import { sample } from 'lodash'
2 import { FindOptions, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
21 FileRedundancyInformation,
22 StreamingPlaylistRedundancyInformation,
24 } from '@shared/models/redundancy/video-redundancy.model'
25 import { CacheFileObject, VideoPrivacy } from '../../../shared'
26 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
27 import { isTestInstance } from '../../helpers/core-utils'
28 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
29 import { logger } from '../../helpers/logger'
30 import { CONFIG } from '../../initializers/config'
31 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
32 import { ActorModel } from '../activitypub/actor'
33 import { ServerModel } from '../server/server'
34 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
35 import { VideoModel } from '../video/video'
36 import { VideoChannelModel } from '../video/video-channel'
37 import { VideoFileModel } from '../video/video-file'
38 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
40 export enum ScopeNames {
41 WITH_VIDEO = 'WITH_VIDEO'
45 [ScopeNames.WITH_VIDEO]: {
48 model: VideoFileModel,
58 model: VideoStreamingPlaylistModel,
72 tableName: 'videoRedundancy',
75 fields: [ 'videoFileId' ]
86 export class VideoRedundancyModel extends Model {
99 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
100 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
104 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
105 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
110 strategy: string // Only used by us
112 @ForeignKey(() => VideoFileModel)
116 @BelongsTo(() => VideoFileModel, {
122 VideoFile: VideoFileModel
124 @ForeignKey(() => VideoStreamingPlaylistModel)
126 videoStreamingPlaylistId: number
128 @BelongsTo(() => VideoStreamingPlaylistModel, {
134 VideoStreamingPlaylist: VideoStreamingPlaylistModel
136 @ForeignKey(() => ActorModel)
140 @BelongsTo(() => ActorModel, {
149 static async removeFile (instance: VideoRedundancyModel) {
150 if (!instance.isOwned()) return
152 if (instance.videoFileId) {
153 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
155 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
156 logger.info('Removing duplicated video file %s.', logIdentifier)
158 videoFile.Video.removeFile(videoFile, true)
159 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
162 if (instance.videoStreamingPlaylistId) {
163 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
165 const videoUUID = videoStreamingPlaylist.Video.uuid
166 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
168 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
169 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
175 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
176 const actor = await getServerActor()
185 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
188 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
189 const actor = await getServerActor()
191 const queryStreamingPlaylist = {
197 model: VideoStreamingPlaylistModel.unscoped(),
201 model: VideoModel.unscoped(),
218 model: VideoFileModel,
234 VideoRedundancyModel.findAll(queryStreamingPlaylist),
235 VideoRedundancyModel.findAll(queryFiles)
236 ]).then(([ r1, r2 ]) => r1.concat(r2))
239 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
240 const actor = await getServerActor()
245 videoStreamingPlaylistId
249 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
252 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
258 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
261 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
269 return VideoRedundancyModel.findOne(query)
272 static async isLocalByVideoUUIDExists (uuid: string) {
273 const actor = await getServerActor()
277 attributes: [ 'id' ],
284 model: VideoFileModel,
300 return VideoRedundancyModel.findOne(query)
304 static async getVideoSample (p: Promise<VideoModel[]>) {
306 if (rows.length === 0) return undefined
308 const ids = rows.map(r => r.id)
309 const id = sample(ids)
311 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
314 static async findMostViewToDuplicate (randomizedFactor: number) {
315 const peertubeActor = await getServerActor()
319 attributes: [ 'id', 'views' ],
320 limit: randomizedFactor,
321 order: getVideoSort('-views'),
323 privacy: VideoPrivacy.PUBLIC,
325 ...this.buildVideoIdsForDuplication(peertubeActor)
328 VideoRedundancyModel.buildServerRedundancyInclude()
332 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
335 static async findTrendingToDuplicate (randomizedFactor: number) {
336 const peertubeActor = await getServerActor()
340 attributes: [ 'id', 'views' ],
342 group: 'VideoModel.id',
343 limit: randomizedFactor,
344 order: getVideoSort('-trending'),
346 privacy: VideoPrivacy.PUBLIC,
348 ...this.buildVideoIdsForDuplication(peertubeActor)
351 VideoRedundancyModel.buildServerRedundancyInclude(),
353 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
357 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
360 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
361 const peertubeActor = await getServerActor()
365 attributes: [ 'id', 'publishedAt' ],
366 limit: randomizedFactor,
367 order: getVideoSort('-publishedAt'),
369 privacy: VideoPrivacy.PUBLIC,
374 ...this.buildVideoIdsForDuplication(peertubeActor)
377 VideoRedundancyModel.buildServerRedundancyInclude()
381 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
384 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
385 const expiredDate = new Date()
386 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
388 const actor = await getServerActor()
400 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
403 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
404 const actor = await getServerActor()
405 const redundancyInclude = {
407 model: VideoRedundancyModel,
415 const queryFiles: FindOptions = {
416 include: [ redundancyInclude ]
419 const queryStreamingPlaylists: FindOptions = {
423 model: VideoModel.unscoped(),
429 model: VideoStreamingPlaylistModel.unscoped(),
440 VideoFileModel.aggregate('size', 'SUM', queryFiles),
441 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
442 ]).then(([ r1, r2 ]) => {
443 return parseAggregateResult(r1) + parseAggregateResult(r2)
447 static async listLocalExpired () {
448 const actor = await getServerActor()
459 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
462 static async listRemoteExpired () {
463 const actor = await getServerActor()
477 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
480 static async listLocalOfServer (serverId: number) {
481 const actor = await getServerActor()
482 const buildVideoInclude = () => ({
488 model: VideoChannelModel.unscoped(),
493 model: ActorModel.unscoped(),
510 model: VideoFileModel,
512 include: [ buildVideoInclude() ]
515 model: VideoStreamingPlaylistModel,
517 include: [ buildVideoInclude() ]
522 return VideoRedundancyModel.findAll(query)
525 static listForApi (options: {
529 target: VideoRedundanciesTarget
532 const { start, count, sort, target, strategy } = options
533 const redundancyWhere: WhereOptions = {}
534 const videosWhere: WhereOptions = {}
535 let redundancySqlSuffix = ''
537 if (target === 'my-videos') {
538 Object.assign(videosWhere, { remote: false })
539 } else if (target === 'remote-videos') {
540 Object.assign(videosWhere, { remote: true })
541 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
542 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
546 Object.assign(redundancyWhere, { strategy: strategy })
549 const videoFilterWhere = {
557 'SELECT "videoId" FROM "videoFile" ' +
558 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
559 redundancySqlSuffix +
568 'select "videoId" FROM "videoStreamingPlaylist" ' +
569 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
570 redundancySqlSuffix +
582 // /!\ On video model /!\
583 const findOptions = {
586 order: getSort(sort),
590 model: VideoFileModel,
593 model: VideoRedundancyModel.unscoped(),
595 where: redundancyWhere
601 model: VideoStreamingPlaylistModel.unscoped(),
604 model: VideoRedundancyModel.unscoped(),
606 where: redundancyWhere
609 model: VideoFileModel,
615 where: videoFilterWhere
618 // /!\ On video model /!\
619 const countOptions = {
620 where: videoFilterWhere
624 VideoModel.findAll(findOptions),
626 VideoModel.count(countOptions)
627 ]).then(([ data, total ]) => ({ total, data }))
630 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
631 const actor = await getServerActor()
633 const sql = `WITH "tmp" AS ` +
635 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
636 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
637 `FROM "videoRedundancy" AS "videoRedundancy" ` +
638 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
639 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
640 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
641 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
642 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
645 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
646 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
649 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
650 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
651 `COUNT(*) AS "totalVideoFiles" ` +
654 return VideoRedundancyModel.sequelize.query<any>(sql, {
655 replacements: { strategy, actorId: actor.id },
656 type: QueryTypes.SELECT
657 }).then(([ row ]) => ({
658 totalUsed: parseAggregateResult(row.totalUsed),
659 totalVideos: row.totalVideos,
660 totalVideoFiles: row.totalVideoFiles
664 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
665 const filesRedundancies: FileRedundancyInformation[] = []
666 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
668 for (const file of video.VideoFiles) {
669 for (const redundancy of file.RedundancyVideos) {
670 filesRedundancies.push({
672 fileUrl: redundancy.fileUrl,
673 strategy: redundancy.strategy,
674 createdAt: redundancy.createdAt,
675 updatedAt: redundancy.updatedAt,
676 expiresOn: redundancy.expiresOn,
682 for (const playlist of video.VideoStreamingPlaylists) {
683 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
685 for (const redundancy of playlist.RedundancyVideos) {
686 streamingPlaylistsRedundancies.push({
688 fileUrl: redundancy.fileUrl,
689 strategy: redundancy.strategy,
690 createdAt: redundancy.createdAt,
691 updatedAt: redundancy.updatedAt,
692 expiresOn: redundancy.expiresOn,
705 files: filesRedundancies,
706 streamingPlaylists: streamingPlaylistsRedundancies
712 if (this.VideoFile?.Video) return this.VideoFile.Video
714 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
720 return !!this.strategy
723 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
724 if (this.VideoStreamingPlaylist) {
727 type: 'CacheFile' as 'CacheFile',
728 object: this.VideoStreamingPlaylist.Video.url,
729 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
732 mediaType: 'application/x-mpegURL',
740 type: 'CacheFile' as 'CacheFile',
741 object: this.VideoFile.Video.url,
742 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
745 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
747 height: this.VideoFile.resolution,
748 size: this.VideoFile.size,
749 fps: this.VideoFile.fps
754 // Don't include video files we already duplicated
755 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
756 const notIn = literal(
758 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
759 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
760 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
762 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
763 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
764 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
775 private static buildServerRedundancyInclude () {
778 model: VideoChannelModel.unscoped(),
783 model: ActorModel.unscoped(),
788 model: ServerModel.unscoped(),
791 redundancyAllowed: true