1 import { sample } from 'lodash'
2 import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { AttributesOnly } from '@shared/typescript-utils'
20 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
22 FileRedundancyInformation,
23 StreamingPlaylistRedundancyInformation,
25 } from '@shared/models/redundancy/video-redundancy.model'
26 import { CacheFileObject, VideoPrivacy } from '../../../shared'
27 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
30 import { logger } from '../../helpers/logger'
31 import { CONFIG } from '../../initializers/config'
32 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
33 import { ActorModel } from '../actor/actor'
34 import { ServerModel } from '../server/server'
35 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
36 import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
37 import { VideoModel } from '../video/video'
38 import { VideoChannelModel } from '../video/video-channel'
39 import { VideoFileModel } from '../video/video-file'
40 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
42 export enum ScopeNames {
43 WITH_VIDEO = 'WITH_VIDEO'
47 [ScopeNames.WITH_VIDEO]: {
50 model: VideoFileModel,
60 model: VideoStreamingPlaylistModel,
74 tableName: 'videoRedundancy',
77 fields: [ 'videoFileId' ]
83 fields: [ 'expiresOn' ]
91 export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
104 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
105 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
109 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
110 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
115 strategy: string // Only used by us
117 @ForeignKey(() => VideoFileModel)
121 @BelongsTo(() => VideoFileModel, {
127 VideoFile: VideoFileModel
129 @ForeignKey(() => VideoStreamingPlaylistModel)
131 videoStreamingPlaylistId: number
133 @BelongsTo(() => VideoStreamingPlaylistModel, {
139 VideoStreamingPlaylist: VideoStreamingPlaylistModel
141 @ForeignKey(() => ActorModel)
145 @BelongsTo(() => ActorModel, {
154 static async removeFile (instance: VideoRedundancyModel) {
155 if (!instance.isOwned()) return
157 if (instance.videoFileId) {
158 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
160 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
161 logger.info('Removing duplicated video file %s.', logIdentifier)
163 videoFile.Video.removeWebTorrentFileAndTorrent(videoFile, true)
164 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
167 if (instance.videoStreamingPlaylistId) {
168 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
170 const videoUUID = videoStreamingPlaylist.Video.uuid
171 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
173 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
174 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
180 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
181 const actor = await getServerActor()
190 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
193 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
194 const actor = await getServerActor()
196 const queryStreamingPlaylist = {
202 model: VideoStreamingPlaylistModel.unscoped(),
206 model: VideoModel.unscoped(),
223 model: VideoFileModel,
239 VideoRedundancyModel.findAll(queryStreamingPlaylist),
240 VideoRedundancyModel.findAll(queryFiles)
241 ]).then(([ r1, r2 ]) => r1.concat(r2))
244 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
245 const actor = await getServerActor()
250 videoStreamingPlaylistId
254 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
257 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
263 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
266 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
274 return VideoRedundancyModel.findOne(query)
277 static async isLocalByVideoUUIDExists (uuid: string) {
278 const actor = await getServerActor()
282 attributes: [ 'id' ],
289 model: VideoFileModel,
305 return VideoRedundancyModel.findOne(query)
309 static async getVideoSample (p: Promise<VideoModel[]>) {
311 if (rows.length === 0) return undefined
313 const ids = rows.map(r => r.id)
314 const id = sample(ids)
316 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
319 static async findMostViewToDuplicate (randomizedFactor: number) {
320 const peertubeActor = await getServerActor()
324 attributes: [ 'id', 'views' ],
325 limit: randomizedFactor,
326 order: getVideoSort('-views'),
328 privacy: VideoPrivacy.PUBLIC,
330 ...this.buildVideoIdsForDuplication(peertubeActor)
333 VideoRedundancyModel.buildServerRedundancyInclude()
337 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
340 static async findTrendingToDuplicate (randomizedFactor: number) {
341 const peertubeActor = await getServerActor()
345 attributes: [ 'id', 'views' ],
347 group: 'VideoModel.id',
348 limit: randomizedFactor,
349 order: getVideoSort('-trending'),
351 privacy: VideoPrivacy.PUBLIC,
353 ...this.buildVideoIdsForDuplication(peertubeActor)
356 VideoRedundancyModel.buildServerRedundancyInclude(),
358 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
362 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
365 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
366 const peertubeActor = await getServerActor()
370 attributes: [ 'id', 'publishedAt' ],
371 limit: randomizedFactor,
372 order: getVideoSort('-publishedAt'),
374 privacy: VideoPrivacy.PUBLIC,
379 ...this.buildVideoIdsForDuplication(peertubeActor)
382 VideoRedundancyModel.buildServerRedundancyInclude(),
384 // Required by publishedAt sort
386 model: ScheduleVideoUpdateModel.unscoped(),
392 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
395 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
396 const expiredDate = new Date()
397 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
399 const actor = await getServerActor()
411 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
414 static async listLocalExpired (): Promise<MVideoRedundancyVideo[]> {
415 const actor = await getServerActor()
426 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
429 static async listRemoteExpired () {
430 const actor = await getServerActor()
444 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
447 static async listLocalOfServer (serverId: number) {
448 const actor = await getServerActor()
449 const buildVideoInclude = () => ({
455 model: VideoChannelModel.unscoped(),
460 model: ActorModel.unscoped(),
480 '$VideoStreamingPlaylist.id$': {
495 model: VideoFileModel.unscoped(),
497 include: [ buildVideoInclude() ]
500 model: VideoStreamingPlaylistModel.unscoped(),
502 include: [ buildVideoInclude() ]
507 return VideoRedundancyModel.findAll(query)
510 static listForApi (options: {
514 target: VideoRedundanciesTarget
517 const { start, count, sort, target, strategy } = options
518 const redundancyWhere: WhereOptions = {}
519 const videosWhere: WhereOptions = {}
520 let redundancySqlSuffix = ''
522 if (target === 'my-videos') {
523 Object.assign(videosWhere, { remote: false })
524 } else if (target === 'remote-videos') {
525 Object.assign(videosWhere, { remote: true })
526 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
527 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
531 Object.assign(redundancyWhere, { strategy: strategy })
534 const videoFilterWhere = {
542 'SELECT "videoId" FROM "videoFile" ' +
543 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
544 redundancySqlSuffix +
553 'select "videoId" FROM "videoStreamingPlaylist" ' +
554 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
555 redundancySqlSuffix +
567 // /!\ On video model /!\
568 const findOptions = {
571 order: getSort(sort),
575 model: VideoFileModel,
578 model: VideoRedundancyModel.unscoped(),
580 where: redundancyWhere
586 model: VideoStreamingPlaylistModel.unscoped(),
589 model: VideoRedundancyModel.unscoped(),
591 where: redundancyWhere
594 model: VideoFileModel,
600 where: videoFilterWhere
603 // /!\ On video model /!\
604 const countOptions = {
605 where: videoFilterWhere
609 VideoModel.findAll(findOptions),
611 VideoModel.count(countOptions)
612 ]).then(([ data, total ]) => ({ total, data }))
615 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
616 const actor = await getServerActor()
618 const sql = `WITH "tmp" AS ` +
620 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
621 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
622 `FROM "videoRedundancy" AS "videoRedundancy" ` +
623 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
624 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
625 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
626 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
627 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
630 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
631 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
634 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
635 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
636 `COUNT(*) AS "totalVideoFiles" ` +
639 return VideoRedundancyModel.sequelize.query<any>(sql, {
640 replacements: { strategy, actorId: actor.id },
641 type: QueryTypes.SELECT
642 }).then(([ row ]) => ({
643 totalUsed: parseAggregateResult(row.totalUsed),
644 totalVideos: row.totalVideos,
645 totalVideoFiles: row.totalVideoFiles
649 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
650 const filesRedundancies: FileRedundancyInformation[] = []
651 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
653 for (const file of video.VideoFiles) {
654 for (const redundancy of file.RedundancyVideos) {
655 filesRedundancies.push({
657 fileUrl: redundancy.fileUrl,
658 strategy: redundancy.strategy,
659 createdAt: redundancy.createdAt,
660 updatedAt: redundancy.updatedAt,
661 expiresOn: redundancy.expiresOn,
667 for (const playlist of video.VideoStreamingPlaylists) {
668 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
670 for (const redundancy of playlist.RedundancyVideos) {
671 streamingPlaylistsRedundancies.push({
673 fileUrl: redundancy.fileUrl,
674 strategy: redundancy.strategy,
675 createdAt: redundancy.createdAt,
676 updatedAt: redundancy.updatedAt,
677 expiresOn: redundancy.expiresOn,
690 files: filesRedundancies,
691 streamingPlaylists: streamingPlaylistsRedundancies
697 if (this.VideoFile?.Video) return this.VideoFile.Video
699 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
705 const video = this.getVideo()
706 if (!video) return undefined
712 return !!this.strategy
715 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
716 if (this.VideoStreamingPlaylist) {
719 type: 'CacheFile' as 'CacheFile',
720 object: this.VideoStreamingPlaylist.Video.url,
721 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
724 mediaType: 'application/x-mpegURL',
732 type: 'CacheFile' as 'CacheFile',
733 object: this.VideoFile.Video.url,
734 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
737 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
739 height: this.VideoFile.resolution,
740 size: this.VideoFile.size,
741 fps: this.VideoFile.fps
746 // Don't include video files we already duplicated
747 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
748 const notIn = literal(
750 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
751 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
752 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
754 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
755 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
756 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
767 private static buildServerRedundancyInclude () {
770 model: VideoChannelModel.unscoped(),
775 model: ActorModel.unscoped(),
780 model: ServerModel.unscoped(),
783 redundancyAllowed: true