1 import { sample } from 'lodash'
2 import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
21 FileRedundancyInformation,
22 StreamingPlaylistRedundancyInformation,
24 VideoRedundanciesTarget,
26 VideoRedundancyStrategy,
27 VideoRedundancyStrategyWithManual
28 } from '@shared/models'
29 import { AttributesOnly } from '@shared/typescript-utils'
30 import { isTestInstance } from '../../helpers/core-utils'
31 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
32 import { logger } from '../../helpers/logger'
33 import { CONFIG } from '../../initializers/config'
34 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
35 import { ActorModel } from '../actor/actor'
36 import { ServerModel } from '../server/server'
37 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared'
38 import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
39 import { VideoModel } from '../video/video'
40 import { VideoChannelModel } from '../video/video-channel'
41 import { VideoFileModel } from '../video/video-file'
42 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
44 export enum ScopeNames {
45 WITH_VIDEO = 'WITH_VIDEO'
49 [ScopeNames.WITH_VIDEO]: {
52 model: VideoFileModel,
62 model: VideoStreamingPlaylistModel,
76 tableName: 'videoRedundancy',
79 fields: [ 'videoFileId' ]
85 fields: [ 'expiresOn' ]
93 export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
106 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
107 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
111 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
112 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
117 strategy: string // Only used by us
119 @ForeignKey(() => VideoFileModel)
123 @BelongsTo(() => VideoFileModel, {
129 VideoFile: VideoFileModel
131 @ForeignKey(() => VideoStreamingPlaylistModel)
133 videoStreamingPlaylistId: number
135 @BelongsTo(() => VideoStreamingPlaylistModel, {
141 VideoStreamingPlaylist: VideoStreamingPlaylistModel
143 @ForeignKey(() => ActorModel)
147 @BelongsTo(() => ActorModel, {
156 static async removeFile (instance: VideoRedundancyModel) {
157 if (!instance.isOwned()) return
159 if (instance.videoFileId) {
160 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
162 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
163 logger.info('Removing duplicated video file %s.', logIdentifier)
165 videoFile.Video.removeWebTorrentFile(videoFile, true)
166 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
169 if (instance.videoStreamingPlaylistId) {
170 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
172 const videoUUID = videoStreamingPlaylist.Video.uuid
173 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
175 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
176 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
182 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
183 const actor = await getServerActor()
192 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
195 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
196 const actor = await getServerActor()
198 const queryStreamingPlaylist = {
204 model: VideoStreamingPlaylistModel.unscoped(),
208 model: VideoModel.unscoped(),
225 model: VideoFileModel,
241 VideoRedundancyModel.findAll(queryStreamingPlaylist),
242 VideoRedundancyModel.findAll(queryFiles)
243 ]).then(([ r1, r2 ]) => r1.concat(r2))
246 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
247 const actor = await getServerActor()
252 videoStreamingPlaylistId
256 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
259 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
265 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
268 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
276 return VideoRedundancyModel.findOne(query)
279 static async isLocalByVideoUUIDExists (uuid: string) {
280 const actor = await getServerActor()
284 attributes: [ 'id' ],
291 model: VideoFileModel,
307 return VideoRedundancyModel.findOne(query)
311 static async getVideoSample (p: Promise<VideoModel[]>) {
313 if (rows.length === 0) return undefined
315 const ids = rows.map(r => r.id)
316 const id = sample(ids)
318 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
321 static async findMostViewToDuplicate (randomizedFactor: number) {
322 const peertubeActor = await getServerActor()
326 attributes: [ 'id', 'views' ],
327 limit: randomizedFactor,
328 order: getVideoSort('-views'),
330 privacy: VideoPrivacy.PUBLIC,
332 ...this.buildVideoIdsForDuplication(peertubeActor)
335 VideoRedundancyModel.buildServerRedundancyInclude()
339 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
342 static async findTrendingToDuplicate (randomizedFactor: number) {
343 const peertubeActor = await getServerActor()
347 attributes: [ 'id', 'views' ],
349 group: 'VideoModel.id',
350 limit: randomizedFactor,
351 order: getVideoSort('-trending'),
353 privacy: VideoPrivacy.PUBLIC,
355 ...this.buildVideoIdsForDuplication(peertubeActor)
358 VideoRedundancyModel.buildServerRedundancyInclude(),
360 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
364 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
367 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
368 const peertubeActor = await getServerActor()
372 attributes: [ 'id', 'publishedAt' ],
373 limit: randomizedFactor,
374 order: getVideoSort('-publishedAt'),
376 privacy: VideoPrivacy.PUBLIC,
381 ...this.buildVideoIdsForDuplication(peertubeActor)
384 VideoRedundancyModel.buildServerRedundancyInclude(),
386 // Required by publishedAt sort
388 model: ScheduleVideoUpdateModel.unscoped(),
394 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
397 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
398 const expiredDate = new Date()
399 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
401 const actor = await getServerActor()
413 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
416 static async listLocalExpired (): Promise<MVideoRedundancyVideo[]> {
417 const actor = await getServerActor()
428 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
431 static async listRemoteExpired () {
432 const actor = await getServerActor()
446 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
449 static async listLocalOfServer (serverId: number) {
450 const actor = await getServerActor()
451 const buildVideoInclude = () => ({
457 model: VideoChannelModel.unscoped(),
462 model: ActorModel.unscoped(),
482 '$VideoStreamingPlaylist.id$': {
497 model: VideoFileModel.unscoped(),
499 include: [ buildVideoInclude() ]
502 model: VideoStreamingPlaylistModel.unscoped(),
504 include: [ buildVideoInclude() ]
509 return VideoRedundancyModel.findAll(query)
512 static listForApi (options: {
516 target: VideoRedundanciesTarget
519 const { start, count, sort, target, strategy } = options
520 const redundancyWhere: WhereOptions = {}
521 const videosWhere: WhereOptions = {}
522 let redundancySqlSuffix = ''
524 if (target === 'my-videos') {
525 Object.assign(videosWhere, { remote: false })
526 } else if (target === 'remote-videos') {
527 Object.assign(videosWhere, { remote: true })
528 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
529 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
533 Object.assign(redundancyWhere, { strategy })
536 const videoFilterWhere = {
544 'SELECT "videoId" FROM "videoFile" ' +
545 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
546 redundancySqlSuffix +
555 'select "videoId" FROM "videoStreamingPlaylist" ' +
556 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
557 redundancySqlSuffix +
569 // /!\ On video model /!\
570 const findOptions = {
573 order: getSort(sort),
577 model: VideoFileModel,
580 model: VideoRedundancyModel.unscoped(),
582 where: redundancyWhere
588 model: VideoStreamingPlaylistModel.unscoped(),
591 model: VideoRedundancyModel.unscoped(),
593 where: redundancyWhere
596 model: VideoFileModel,
602 where: videoFilterWhere
605 // /!\ On video model /!\
606 const countOptions = {
607 where: videoFilterWhere
611 VideoModel.findAll(findOptions),
613 VideoModel.count(countOptions)
614 ]).then(([ data, total ]) => ({ total, data }))
617 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
618 const actor = await getServerActor()
620 const sql = `WITH "tmp" AS ` +
622 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
623 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
624 `FROM "videoRedundancy" AS "videoRedundancy" ` +
625 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
626 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
627 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
628 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
629 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
632 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
633 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
636 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
637 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
638 `COUNT(*) AS "totalVideoFiles" ` +
641 return VideoRedundancyModel.sequelize.query<any>(sql, {
642 replacements: { strategy, actorId: actor.id },
643 type: QueryTypes.SELECT
644 }).then(([ row ]) => ({
645 totalUsed: parseAggregateResult(row.totalUsed),
646 totalVideos: row.totalVideos,
647 totalVideoFiles: row.totalVideoFiles
651 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
652 const filesRedundancies: FileRedundancyInformation[] = []
653 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
655 for (const file of video.VideoFiles) {
656 for (const redundancy of file.RedundancyVideos) {
657 filesRedundancies.push({
659 fileUrl: redundancy.fileUrl,
660 strategy: redundancy.strategy,
661 createdAt: redundancy.createdAt,
662 updatedAt: redundancy.updatedAt,
663 expiresOn: redundancy.expiresOn,
669 for (const playlist of video.VideoStreamingPlaylists) {
670 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
672 for (const redundancy of playlist.RedundancyVideos) {
673 streamingPlaylistsRedundancies.push({
675 fileUrl: redundancy.fileUrl,
676 strategy: redundancy.strategy,
677 createdAt: redundancy.createdAt,
678 updatedAt: redundancy.updatedAt,
679 expiresOn: redundancy.expiresOn,
692 files: filesRedundancies,
693 streamingPlaylists: streamingPlaylistsRedundancies
699 if (this.VideoFile?.Video) return this.VideoFile.Video
701 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
707 const video = this.getVideo()
708 if (!video) return undefined
714 return !!this.strategy
717 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
718 if (this.VideoStreamingPlaylist) {
721 type: 'CacheFile' as 'CacheFile',
722 object: this.VideoStreamingPlaylist.Video.url,
723 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
726 mediaType: 'application/x-mpegURL',
734 type: 'CacheFile' as 'CacheFile',
735 object: this.VideoFile.Video.url,
736 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
739 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
741 height: this.VideoFile.resolution,
742 size: this.VideoFile.size,
743 fps: this.VideoFile.fps
748 // Don't include video files we already duplicated
749 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
750 const notIn = literal(
752 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
753 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
754 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
756 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
757 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
758 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
769 private static buildServerRedundancyInclude () {
772 model: VideoChannelModel.unscoped(),
777 model: ActorModel.unscoped(),
782 model: ServerModel.unscoped(),
785 redundancyAllowed: true