aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/models/redundancy/video-redundancy.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/models/redundancy/video-redundancy.ts')
-rw-r--r--server/models/redundancy/video-redundancy.ts793
1 files changed, 0 insertions, 793 deletions
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
deleted file mode 100644
index cebf47dfd..000000000
--- a/server/models/redundancy/video-redundancy.ts
+++ /dev/null
@@ -1,793 +0,0 @@
1import { sample } from 'lodash'
2import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
3import {
4 AllowNull,
5 BeforeDestroy,
6 BelongsTo,
7 Column,
8 CreatedAt,
9 DataType,
10 ForeignKey,
11 Is,
12 Model,
13 Scopes,
14 Table,
15 UpdatedAt
16} from 'sequelize-typescript'
17import { getServerActor } from '@server/models/application/application'
18import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19import {
20 CacheFileObject,
21 FileRedundancyInformation,
22 StreamingPlaylistRedundancyInformation,
23 VideoPrivacy,
24 VideoRedundanciesTarget,
25 VideoRedundancy,
26 VideoRedundancyStrategy,
27 VideoRedundancyStrategyWithManual
28} from '@shared/models'
29import { AttributesOnly } from '@shared/typescript-utils'
30import { isTestInstance } from '../../helpers/core-utils'
31import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
32import { logger } from '../../helpers/logger'
33import { CONFIG } from '../../initializers/config'
34import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
35import { ActorModel } from '../actor/actor'
36import { ServerModel } from '../server/server'
37import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared'
38import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
39import { VideoModel } from '../video/video'
40import { VideoChannelModel } from '../video/video-channel'
41import { VideoFileModel } from '../video/video-file'
42import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
43
44export enum ScopeNames {
45 WITH_VIDEO = 'WITH_VIDEO'
46}
47
48@Scopes(() => ({
49 [ScopeNames.WITH_VIDEO]: {
50 include: [
51 {
52 model: VideoFileModel,
53 required: false,
54 include: [
55 {
56 model: VideoModel,
57 required: true
58 }
59 ]
60 },
61 {
62 model: VideoStreamingPlaylistModel,
63 required: false,
64 include: [
65 {
66 model: VideoModel,
67 required: true
68 }
69 ]
70 }
71 ]
72 }
73}))
74
75@Table({
76 tableName: 'videoRedundancy',
77 indexes: [
78 {
79 fields: [ 'videoFileId' ]
80 },
81 {
82 fields: [ 'actorId' ]
83 },
84 {
85 fields: [ 'expiresOn' ]
86 },
87 {
88 fields: [ 'url' ],
89 unique: true
90 }
91 ]
92})
93export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
94
95 @CreatedAt
96 createdAt: Date
97
98 @UpdatedAt
99 updatedAt: Date
100
101 @AllowNull(true)
102 @Column
103 expiresOn: Date
104
105 @AllowNull(false)
106 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
107 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
108 fileUrl: string
109
110 @AllowNull(false)
111 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
112 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
113 url: string
114
115 @AllowNull(true)
116 @Column
117 strategy: string // Only used by us
118
119 @ForeignKey(() => VideoFileModel)
120 @Column
121 videoFileId: number
122
123 @BelongsTo(() => VideoFileModel, {
124 foreignKey: {
125 allowNull: true
126 },
127 onDelete: 'cascade'
128 })
129 VideoFile: VideoFileModel
130
131 @ForeignKey(() => VideoStreamingPlaylistModel)
132 @Column
133 videoStreamingPlaylistId: number
134
135 @BelongsTo(() => VideoStreamingPlaylistModel, {
136 foreignKey: {
137 allowNull: true
138 },
139 onDelete: 'cascade'
140 })
141 VideoStreamingPlaylist: VideoStreamingPlaylistModel
142
143 @ForeignKey(() => ActorModel)
144 @Column
145 actorId: number
146
147 @BelongsTo(() => ActorModel, {
148 foreignKey: {
149 allowNull: false
150 },
151 onDelete: 'cascade'
152 })
153 Actor: ActorModel
154
155 @BeforeDestroy
156 static async removeFile (instance: VideoRedundancyModel) {
157 if (!instance.isOwned()) return
158
159 if (instance.videoFileId) {
160 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
161
162 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
163 logger.info('Removing duplicated video file %s.', logIdentifier)
164
165 videoFile.Video.removeWebVideoFile(videoFile, true)
166 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
167 }
168
169 if (instance.videoStreamingPlaylistId) {
170 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
171
172 const videoUUID = videoStreamingPlaylist.Video.uuid
173 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
174
175 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
176 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
177 }
178
179 return undefined
180 }
181
182 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
183 const actor = await getServerActor()
184
185 const query = {
186 where: {
187 actorId: actor.id,
188 videoFileId
189 }
190 }
191
192 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
193 }
194
195 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
196 const actor = await getServerActor()
197
198 const queryStreamingPlaylist = {
199 where: {
200 actorId: actor.id
201 },
202 include: [
203 {
204 model: VideoStreamingPlaylistModel.unscoped(),
205 required: true,
206 include: [
207 {
208 model: VideoModel.unscoped(),
209 required: true,
210 where: {
211 id: videoId
212 }
213 }
214 ]
215 }
216 ]
217 }
218
219 const queryFiles = {
220 where: {
221 actorId: actor.id
222 },
223 include: [
224 {
225 model: VideoFileModel,
226 required: true,
227 include: [
228 {
229 model: VideoModel,
230 required: true,
231 where: {
232 id: videoId
233 }
234 }
235 ]
236 }
237 ]
238 }
239
240 return Promise.all([
241 VideoRedundancyModel.findAll(queryStreamingPlaylist),
242 VideoRedundancyModel.findAll(queryFiles)
243 ]).then(([ r1, r2 ]) => r1.concat(r2))
244 }
245
246 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
247 const actor = await getServerActor()
248
249 const query = {
250 where: {
251 actorId: actor.id,
252 videoStreamingPlaylistId
253 }
254 }
255
256 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
257 }
258
259 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
260 const query = {
261 where: { id },
262 transaction
263 }
264
265 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
266 }
267
268 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
269 const query = {
270 where: {
271 url
272 },
273 transaction
274 }
275
276 return VideoRedundancyModel.findOne(query)
277 }
278
279 static async isLocalByVideoUUIDExists (uuid: string) {
280 const actor = await getServerActor()
281
282 const query = {
283 raw: true,
284 attributes: [ 'id' ],
285 where: {
286 actorId: actor.id
287 },
288 include: [
289 {
290 attributes: [],
291 model: VideoFileModel,
292 required: true,
293 include: [
294 {
295 attributes: [],
296 model: VideoModel,
297 required: true,
298 where: {
299 uuid
300 }
301 }
302 ]
303 }
304 ]
305 }
306
307 return VideoRedundancyModel.findOne(query)
308 .then(r => !!r)
309 }
310
311 static async getVideoSample (p: Promise<VideoModel[]>) {
312 const rows = await p
313 if (rows.length === 0) return undefined
314
315 const ids = rows.map(r => r.id)
316 const id = sample(ids)
317
318 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
319 }
320
321 static async findMostViewToDuplicate (randomizedFactor: number) {
322 const peertubeActor = await getServerActor()
323
324 // On VideoModel!
325 const query = {
326 attributes: [ 'id', 'views' ],
327 limit: randomizedFactor,
328 order: getVideoSort('-views'),
329 where: {
330 privacy: VideoPrivacy.PUBLIC,
331 isLive: false,
332 ...this.buildVideoIdsForDuplication(peertubeActor)
333 },
334 include: [
335 VideoRedundancyModel.buildServerRedundancyInclude()
336 ]
337 }
338
339 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
340 }
341
342 static async findTrendingToDuplicate (randomizedFactor: number) {
343 const peertubeActor = await getServerActor()
344
345 // On VideoModel!
346 const query = {
347 attributes: [ 'id', 'views' ],
348 subQuery: false,
349 group: 'VideoModel.id',
350 limit: randomizedFactor,
351 order: getVideoSort('-trending'),
352 where: {
353 privacy: VideoPrivacy.PUBLIC,
354 isLive: false,
355 ...this.buildVideoIdsForDuplication(peertubeActor)
356 },
357 include: [
358 VideoRedundancyModel.buildServerRedundancyInclude(),
359
360 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
361 ]
362 }
363
364 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
365 }
366
367 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
368 const peertubeActor = await getServerActor()
369
370 // On VideoModel!
371 const query = {
372 attributes: [ 'id', 'publishedAt' ],
373 limit: randomizedFactor,
374 order: getVideoSort('-publishedAt'),
375 where: {
376 privacy: VideoPrivacy.PUBLIC,
377 isLive: false,
378 views: {
379 [Op.gte]: minViews
380 },
381 ...this.buildVideoIdsForDuplication(peertubeActor)
382 },
383 include: [
384 VideoRedundancyModel.buildServerRedundancyInclude(),
385
386 // Required by publishedAt sort
387 {
388 model: ScheduleVideoUpdateModel.unscoped(),
389 required: false
390 }
391 ]
392 }
393
394 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
395 }
396
397 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
398 const expiredDate = new Date()
399 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
400
401 const actor = await getServerActor()
402
403 const query = {
404 where: {
405 actorId: actor.id,
406 strategy,
407 createdAt: {
408 [Op.lt]: expiredDate
409 }
410 }
411 }
412
413 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
414 }
415
416 static async listLocalExpired (): Promise<MVideoRedundancyVideo[]> {
417 const actor = await getServerActor()
418
419 const query = {
420 where: {
421 actorId: actor.id,
422 expiresOn: {
423 [Op.lt]: new Date()
424 }
425 }
426 }
427
428 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
429 }
430
431 static async listRemoteExpired () {
432 const actor = await getServerActor()
433
434 const query = {
435 where: {
436 actorId: {
437 [Op.ne]: actor.id
438 },
439 expiresOn: {
440 [Op.lt]: new Date(),
441 [Op.ne]: null
442 }
443 }
444 }
445
446 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
447 }
448
449 static async listLocalOfServer (serverId: number) {
450 const actor = await getServerActor()
451 const buildVideoInclude = () => ({
452 model: VideoModel,
453 required: true,
454 include: [
455 {
456 attributes: [],
457 model: VideoChannelModel.unscoped(),
458 required: true,
459 include: [
460 {
461 attributes: [],
462 model: ActorModel.unscoped(),
463 required: true,
464 where: {
465 serverId
466 }
467 }
468 ]
469 }
470 ]
471 })
472
473 const query = {
474 where: {
475 [Op.and]: [
476 {
477 actorId: actor.id
478 },
479 {
480 [Op.or]: [
481 {
482 '$VideoStreamingPlaylist.id$': {
483 [Op.ne]: null
484 }
485 },
486 {
487 '$VideoFile.id$': {
488 [Op.ne]: null
489 }
490 }
491 ]
492 }
493 ]
494 },
495 include: [
496 {
497 model: VideoFileModel.unscoped(),
498 required: false,
499 include: [ buildVideoInclude() ]
500 },
501 {
502 model: VideoStreamingPlaylistModel.unscoped(),
503 required: false,
504 include: [ buildVideoInclude() ]
505 }
506 ]
507 }
508
509 return VideoRedundancyModel.findAll(query)
510 }
511
512 static listForApi (options: {
513 start: number
514 count: number
515 sort: string
516 target: VideoRedundanciesTarget
517 strategy?: string
518 }) {
519 const { start, count, sort, target, strategy } = options
520 const redundancyWhere: WhereOptions = {}
521 const videosWhere: WhereOptions = {}
522 let redundancySqlSuffix = ''
523
524 if (target === 'my-videos') {
525 Object.assign(videosWhere, { remote: false })
526 } else if (target === 'remote-videos') {
527 Object.assign(videosWhere, { remote: true })
528 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
529 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
530 }
531
532 if (strategy) {
533 Object.assign(redundancyWhere, { strategy })
534 }
535
536 const videoFilterWhere = {
537 [Op.and]: [
538 {
539 [Op.or]: [
540 {
541 id: {
542 [Op.in]: literal(
543 '(' +
544 'SELECT "videoId" FROM "videoFile" ' +
545 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
546 redundancySqlSuffix +
547 ')'
548 )
549 }
550 },
551 {
552 id: {
553 [Op.in]: literal(
554 '(' +
555 'select "videoId" FROM "videoStreamingPlaylist" ' +
556 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
557 redundancySqlSuffix +
558 ')'
559 )
560 }
561 }
562 ]
563 },
564
565 videosWhere
566 ]
567 }
568
569 // /!\ On video model /!\
570 const findOptions = {
571 offset: start,
572 limit: count,
573 order: getSort(sort),
574 include: [
575 {
576 required: false,
577 model: VideoFileModel,
578 include: [
579 {
580 model: VideoRedundancyModel.unscoped(),
581 required: false,
582 where: redundancyWhere
583 }
584 ]
585 },
586 {
587 required: false,
588 model: VideoStreamingPlaylistModel.unscoped(),
589 include: [
590 {
591 model: VideoRedundancyModel.unscoped(),
592 required: false,
593 where: redundancyWhere
594 },
595 {
596 model: VideoFileModel,
597 required: false
598 }
599 ]
600 }
601 ],
602 where: videoFilterWhere
603 }
604
605 // /!\ On video model /!\
606 const countOptions = {
607 where: videoFilterWhere
608 }
609
610 return Promise.all([
611 VideoModel.findAll(findOptions),
612
613 VideoModel.count(countOptions)
614 ]).then(([ data, total ]) => ({ total, data }))
615 }
616
617 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
618 const actor = await getServerActor()
619
620 const sql = `WITH "tmp" AS ` +
621 `(` +
622 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
623 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
624 `FROM "videoRedundancy" AS "videoRedundancy" ` +
625 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
626 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
627 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
628 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
629 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
630 `), ` +
631 `"videoIds" AS (` +
632 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
633 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
634 `) ` +
635 `SELECT ` +
636 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
637 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
638 `COUNT(*) AS "totalVideoFiles" ` +
639 `FROM "tmp"`
640
641 return VideoRedundancyModel.sequelize.query<any>(sql, {
642 replacements: { strategy, actorId: actor.id },
643 type: QueryTypes.SELECT
644 }).then(([ row ]) => ({
645 totalUsed: parseAggregateResult(row.totalUsed),
646 totalVideos: row.totalVideos,
647 totalVideoFiles: row.totalVideoFiles
648 }))
649 }
650
651 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
652 const filesRedundancies: FileRedundancyInformation[] = []
653 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
654
655 for (const file of video.VideoFiles) {
656 for (const redundancy of file.RedundancyVideos) {
657 filesRedundancies.push({
658 id: redundancy.id,
659 fileUrl: redundancy.fileUrl,
660 strategy: redundancy.strategy,
661 createdAt: redundancy.createdAt,
662 updatedAt: redundancy.updatedAt,
663 expiresOn: redundancy.expiresOn,
664 size: file.size
665 })
666 }
667 }
668
669 for (const playlist of video.VideoStreamingPlaylists) {
670 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
671
672 for (const redundancy of playlist.RedundancyVideos) {
673 streamingPlaylistsRedundancies.push({
674 id: redundancy.id,
675 fileUrl: redundancy.fileUrl,
676 strategy: redundancy.strategy,
677 createdAt: redundancy.createdAt,
678 updatedAt: redundancy.updatedAt,
679 expiresOn: redundancy.expiresOn,
680 size
681 })
682 }
683 }
684
685 return {
686 id: video.id,
687 name: video.name,
688 url: video.url,
689 uuid: video.uuid,
690
691 redundancies: {
692 files: filesRedundancies,
693 streamingPlaylists: streamingPlaylistsRedundancies
694 }
695 }
696 }
697
698 getVideo () {
699 if (this.VideoFile?.Video) return this.VideoFile.Video
700
701 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
702
703 return undefined
704 }
705
706 getVideoUUID () {
707 const video = this.getVideo()
708 if (!video) return undefined
709
710 return video.uuid
711 }
712
713 isOwned () {
714 return !!this.strategy
715 }
716
717 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
718 if (this.VideoStreamingPlaylist) {
719 return {
720 id: this.url,
721 type: 'CacheFile' as 'CacheFile',
722 object: this.VideoStreamingPlaylist.Video.url,
723 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
724 url: {
725 type: 'Link',
726 mediaType: 'application/x-mpegURL',
727 href: this.fileUrl
728 }
729 }
730 }
731
732 return {
733 id: this.url,
734 type: 'CacheFile' as 'CacheFile',
735 object: this.VideoFile.Video.url,
736 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
737 url: {
738 type: 'Link',
739 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
740 href: this.fileUrl,
741 height: this.VideoFile.resolution,
742 size: this.VideoFile.size,
743 fps: this.VideoFile.fps
744 }
745 }
746 }
747
748 // Don't include video files we already duplicated
749 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
750 const notIn = literal(
751 '(' +
752 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
753 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
754 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
755 `UNION ` +
756 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
757 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
758 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
759 ')'
760 )
761
762 return {
763 id: {
764 [Op.notIn]: notIn
765 }
766 }
767 }
768
769 private static buildServerRedundancyInclude () {
770 return {
771 attributes: [],
772 model: VideoChannelModel.unscoped(),
773 required: true,
774 include: [
775 {
776 attributes: [],
777 model: ActorModel.unscoped(),
778 required: true,
779 include: [
780 {
781 attributes: [],
782 model: ServerModel.unscoped(),
783 required: true,
784 where: {
785 redundancyAllowed: true
786 }
787 }
788 ]
789 }
790 ]
791 }
792 }
793}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-16 05:14:48 +0000