]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/models/redundancy/video-redundancy.ts
Fix redundancy remove on host redundancy update
[github/Chocobozzz/PeerTube.git] / server / models / redundancy / video-redundancy.ts
1 import { sample } from 'lodash'
2 import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
3 import {
4 AllowNull,
5 BeforeDestroy,
6 BelongsTo,
7 Column,
8 CreatedAt,
9 DataType,
10 ForeignKey,
11 Is,
12 Model,
13 Scopes,
14 Table,
15 UpdatedAt
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { AttributesOnly } from '@shared/core-utils'
20 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
21 import {
22 FileRedundancyInformation,
23 StreamingPlaylistRedundancyInformation,
24 VideoRedundancy
25 } from '@shared/models/redundancy/video-redundancy.model'
26 import { CacheFileObject, VideoPrivacy } from '../../../shared'
27 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
30 import { logger } from '../../helpers/logger'
31 import { CONFIG } from '../../initializers/config'
32 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
33 import { ActorModel } from '../actor/actor'
34 import { ServerModel } from '../server/server'
35 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
36 import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
37 import { VideoModel } from '../video/video'
38 import { VideoChannelModel } from '../video/video-channel'
39 import { VideoFileModel } from '../video/video-file'
40 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
41
42 export enum ScopeNames {
43 WITH_VIDEO = 'WITH_VIDEO'
44 }
45
46 @Scopes(() => ({
47 [ScopeNames.WITH_VIDEO]: {
48 include: [
49 {
50 model: VideoFileModel,
51 required: false,
52 include: [
53 {
54 model: VideoModel,
55 required: true
56 }
57 ]
58 },
59 {
60 model: VideoStreamingPlaylistModel,
61 required: false,
62 include: [
63 {
64 model: VideoModel,
65 required: true
66 }
67 ]
68 }
69 ]
70 }
71 }))
72
73 @Table({
74 tableName: 'videoRedundancy',
75 indexes: [
76 {
77 fields: [ 'videoFileId' ]
78 },
79 {
80 fields: [ 'actorId' ]
81 },
82 {
83 fields: [ 'expiresOn' ]
84 },
85 {
86 fields: [ 'url' ],
87 unique: true
88 }
89 ]
90 })
91 export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
92
93 @CreatedAt
94 createdAt: Date
95
96 @UpdatedAt
97 updatedAt: Date
98
99 @AllowNull(true)
100 @Column
101 expiresOn: Date
102
103 @AllowNull(false)
104 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
105 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
106 fileUrl: string
107
108 @AllowNull(false)
109 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
110 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
111 url: string
112
113 @AllowNull(true)
114 @Column
115 strategy: string // Only used by us
116
117 @ForeignKey(() => VideoFileModel)
118 @Column
119 videoFileId: number
120
121 @BelongsTo(() => VideoFileModel, {
122 foreignKey: {
123 allowNull: true
124 },
125 onDelete: 'cascade'
126 })
127 VideoFile: VideoFileModel
128
129 @ForeignKey(() => VideoStreamingPlaylistModel)
130 @Column
131 videoStreamingPlaylistId: number
132
133 @BelongsTo(() => VideoStreamingPlaylistModel, {
134 foreignKey: {
135 allowNull: true
136 },
137 onDelete: 'cascade'
138 })
139 VideoStreamingPlaylist: VideoStreamingPlaylistModel
140
141 @ForeignKey(() => ActorModel)
142 @Column
143 actorId: number
144
145 @BelongsTo(() => ActorModel, {
146 foreignKey: {
147 allowNull: false
148 },
149 onDelete: 'cascade'
150 })
151 Actor: ActorModel
152
153 @BeforeDestroy
154 static async removeFile (instance: VideoRedundancyModel) {
155 if (!instance.isOwned()) return
156
157 if (instance.videoFileId) {
158 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
159
160 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
161 logger.info('Removing duplicated video file %s.', logIdentifier)
162
163 videoFile.Video.removeFileAndTorrent(videoFile, true)
164 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
165 }
166
167 if (instance.videoStreamingPlaylistId) {
168 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
169
170 const videoUUID = videoStreamingPlaylist.Video.uuid
171 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
172
173 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
174 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
175 }
176
177 return undefined
178 }
179
180 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
181 const actor = await getServerActor()
182
183 const query = {
184 where: {
185 actorId: actor.id,
186 videoFileId
187 }
188 }
189
190 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
191 }
192
193 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
194 const actor = await getServerActor()
195
196 const queryStreamingPlaylist = {
197 where: {
198 actorId: actor.id
199 },
200 include: [
201 {
202 model: VideoStreamingPlaylistModel.unscoped(),
203 required: true,
204 include: [
205 {
206 model: VideoModel.unscoped(),
207 required: true,
208 where: {
209 id: videoId
210 }
211 }
212 ]
213 }
214 ]
215 }
216
217 const queryFiles = {
218 where: {
219 actorId: actor.id
220 },
221 include: [
222 {
223 model: VideoFileModel,
224 required: true,
225 include: [
226 {
227 model: VideoModel,
228 required: true,
229 where: {
230 id: videoId
231 }
232 }
233 ]
234 }
235 ]
236 }
237
238 return Promise.all([
239 VideoRedundancyModel.findAll(queryStreamingPlaylist),
240 VideoRedundancyModel.findAll(queryFiles)
241 ]).then(([ r1, r2 ]) => r1.concat(r2))
242 }
243
244 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
245 const actor = await getServerActor()
246
247 const query = {
248 where: {
249 actorId: actor.id,
250 videoStreamingPlaylistId
251 }
252 }
253
254 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
255 }
256
257 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
258 const query = {
259 where: { id },
260 transaction
261 }
262
263 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
264 }
265
266 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
267 const query = {
268 where: {
269 url
270 },
271 transaction
272 }
273
274 return VideoRedundancyModel.findOne(query)
275 }
276
277 static async isLocalByVideoUUIDExists (uuid: string) {
278 const actor = await getServerActor()
279
280 const query = {
281 raw: true,
282 attributes: [ 'id' ],
283 where: {
284 actorId: actor.id
285 },
286 include: [
287 {
288 attributes: [],
289 model: VideoFileModel,
290 required: true,
291 include: [
292 {
293 attributes: [],
294 model: VideoModel,
295 required: true,
296 where: {
297 uuid
298 }
299 }
300 ]
301 }
302 ]
303 }
304
305 return VideoRedundancyModel.findOne(query)
306 .then(r => !!r)
307 }
308
309 static async getVideoSample (p: Promise<VideoModel[]>) {
310 const rows = await p
311 if (rows.length === 0) return undefined
312
313 const ids = rows.map(r => r.id)
314 const id = sample(ids)
315
316 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
317 }
318
319 static async findMostViewToDuplicate (randomizedFactor: number) {
320 const peertubeActor = await getServerActor()
321
322 // On VideoModel!
323 const query = {
324 attributes: [ 'id', 'views' ],
325 limit: randomizedFactor,
326 order: getVideoSort('-views'),
327 where: {
328 privacy: VideoPrivacy.PUBLIC,
329 isLive: false,
330 ...this.buildVideoIdsForDuplication(peertubeActor)
331 },
332 include: [
333 VideoRedundancyModel.buildServerRedundancyInclude()
334 ]
335 }
336
337 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
338 }
339
340 static async findTrendingToDuplicate (randomizedFactor: number) {
341 const peertubeActor = await getServerActor()
342
343 // On VideoModel!
344 const query = {
345 attributes: [ 'id', 'views' ],
346 subQuery: false,
347 group: 'VideoModel.id',
348 limit: randomizedFactor,
349 order: getVideoSort('-trending'),
350 where: {
351 privacy: VideoPrivacy.PUBLIC,
352 isLive: false,
353 ...this.buildVideoIdsForDuplication(peertubeActor)
354 },
355 include: [
356 VideoRedundancyModel.buildServerRedundancyInclude(),
357
358 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
359 ]
360 }
361
362 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
363 }
364
365 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
366 const peertubeActor = await getServerActor()
367
368 // On VideoModel!
369 const query = {
370 attributes: [ 'id', 'publishedAt' ],
371 limit: randomizedFactor,
372 order: getVideoSort('-publishedAt'),
373 where: {
374 privacy: VideoPrivacy.PUBLIC,
375 isLive: false,
376 views: {
377 [Op.gte]: minViews
378 },
379 ...this.buildVideoIdsForDuplication(peertubeActor)
380 },
381 include: [
382 VideoRedundancyModel.buildServerRedundancyInclude(),
383
384 // Required by publishedAt sort
385 {
386 model: ScheduleVideoUpdateModel.unscoped(),
387 required: false
388 }
389 ]
390 }
391
392 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
393 }
394
395 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
396 const expiredDate = new Date()
397 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
398
399 const actor = await getServerActor()
400
401 const query = {
402 where: {
403 actorId: actor.id,
404 strategy,
405 createdAt: {
406 [Op.lt]: expiredDate
407 }
408 }
409 }
410
411 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
412 }
413
414 static async listLocalExpired () {
415 const actor = await getServerActor()
416
417 const query = {
418 where: {
419 actorId: actor.id,
420 expiresOn: {
421 [Op.lt]: new Date()
422 }
423 }
424 }
425
426 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
427 }
428
429 static async listRemoteExpired () {
430 const actor = await getServerActor()
431
432 const query = {
433 where: {
434 actorId: {
435 [Op.ne]: actor.id
436 },
437 expiresOn: {
438 [Op.lt]: new Date(),
439 [Op.ne]: null
440 }
441 }
442 }
443
444 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
445 }
446
447 static async listLocalOfServer (serverId: number) {
448 const actor = await getServerActor()
449 const buildVideoInclude = () => ({
450 model: VideoModel,
451 required: true,
452 include: [
453 {
454 attributes: [],
455 model: VideoChannelModel.unscoped(),
456 required: true,
457 include: [
458 {
459 attributes: [],
460 model: ActorModel.unscoped(),
461 required: true,
462 where: {
463 serverId
464 }
465 }
466 ]
467 }
468 ]
469 })
470
471 const query = {
472 where: {
473 [Op.and]: [
474 {
475 actorId: actor.id
476 },
477 {
478 [Op.or]: [
479 {
480 '$VideoStreamingPlaylist.id$': {
481 [Op.ne]: null
482 }
483 },
484 {
485 '$VideoFile.id$': {
486 [Op.ne]: null
487 }
488 }
489 ]
490 }
491 ]
492 },
493 include: [
494 {
495 model: VideoFileModel.unscoped(),
496 required: false,
497 include: [ buildVideoInclude() ]
498 },
499 {
500 model: VideoStreamingPlaylistModel.unscoped(),
501 required: false,
502 include: [ buildVideoInclude() ]
503 }
504 ]
505 }
506
507 return VideoRedundancyModel.findAll(query)
508 }
509
510 static listForApi (options: {
511 start: number
512 count: number
513 sort: string
514 target: VideoRedundanciesTarget
515 strategy?: string
516 }) {
517 const { start, count, sort, target, strategy } = options
518 const redundancyWhere: WhereOptions = {}
519 const videosWhere: WhereOptions = {}
520 let redundancySqlSuffix = ''
521
522 if (target === 'my-videos') {
523 Object.assign(videosWhere, { remote: false })
524 } else if (target === 'remote-videos') {
525 Object.assign(videosWhere, { remote: true })
526 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
527 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
528 }
529
530 if (strategy) {
531 Object.assign(redundancyWhere, { strategy: strategy })
532 }
533
534 const videoFilterWhere = {
535 [Op.and]: [
536 {
537 [Op.or]: [
538 {
539 id: {
540 [Op.in]: literal(
541 '(' +
542 'SELECT "videoId" FROM "videoFile" ' +
543 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
544 redundancySqlSuffix +
545 ')'
546 )
547 }
548 },
549 {
550 id: {
551 [Op.in]: literal(
552 '(' +
553 'select "videoId" FROM "videoStreamingPlaylist" ' +
554 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
555 redundancySqlSuffix +
556 ')'
557 )
558 }
559 }
560 ]
561 },
562
563 videosWhere
564 ]
565 }
566
567 // /!\ On video model /!\
568 const findOptions = {
569 offset: start,
570 limit: count,
571 order: getSort(sort),
572 include: [
573 {
574 required: false,
575 model: VideoFileModel,
576 include: [
577 {
578 model: VideoRedundancyModel.unscoped(),
579 required: false,
580 where: redundancyWhere
581 }
582 ]
583 },
584 {
585 required: false,
586 model: VideoStreamingPlaylistModel.unscoped(),
587 include: [
588 {
589 model: VideoRedundancyModel.unscoped(),
590 required: false,
591 where: redundancyWhere
592 },
593 {
594 model: VideoFileModel,
595 required: false
596 }
597 ]
598 }
599 ],
600 where: videoFilterWhere
601 }
602
603 // /!\ On video model /!\
604 const countOptions = {
605 where: videoFilterWhere
606 }
607
608 return Promise.all([
609 VideoModel.findAll(findOptions),
610
611 VideoModel.count(countOptions)
612 ]).then(([ data, total ]) => ({ total, data }))
613 }
614
615 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
616 const actor = await getServerActor()
617
618 const sql = `WITH "tmp" AS ` +
619 `(` +
620 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
621 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
622 `FROM "videoRedundancy" AS "videoRedundancy" ` +
623 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
624 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
625 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
626 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
627 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
628 `), ` +
629 `"videoIds" AS (` +
630 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
631 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
632 `) ` +
633 `SELECT ` +
634 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
635 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
636 `COUNT(*) AS "totalVideoFiles" ` +
637 `FROM "tmp"`
638
639 return VideoRedundancyModel.sequelize.query<any>(sql, {
640 replacements: { strategy, actorId: actor.id },
641 type: QueryTypes.SELECT
642 }).then(([ row ]) => ({
643 totalUsed: parseAggregateResult(row.totalUsed),
644 totalVideos: row.totalVideos,
645 totalVideoFiles: row.totalVideoFiles
646 }))
647 }
648
649 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
650 const filesRedundancies: FileRedundancyInformation[] = []
651 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
652
653 for (const file of video.VideoFiles) {
654 for (const redundancy of file.RedundancyVideos) {
655 filesRedundancies.push({
656 id: redundancy.id,
657 fileUrl: redundancy.fileUrl,
658 strategy: redundancy.strategy,
659 createdAt: redundancy.createdAt,
660 updatedAt: redundancy.updatedAt,
661 expiresOn: redundancy.expiresOn,
662 size: file.size
663 })
664 }
665 }
666
667 for (const playlist of video.VideoStreamingPlaylists) {
668 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
669
670 for (const redundancy of playlist.RedundancyVideos) {
671 streamingPlaylistsRedundancies.push({
672 id: redundancy.id,
673 fileUrl: redundancy.fileUrl,
674 strategy: redundancy.strategy,
675 createdAt: redundancy.createdAt,
676 updatedAt: redundancy.updatedAt,
677 expiresOn: redundancy.expiresOn,
678 size
679 })
680 }
681 }
682
683 return {
684 id: video.id,
685 name: video.name,
686 url: video.url,
687 uuid: video.uuid,
688
689 redundancies: {
690 files: filesRedundancies,
691 streamingPlaylists: streamingPlaylistsRedundancies
692 }
693 }
694 }
695
696 getVideo () {
697 if (this.VideoFile?.Video) return this.VideoFile.Video
698
699 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
700
701 return undefined
702 }
703
704 isOwned () {
705 return !!this.strategy
706 }
707
708 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
709 if (this.VideoStreamingPlaylist) {
710 return {
711 id: this.url,
712 type: 'CacheFile' as 'CacheFile',
713 object: this.VideoStreamingPlaylist.Video.url,
714 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
715 url: {
716 type: 'Link',
717 mediaType: 'application/x-mpegURL',
718 href: this.fileUrl
719 }
720 }
721 }
722
723 return {
724 id: this.url,
725 type: 'CacheFile' as 'CacheFile',
726 object: this.VideoFile.Video.url,
727 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
728 url: {
729 type: 'Link',
730 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
731 href: this.fileUrl,
732 height: this.VideoFile.resolution,
733 size: this.VideoFile.size,
734 fps: this.VideoFile.fps
735 }
736 }
737 }
738
739 // Don't include video files we already duplicated
740 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
741 const notIn = literal(
742 '(' +
743 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
744 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
745 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
746 `UNION ` +
747 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
748 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
749 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
750 ')'
751 )
752
753 return {
754 id: {
755 [Op.notIn]: notIn
756 }
757 }
758 }
759
760 private static buildServerRedundancyInclude () {
761 return {
762 attributes: [],
763 model: VideoChannelModel.unscoped(),
764 required: true,
765 include: [
766 {
767 attributes: [],
768 model: ActorModel.unscoped(),
769 required: true,
770 include: [
771 {
772 attributes: [],
773 model: ServerModel.unscoped(),
774 required: true,
775 where: {
776 redundancyAllowed: true
777 }
778 }
779 ]
780 }
781 ]
782 }
783 }
784 }