]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/models/redundancy/video-redundancy.ts
Fix tests
[github/Chocobozzz/PeerTube.git] / server / models / redundancy / video-redundancy.ts
1 import { sample } from 'lodash'
2 import { FindOptions, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
3 import {
4 AllowNull,
5 BeforeDestroy,
6 BelongsTo,
7 Column,
8 CreatedAt,
9 DataType,
10 ForeignKey,
11 Is,
12 Model,
13 Scopes,
14 Table,
15 UpdatedAt
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
20 import {
21 FileRedundancyInformation,
22 StreamingPlaylistRedundancyInformation,
23 VideoRedundancy
24 } from '@shared/models/redundancy/video-redundancy.model'
25 import { CacheFileObject, VideoPrivacy } from '../../../shared'
26 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
27 import { isTestInstance } from '../../helpers/core-utils'
28 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
29 import { logger } from '../../helpers/logger'
30 import { CONFIG } from '../../initializers/config'
31 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
32 import { ActorModel } from '../activitypub/actor'
33 import { ServerModel } from '../server/server'
34 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
35 import { VideoModel } from '../video/video'
36 import { VideoChannelModel } from '../video/video-channel'
37 import { VideoFileModel } from '../video/video-file'
38 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
39
40 export enum ScopeNames {
41 WITH_VIDEO = 'WITH_VIDEO'
42 }
43
44 @Scopes(() => ({
45 [ScopeNames.WITH_VIDEO]: {
46 include: [
47 {
48 model: VideoFileModel,
49 required: false,
50 include: [
51 {
52 model: VideoModel,
53 required: true
54 }
55 ]
56 },
57 {
58 model: VideoStreamingPlaylistModel,
59 required: false,
60 include: [
61 {
62 model: VideoModel,
63 required: true
64 }
65 ]
66 }
67 ]
68 }
69 }))
70
71 @Table({
72 tableName: 'videoRedundancy',
73 indexes: [
74 {
75 fields: [ 'videoFileId' ]
76 },
77 {
78 fields: [ 'actorId' ]
79 },
80 {
81 fields: [ 'url' ],
82 unique: true
83 }
84 ]
85 })
86 export class VideoRedundancyModel extends Model {
87
88 @CreatedAt
89 createdAt: Date
90
91 @UpdatedAt
92 updatedAt: Date
93
94 @AllowNull(true)
95 @Column
96 expiresOn: Date
97
98 @AllowNull(false)
99 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
100 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
101 fileUrl: string
102
103 @AllowNull(false)
104 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
105 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
106 url: string
107
108 @AllowNull(true)
109 @Column
110 strategy: string // Only used by us
111
112 @ForeignKey(() => VideoFileModel)
113 @Column
114 videoFileId: number
115
116 @BelongsTo(() => VideoFileModel, {
117 foreignKey: {
118 allowNull: true
119 },
120 onDelete: 'cascade'
121 })
122 VideoFile: VideoFileModel
123
124 @ForeignKey(() => VideoStreamingPlaylistModel)
125 @Column
126 videoStreamingPlaylistId: number
127
128 @BelongsTo(() => VideoStreamingPlaylistModel, {
129 foreignKey: {
130 allowNull: true
131 },
132 onDelete: 'cascade'
133 })
134 VideoStreamingPlaylist: VideoStreamingPlaylistModel
135
136 @ForeignKey(() => ActorModel)
137 @Column
138 actorId: number
139
140 @BelongsTo(() => ActorModel, {
141 foreignKey: {
142 allowNull: false
143 },
144 onDelete: 'cascade'
145 })
146 Actor: ActorModel
147
148 @BeforeDestroy
149 static async removeFile (instance: VideoRedundancyModel) {
150 if (!instance.isOwned()) return
151
152 if (instance.videoFileId) {
153 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
154
155 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
156 logger.info('Removing duplicated video file %s.', logIdentifier)
157
158 videoFile.Video.removeFile(videoFile, true)
159 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
160 }
161
162 if (instance.videoStreamingPlaylistId) {
163 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
164
165 const videoUUID = videoStreamingPlaylist.Video.uuid
166 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
167
168 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
169 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
170 }
171
172 return undefined
173 }
174
175 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
176 const actor = await getServerActor()
177
178 const query = {
179 where: {
180 actorId: actor.id,
181 videoFileId
182 }
183 }
184
185 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
186 }
187
188 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
189 const actor = await getServerActor()
190
191 const query = {
192 where: {
193 actorId: actor.id,
194 videoStreamingPlaylistId
195 }
196 }
197
198 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
199 }
200
201 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
202 const query = {
203 where: { id },
204 transaction
205 }
206
207 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
208 }
209
210 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
211 const query = {
212 where: {
213 url
214 },
215 transaction
216 }
217
218 return VideoRedundancyModel.findOne(query)
219 }
220
221 static async isLocalByVideoUUIDExists (uuid: string) {
222 const actor = await getServerActor()
223
224 const query = {
225 raw: true,
226 attributes: [ 'id' ],
227 where: {
228 actorId: actor.id
229 },
230 include: [
231 {
232 attributes: [],
233 model: VideoFileModel,
234 required: true,
235 include: [
236 {
237 attributes: [],
238 model: VideoModel,
239 required: true,
240 where: {
241 uuid
242 }
243 }
244 ]
245 }
246 ]
247 }
248
249 return VideoRedundancyModel.findOne(query)
250 .then(r => !!r)
251 }
252
253 static async getVideoSample (p: Promise<VideoModel[]>) {
254 const rows = await p
255 if (rows.length === 0) return undefined
256
257 const ids = rows.map(r => r.id)
258 const id = sample(ids)
259
260 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
261 }
262
263 static async findMostViewToDuplicate (randomizedFactor: number) {
264 const peertubeActor = await getServerActor()
265
266 // On VideoModel!
267 const query = {
268 attributes: [ 'id', 'views' ],
269 limit: randomizedFactor,
270 order: getVideoSort('-views'),
271 where: {
272 privacy: VideoPrivacy.PUBLIC,
273 isLive: false,
274 ...this.buildVideoIdsForDuplication(peertubeActor)
275 },
276 include: [
277 VideoRedundancyModel.buildServerRedundancyInclude()
278 ]
279 }
280
281 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
282 }
283
284 static async findTrendingToDuplicate (randomizedFactor: number) {
285 const peertubeActor = await getServerActor()
286
287 // On VideoModel!
288 const query = {
289 attributes: [ 'id', 'views' ],
290 subQuery: false,
291 group: 'VideoModel.id',
292 limit: randomizedFactor,
293 order: getVideoSort('-trending'),
294 where: {
295 privacy: VideoPrivacy.PUBLIC,
296 isLive: false,
297 ...this.buildVideoIdsForDuplication(peertubeActor)
298 },
299 include: [
300 VideoRedundancyModel.buildServerRedundancyInclude(),
301
302 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
303 ]
304 }
305
306 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
307 }
308
309 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
310 const peertubeActor = await getServerActor()
311
312 // On VideoModel!
313 const query = {
314 attributes: [ 'id', 'publishedAt' ],
315 limit: randomizedFactor,
316 order: getVideoSort('-publishedAt'),
317 where: {
318 privacy: VideoPrivacy.PUBLIC,
319 isLive: false,
320 views: {
321 [Op.gte]: minViews
322 },
323 ...this.buildVideoIdsForDuplication(peertubeActor)
324 },
325 include: [
326 VideoRedundancyModel.buildServerRedundancyInclude()
327 ]
328 }
329
330 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
331 }
332
333 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
334 const expiredDate = new Date()
335 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
336
337 const actor = await getServerActor()
338
339 const query = {
340 where: {
341 actorId: actor.id,
342 strategy,
343 createdAt: {
344 [Op.lt]: expiredDate
345 }
346 }
347 }
348
349 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
350 }
351
352 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
353 const actor = await getServerActor()
354 const redundancyInclude = {
355 attributes: [],
356 model: VideoRedundancyModel,
357 required: true,
358 where: {
359 actorId: actor.id,
360 strategy
361 }
362 }
363
364 const queryFiles: FindOptions = {
365 include: [ redundancyInclude ]
366 }
367
368 const queryStreamingPlaylists: FindOptions = {
369 include: [
370 {
371 attributes: [],
372 model: VideoModel.unscoped(),
373 required: true,
374 include: [
375 {
376 required: true,
377 attributes: [],
378 model: VideoStreamingPlaylistModel.unscoped(),
379 include: [
380 redundancyInclude
381 ]
382 }
383 ]
384 }
385 ]
386 }
387
388 return Promise.all([
389 VideoFileModel.aggregate('size', 'SUM', queryFiles),
390 VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
391 ]).then(([ r1, r2 ]) => {
392 return parseAggregateResult(r1) + parseAggregateResult(r2)
393 })
394 }
395
396 static async listLocalExpired () {
397 const actor = await getServerActor()
398
399 const query = {
400 where: {
401 actorId: actor.id,
402 expiresOn: {
403 [Op.lt]: new Date()
404 }
405 }
406 }
407
408 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
409 }
410
411 static async listRemoteExpired () {
412 const actor = await getServerActor()
413
414 const query = {
415 where: {
416 actorId: {
417 [Op.ne]: actor.id
418 },
419 expiresOn: {
420 [Op.lt]: new Date(),
421 [Op.ne]: null
422 }
423 }
424 }
425
426 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
427 }
428
429 static async listLocalOfServer (serverId: number) {
430 const actor = await getServerActor()
431 const buildVideoInclude = () => ({
432 model: VideoModel,
433 required: true,
434 include: [
435 {
436 attributes: [],
437 model: VideoChannelModel.unscoped(),
438 required: true,
439 include: [
440 {
441 attributes: [],
442 model: ActorModel.unscoped(),
443 required: true,
444 where: {
445 serverId
446 }
447 }
448 ]
449 }
450 ]
451 })
452
453 const query = {
454 where: {
455 actorId: actor.id
456 },
457 include: [
458 {
459 model: VideoFileModel,
460 required: false,
461 include: [ buildVideoInclude() ]
462 },
463 {
464 model: VideoStreamingPlaylistModel,
465 required: false,
466 include: [ buildVideoInclude() ]
467 }
468 ]
469 }
470
471 return VideoRedundancyModel.findAll(query)
472 }
473
474 static listForApi (options: {
475 start: number
476 count: number
477 sort: string
478 target: VideoRedundanciesTarget
479 strategy?: string
480 }) {
481 const { start, count, sort, target, strategy } = options
482 const redundancyWhere: WhereOptions = {}
483 const videosWhere: WhereOptions = {}
484 let redundancySqlSuffix = ''
485
486 if (target === 'my-videos') {
487 Object.assign(videosWhere, { remote: false })
488 } else if (target === 'remote-videos') {
489 Object.assign(videosWhere, { remote: true })
490 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
491 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
492 }
493
494 if (strategy) {
495 Object.assign(redundancyWhere, { strategy: strategy })
496 }
497
498 const videoFilterWhere = {
499 [Op.and]: [
500 {
501 [Op.or]: [
502 {
503 id: {
504 [Op.in]: literal(
505 '(' +
506 'SELECT "videoId" FROM "videoFile" ' +
507 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
508 redundancySqlSuffix +
509 ')'
510 )
511 }
512 },
513 {
514 id: {
515 [Op.in]: literal(
516 '(' +
517 'select "videoId" FROM "videoStreamingPlaylist" ' +
518 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
519 redundancySqlSuffix +
520 ')'
521 )
522 }
523 }
524 ]
525 },
526
527 videosWhere
528 ]
529 }
530
531 // /!\ On video model /!\
532 const findOptions = {
533 offset: start,
534 limit: count,
535 order: getSort(sort),
536 include: [
537 {
538 required: false,
539 model: VideoFileModel,
540 include: [
541 {
542 model: VideoRedundancyModel.unscoped(),
543 required: false,
544 where: redundancyWhere
545 }
546 ]
547 },
548 {
549 required: false,
550 model: VideoStreamingPlaylistModel.unscoped(),
551 include: [
552 {
553 model: VideoRedundancyModel.unscoped(),
554 required: false,
555 where: redundancyWhere
556 },
557 {
558 model: VideoFileModel,
559 required: false
560 }
561 ]
562 }
563 ],
564 where: videoFilterWhere
565 }
566
567 // /!\ On video model /!\
568 const countOptions = {
569 where: videoFilterWhere
570 }
571
572 return Promise.all([
573 VideoModel.findAll(findOptions),
574
575 VideoModel.count(countOptions)
576 ]).then(([ data, total ]) => ({ total, data }))
577 }
578
579 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
580 const actor = await getServerActor()
581
582 const sql = `WITH "tmp" AS ` +
583 `(` +
584 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
585 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
586 `FROM "videoRedundancy" AS "videoRedundancy" ` +
587 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
588 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
589 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
590 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
591 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
592 `), ` +
593 `"videoIds" AS (` +
594 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
595 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
596 `) ` +
597 `SELECT ` +
598 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
599 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
600 `COUNT(*) AS "totalVideoFiles" ` +
601 `FROM "tmp"`
602
603 return VideoRedundancyModel.sequelize.query<any>(sql, {
604 replacements: { strategy, actorId: actor.id },
605 type: QueryTypes.SELECT
606 }).then(([ row ]) => ({
607 totalUsed: parseAggregateResult(row.totalUsed),
608 totalVideos: row.totalVideos,
609 totalVideoFiles: row.totalVideoFiles
610 }))
611 }
612
613 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
614 const filesRedundancies: FileRedundancyInformation[] = []
615 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
616
617 for (const file of video.VideoFiles) {
618 for (const redundancy of file.RedundancyVideos) {
619 filesRedundancies.push({
620 id: redundancy.id,
621 fileUrl: redundancy.fileUrl,
622 strategy: redundancy.strategy,
623 createdAt: redundancy.createdAt,
624 updatedAt: redundancy.updatedAt,
625 expiresOn: redundancy.expiresOn,
626 size: file.size
627 })
628 }
629 }
630
631 for (const playlist of video.VideoStreamingPlaylists) {
632 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
633
634 for (const redundancy of playlist.RedundancyVideos) {
635 streamingPlaylistsRedundancies.push({
636 id: redundancy.id,
637 fileUrl: redundancy.fileUrl,
638 strategy: redundancy.strategy,
639 createdAt: redundancy.createdAt,
640 updatedAt: redundancy.updatedAt,
641 expiresOn: redundancy.expiresOn,
642 size
643 })
644 }
645 }
646
647 return {
648 id: video.id,
649 name: video.name,
650 url: video.url,
651 uuid: video.uuid,
652
653 redundancies: {
654 files: filesRedundancies,
655 streamingPlaylists: streamingPlaylistsRedundancies
656 }
657 }
658 }
659
660 getVideo () {
661 if (this.VideoFile) return this.VideoFile.Video
662
663 if (this.VideoStreamingPlaylist.Video) return this.VideoStreamingPlaylist.Video
664
665 return undefined
666 }
667
668 isOwned () {
669 return !!this.strategy
670 }
671
672 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
673 if (this.VideoStreamingPlaylist) {
674 return {
675 id: this.url,
676 type: 'CacheFile' as 'CacheFile',
677 object: this.VideoStreamingPlaylist.Video.url,
678 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
679 url: {
680 type: 'Link',
681 mediaType: 'application/x-mpegURL',
682 href: this.fileUrl
683 }
684 }
685 }
686
687 return {
688 id: this.url,
689 type: 'CacheFile' as 'CacheFile',
690 object: this.VideoFile.Video.url,
691 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
692 url: {
693 type: 'Link',
694 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
695 href: this.fileUrl,
696 height: this.VideoFile.resolution,
697 size: this.VideoFile.size,
698 fps: this.VideoFile.fps
699 }
700 }
701 }
702
703 // Don't include video files we already duplicated
704 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
705 const notIn = literal(
706 '(' +
707 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
708 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
709 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
710 `UNION ` +
711 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
712 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
713 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
714 ')'
715 )
716
717 return {
718 id: {
719 [Op.notIn]: notIn
720 }
721 }
722 }
723
724 private static buildServerRedundancyInclude () {
725 return {
726 attributes: [],
727 model: VideoChannelModel.unscoped(),
728 required: true,
729 include: [
730 {
731 attributes: [],
732 model: ActorModel.unscoped(),
733 required: true,
734 include: [
735 {
736 attributes: [],
737 model: ServerModel.unscoped(),
738 required: true,
739 where: {
740 redundancyAllowed: true
741 }
742 }
743 ]
744 }
745 ]
746 }
747 }
748 }