]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/models/redundancy/video-redundancy.ts
Merge branch 'develop' into shorter-URLs-channels-accounts
[github/Chocobozzz/PeerTube.git] / server / models / redundancy / video-redundancy.ts
1 import { sample } from 'lodash'
2 import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
3 import {
4 AllowNull,
5 BeforeDestroy,
6 BelongsTo,
7 Column,
8 CreatedAt,
9 DataType,
10 ForeignKey,
11 Is,
12 Model,
13 Scopes,
14 Table,
15 UpdatedAt
16 } from 'sequelize-typescript'
17 import { getServerActor } from '@server/models/application/application'
18 import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
19 import { AttributesOnly } from '@shared/core-utils'
20 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
21 import {
22 FileRedundancyInformation,
23 StreamingPlaylistRedundancyInformation,
24 VideoRedundancy
25 } from '@shared/models/redundancy/video-redundancy.model'
26 import { CacheFileObject, VideoPrivacy } from '../../../shared'
27 import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
30 import { logger } from '../../helpers/logger'
31 import { CONFIG } from '../../initializers/config'
32 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
33 import { ActorModel } from '../actor/actor'
34 import { ServerModel } from '../server/server'
35 import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
36 import { ScheduleVideoUpdateModel } from '../video/schedule-video-update'
37 import { VideoModel } from '../video/video'
38 import { VideoChannelModel } from '../video/video-channel'
39 import { VideoFileModel } from '../video/video-file'
40 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
41
42 export enum ScopeNames {
43 WITH_VIDEO = 'WITH_VIDEO'
44 }
45
46 @Scopes(() => ({
47 [ScopeNames.WITH_VIDEO]: {
48 include: [
49 {
50 model: VideoFileModel,
51 required: false,
52 include: [
53 {
54 model: VideoModel,
55 required: true
56 }
57 ]
58 },
59 {
60 model: VideoStreamingPlaylistModel,
61 required: false,
62 include: [
63 {
64 model: VideoModel,
65 required: true
66 }
67 ]
68 }
69 ]
70 }
71 }))
72
73 @Table({
74 tableName: 'videoRedundancy',
75 indexes: [
76 {
77 fields: [ 'videoFileId' ]
78 },
79 {
80 fields: [ 'actorId' ]
81 },
82 {
83 fields: [ 'url' ],
84 unique: true
85 }
86 ]
87 })
88 export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedundancyModel>>> {
89
90 @CreatedAt
91 createdAt: Date
92
93 @UpdatedAt
94 updatedAt: Date
95
96 @AllowNull(true)
97 @Column
98 expiresOn: Date
99
100 @AllowNull(false)
101 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
102 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
103 fileUrl: string
104
105 @AllowNull(false)
106 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
107 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
108 url: string
109
110 @AllowNull(true)
111 @Column
112 strategy: string // Only used by us
113
114 @ForeignKey(() => VideoFileModel)
115 @Column
116 videoFileId: number
117
118 @BelongsTo(() => VideoFileModel, {
119 foreignKey: {
120 allowNull: true
121 },
122 onDelete: 'cascade'
123 })
124 VideoFile: VideoFileModel
125
126 @ForeignKey(() => VideoStreamingPlaylistModel)
127 @Column
128 videoStreamingPlaylistId: number
129
130 @BelongsTo(() => VideoStreamingPlaylistModel, {
131 foreignKey: {
132 allowNull: true
133 },
134 onDelete: 'cascade'
135 })
136 VideoStreamingPlaylist: VideoStreamingPlaylistModel
137
138 @ForeignKey(() => ActorModel)
139 @Column
140 actorId: number
141
142 @BelongsTo(() => ActorModel, {
143 foreignKey: {
144 allowNull: false
145 },
146 onDelete: 'cascade'
147 })
148 Actor: ActorModel
149
150 @BeforeDestroy
151 static async removeFile (instance: VideoRedundancyModel) {
152 if (!instance.isOwned()) return
153
154 if (instance.videoFileId) {
155 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
156
157 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
158 logger.info('Removing duplicated video file %s.', logIdentifier)
159
160 videoFile.Video.removeFile(videoFile, true)
161 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
162 }
163
164 if (instance.videoStreamingPlaylistId) {
165 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
166
167 const videoUUID = videoStreamingPlaylist.Video.uuid
168 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
169
170 videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
171 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
172 }
173
174 return undefined
175 }
176
177 static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
178 const actor = await getServerActor()
179
180 const query = {
181 where: {
182 actorId: actor.id,
183 videoFileId
184 }
185 }
186
187 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
188 }
189
190 static async listLocalByVideoId (videoId: number): Promise<MVideoRedundancyVideo[]> {
191 const actor = await getServerActor()
192
193 const queryStreamingPlaylist = {
194 where: {
195 actorId: actor.id
196 },
197 include: [
198 {
199 model: VideoStreamingPlaylistModel.unscoped(),
200 required: true,
201 include: [
202 {
203 model: VideoModel.unscoped(),
204 required: true,
205 where: {
206 id: videoId
207 }
208 }
209 ]
210 }
211 ]
212 }
213
214 const queryFiles = {
215 where: {
216 actorId: actor.id
217 },
218 include: [
219 {
220 model: VideoFileModel,
221 required: true,
222 include: [
223 {
224 model: VideoModel,
225 required: true,
226 where: {
227 id: videoId
228 }
229 }
230 ]
231 }
232 ]
233 }
234
235 return Promise.all([
236 VideoRedundancyModel.findAll(queryStreamingPlaylist),
237 VideoRedundancyModel.findAll(queryFiles)
238 ]).then(([ r1, r2 ]) => r1.concat(r2))
239 }
240
241 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
242 const actor = await getServerActor()
243
244 const query = {
245 where: {
246 actorId: actor.id,
247 videoStreamingPlaylistId
248 }
249 }
250
251 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
252 }
253
254 static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MVideoRedundancyVideo> {
255 const query = {
256 where: { id },
257 transaction
258 }
259
260 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
261 }
262
263 static loadByUrl (url: string, transaction?: Transaction): Promise<MVideoRedundancy> {
264 const query = {
265 where: {
266 url
267 },
268 transaction
269 }
270
271 return VideoRedundancyModel.findOne(query)
272 }
273
274 static async isLocalByVideoUUIDExists (uuid: string) {
275 const actor = await getServerActor()
276
277 const query = {
278 raw: true,
279 attributes: [ 'id' ],
280 where: {
281 actorId: actor.id
282 },
283 include: [
284 {
285 attributes: [],
286 model: VideoFileModel,
287 required: true,
288 include: [
289 {
290 attributes: [],
291 model: VideoModel,
292 required: true,
293 where: {
294 uuid
295 }
296 }
297 ]
298 }
299 ]
300 }
301
302 return VideoRedundancyModel.findOne(query)
303 .then(r => !!r)
304 }
305
306 static async getVideoSample (p: Promise<VideoModel[]>) {
307 const rows = await p
308 if (rows.length === 0) return undefined
309
310 const ids = rows.map(r => r.id)
311 const id = sample(ids)
312
313 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
314 }
315
316 static async findMostViewToDuplicate (randomizedFactor: number) {
317 const peertubeActor = await getServerActor()
318
319 // On VideoModel!
320 const query = {
321 attributes: [ 'id', 'views' ],
322 limit: randomizedFactor,
323 order: getVideoSort('-views'),
324 where: {
325 privacy: VideoPrivacy.PUBLIC,
326 isLive: false,
327 ...this.buildVideoIdsForDuplication(peertubeActor)
328 },
329 include: [
330 VideoRedundancyModel.buildServerRedundancyInclude()
331 ]
332 }
333
334 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
335 }
336
337 static async findTrendingToDuplicate (randomizedFactor: number) {
338 const peertubeActor = await getServerActor()
339
340 // On VideoModel!
341 const query = {
342 attributes: [ 'id', 'views' ],
343 subQuery: false,
344 group: 'VideoModel.id',
345 limit: randomizedFactor,
346 order: getVideoSort('-trending'),
347 where: {
348 privacy: VideoPrivacy.PUBLIC,
349 isLive: false,
350 ...this.buildVideoIdsForDuplication(peertubeActor)
351 },
352 include: [
353 VideoRedundancyModel.buildServerRedundancyInclude(),
354
355 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
356 ]
357 }
358
359 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
360 }
361
362 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
363 const peertubeActor = await getServerActor()
364
365 // On VideoModel!
366 const query = {
367 attributes: [ 'id', 'publishedAt' ],
368 limit: randomizedFactor,
369 order: getVideoSort('-publishedAt'),
370 where: {
371 privacy: VideoPrivacy.PUBLIC,
372 isLive: false,
373 views: {
374 [Op.gte]: minViews
375 },
376 ...this.buildVideoIdsForDuplication(peertubeActor)
377 },
378 include: [
379 VideoRedundancyModel.buildServerRedundancyInclude(),
380
381 // Required by publishedAt sort
382 {
383 model: ScheduleVideoUpdateModel.unscoped(),
384 required: false
385 }
386 ]
387 }
388
389 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
390 }
391
392 static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
393 const expiredDate = new Date()
394 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
395
396 const actor = await getServerActor()
397
398 const query = {
399 where: {
400 actorId: actor.id,
401 strategy,
402 createdAt: {
403 [Op.lt]: expiredDate
404 }
405 }
406 }
407
408 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
409 }
410
411 static async listLocalExpired () {
412 const actor = await getServerActor()
413
414 const query = {
415 where: {
416 actorId: actor.id,
417 expiresOn: {
418 [Op.lt]: new Date()
419 }
420 }
421 }
422
423 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
424 }
425
426 static async listRemoteExpired () {
427 const actor = await getServerActor()
428
429 const query = {
430 where: {
431 actorId: {
432 [Op.ne]: actor.id
433 },
434 expiresOn: {
435 [Op.lt]: new Date(),
436 [Op.ne]: null
437 }
438 }
439 }
440
441 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
442 }
443
444 static async listLocalOfServer (serverId: number) {
445 const actor = await getServerActor()
446 const buildVideoInclude = () => ({
447 model: VideoModel,
448 required: true,
449 include: [
450 {
451 attributes: [],
452 model: VideoChannelModel.unscoped(),
453 required: true,
454 include: [
455 {
456 attributes: [],
457 model: ActorModel.unscoped(),
458 required: true,
459 where: {
460 serverId
461 }
462 }
463 ]
464 }
465 ]
466 })
467
468 const query = {
469 where: {
470 actorId: actor.id
471 },
472 include: [
473 {
474 model: VideoFileModel,
475 required: false,
476 include: [ buildVideoInclude() ]
477 },
478 {
479 model: VideoStreamingPlaylistModel,
480 required: false,
481 include: [ buildVideoInclude() ]
482 }
483 ]
484 }
485
486 return VideoRedundancyModel.findAll(query)
487 }
488
489 static listForApi (options: {
490 start: number
491 count: number
492 sort: string
493 target: VideoRedundanciesTarget
494 strategy?: string
495 }) {
496 const { start, count, sort, target, strategy } = options
497 const redundancyWhere: WhereOptions = {}
498 const videosWhere: WhereOptions = {}
499 let redundancySqlSuffix = ''
500
501 if (target === 'my-videos') {
502 Object.assign(videosWhere, { remote: false })
503 } else if (target === 'remote-videos') {
504 Object.assign(videosWhere, { remote: true })
505 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
506 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
507 }
508
509 if (strategy) {
510 Object.assign(redundancyWhere, { strategy: strategy })
511 }
512
513 const videoFilterWhere = {
514 [Op.and]: [
515 {
516 [Op.or]: [
517 {
518 id: {
519 [Op.in]: literal(
520 '(' +
521 'SELECT "videoId" FROM "videoFile" ' +
522 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
523 redundancySqlSuffix +
524 ')'
525 )
526 }
527 },
528 {
529 id: {
530 [Op.in]: literal(
531 '(' +
532 'select "videoId" FROM "videoStreamingPlaylist" ' +
533 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
534 redundancySqlSuffix +
535 ')'
536 )
537 }
538 }
539 ]
540 },
541
542 videosWhere
543 ]
544 }
545
546 // /!\ On video model /!\
547 const findOptions = {
548 offset: start,
549 limit: count,
550 order: getSort(sort),
551 include: [
552 {
553 required: false,
554 model: VideoFileModel,
555 include: [
556 {
557 model: VideoRedundancyModel.unscoped(),
558 required: false,
559 where: redundancyWhere
560 }
561 ]
562 },
563 {
564 required: false,
565 model: VideoStreamingPlaylistModel.unscoped(),
566 include: [
567 {
568 model: VideoRedundancyModel.unscoped(),
569 required: false,
570 where: redundancyWhere
571 },
572 {
573 model: VideoFileModel,
574 required: false
575 }
576 ]
577 }
578 ],
579 where: videoFilterWhere
580 }
581
582 // /!\ On video model /!\
583 const countOptions = {
584 where: videoFilterWhere
585 }
586
587 return Promise.all([
588 VideoModel.findAll(findOptions),
589
590 VideoModel.count(countOptions)
591 ]).then(([ data, total ]) => ({ total, data }))
592 }
593
594 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
595 const actor = await getServerActor()
596
597 const sql = `WITH "tmp" AS ` +
598 `(` +
599 `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
600 `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
601 `FROM "videoRedundancy" AS "videoRedundancy" ` +
602 `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
603 `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
604 `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
605 `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
606 `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
607 `), ` +
608 `"videoIds" AS (` +
609 `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
610 `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
611 `) ` +
612 `SELECT ` +
613 `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
614 `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
615 `COUNT(*) AS "totalVideoFiles" ` +
616 `FROM "tmp"`
617
618 return VideoRedundancyModel.sequelize.query<any>(sql, {
619 replacements: { strategy, actorId: actor.id },
620 type: QueryTypes.SELECT
621 }).then(([ row ]) => ({
622 totalUsed: parseAggregateResult(row.totalUsed),
623 totalVideos: row.totalVideos,
624 totalVideoFiles: row.totalVideoFiles
625 }))
626 }
627
628 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
629 const filesRedundancies: FileRedundancyInformation[] = []
630 const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
631
632 for (const file of video.VideoFiles) {
633 for (const redundancy of file.RedundancyVideos) {
634 filesRedundancies.push({
635 id: redundancy.id,
636 fileUrl: redundancy.fileUrl,
637 strategy: redundancy.strategy,
638 createdAt: redundancy.createdAt,
639 updatedAt: redundancy.updatedAt,
640 expiresOn: redundancy.expiresOn,
641 size: file.size
642 })
643 }
644 }
645
646 for (const playlist of video.VideoStreamingPlaylists) {
647 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
648
649 for (const redundancy of playlist.RedundancyVideos) {
650 streamingPlaylistsRedundancies.push({
651 id: redundancy.id,
652 fileUrl: redundancy.fileUrl,
653 strategy: redundancy.strategy,
654 createdAt: redundancy.createdAt,
655 updatedAt: redundancy.updatedAt,
656 expiresOn: redundancy.expiresOn,
657 size
658 })
659 }
660 }
661
662 return {
663 id: video.id,
664 name: video.name,
665 url: video.url,
666 uuid: video.uuid,
667
668 redundancies: {
669 files: filesRedundancies,
670 streamingPlaylists: streamingPlaylistsRedundancies
671 }
672 }
673 }
674
675 getVideo () {
676 if (this.VideoFile?.Video) return this.VideoFile.Video
677
678 if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video
679
680 return undefined
681 }
682
683 isOwned () {
684 return !!this.strategy
685 }
686
687 toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
688 if (this.VideoStreamingPlaylist) {
689 return {
690 id: this.url,
691 type: 'CacheFile' as 'CacheFile',
692 object: this.VideoStreamingPlaylist.Video.url,
693 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
694 url: {
695 type: 'Link',
696 mediaType: 'application/x-mpegURL',
697 href: this.fileUrl
698 }
699 }
700 }
701
702 return {
703 id: this.url,
704 type: 'CacheFile' as 'CacheFile',
705 object: this.VideoFile.Video.url,
706 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
707 url: {
708 type: 'Link',
709 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
710 href: this.fileUrl,
711 height: this.VideoFile.resolution,
712 size: this.VideoFile.size,
713 fps: this.VideoFile.fps
714 }
715 }
716 }
717
718 // Don't include video files we already duplicated
719 private static buildVideoIdsForDuplication (peertubeActor: MActor) {
720 const notIn = literal(
721 '(' +
722 `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
723 `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
724 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
725 `UNION ` +
726 `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
727 `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
728 `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
729 ')'
730 )
731
732 return {
733 id: {
734 [Op.notIn]: notIn
735 }
736 }
737 }
738
739 private static buildServerRedundancyInclude () {
740 return {
741 attributes: [],
742 model: VideoChannelModel.unscoped(),
743 required: true,
744 include: [
745 {
746 attributes: [],
747 model: ActorModel.unscoped(),
748 required: true,
749 include: [
750 {
751 attributes: [],
752 model: ServerModel.unscoped(),
753 required: true,
754 where: {
755 redundancyAllowed: true
756 }
757 }
758 ]
759 }
760 ]
761 }
762 }
763 }