]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/models/redundancy/video-redundancy.ts
Support additional video extensions
[github/Chocobozzz/PeerTube.git] / server / models / redundancy / video-redundancy.ts
1 import {
2 AllowNull,
3 BeforeDestroy,
4 BelongsTo,
5 Column,
6 CreatedAt,
7 DataType,
8 ForeignKey,
9 Is,
10 Model,
11 Scopes,
12 Table,
13 UpdatedAt
14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getVideoSort, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONFIG, CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers'
19 import { VideoFileModel } from '../video/video-file'
20 import { getServerActor } from '../../helpers/utils'
21 import { VideoModel } from '../video/video'
22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
23 import { logger } from '../../helpers/logger'
24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
25 import { VideoChannelModel } from '../video/video-channel'
26 import { ServerModel } from '../server/server'
27 import { sample } from 'lodash'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import * as Bluebird from 'bluebird'
30 import * as Sequelize from 'sequelize'
31
32 export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO'
34 }
35
36 @Scopes({
37 [ ScopeNames.WITH_VIDEO ]: {
38 include: [
39 {
40 model: () => VideoFileModel,
41 required: true,
42 include: [
43 {
44 model: () => VideoModel,
45 required: true
46 }
47 ]
48 }
49 ]
50 }
51 })
52
53 @Table({
54 tableName: 'videoRedundancy',
55 indexes: [
56 {
57 fields: [ 'videoFileId' ]
58 },
59 {
60 fields: [ 'actorId' ]
61 },
62 {
63 fields: [ 'url' ],
64 unique: true
65 }
66 ]
67 })
68 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
69
70 @CreatedAt
71 createdAt: Date
72
73 @UpdatedAt
74 updatedAt: Date
75
76 @AllowNull(false)
77 @Column
78 expiresOn: Date
79
80 @AllowNull(false)
81 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
82 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
83 fileUrl: string
84
85 @AllowNull(false)
86 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
87 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
88 url: string
89
90 @AllowNull(true)
91 @Column
92 strategy: string // Only used by us
93
94 @ForeignKey(() => VideoFileModel)
95 @Column
96 videoFileId: number
97
98 @BelongsTo(() => VideoFileModel, {
99 foreignKey: {
100 allowNull: false
101 },
102 onDelete: 'cascade'
103 })
104 VideoFile: VideoFileModel
105
106 @ForeignKey(() => ActorModel)
107 @Column
108 actorId: number
109
110 @BelongsTo(() => ActorModel, {
111 foreignKey: {
112 allowNull: false
113 },
114 onDelete: 'cascade'
115 })
116 Actor: ActorModel
117
118 @BeforeDestroy
119 static async removeFile (instance: VideoRedundancyModel) {
120 if (!instance.isOwned()) return
121
122 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
123
124 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
125 logger.info('Removing duplicated video file %s.', logIdentifier)
126
127 videoFile.Video.removeFile(videoFile, true)
128 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
129
130 return undefined
131 }
132
133 static async loadLocalByFileId (videoFileId: number) {
134 const actor = await getServerActor()
135
136 const query = {
137 where: {
138 actorId: actor.id,
139 videoFileId
140 }
141 }
142
143 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
144 }
145
146 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
147 const query = {
148 where: {
149 url
150 },
151 transaction
152 }
153
154 return VideoRedundancyModel.findOne(query)
155 }
156
157 static async isLocalByVideoUUIDExists (uuid: string) {
158 const actor = await getServerActor()
159
160 const query = {
161 raw: true,
162 attributes: [ 'id' ],
163 where: {
164 actorId: actor.id
165 },
166 include: [
167 {
168 attributes: [ ],
169 model: VideoFileModel,
170 required: true,
171 include: [
172 {
173 attributes: [ ],
174 model: VideoModel,
175 required: true,
176 where: {
177 uuid
178 }
179 }
180 ]
181 }
182 ]
183 }
184
185 return VideoRedundancyModel.findOne(query)
186 .then(r => !!r)
187 }
188
189 static async getVideoSample (p: Bluebird<VideoModel[]>) {
190 const rows = await p
191 const ids = rows.map(r => r.id)
192 const id = sample(ids)
193
194 return VideoModel.loadWithFile(id, undefined, !isTestInstance())
195 }
196
197 static async findMostViewToDuplicate (randomizedFactor: number) {
198 // On VideoModel!
199 const query = {
200 attributes: [ 'id', 'views' ],
201 limit: randomizedFactor,
202 order: getVideoSort('-views'),
203 where: {
204 privacy: VideoPrivacy.PUBLIC
205 },
206 include: [
207 await VideoRedundancyModel.buildVideoFileForDuplication(),
208 VideoRedundancyModel.buildServerRedundancyInclude()
209 ]
210 }
211
212 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
213 }
214
215 static async findTrendingToDuplicate (randomizedFactor: number) {
216 // On VideoModel!
217 const query = {
218 attributes: [ 'id', 'views' ],
219 subQuery: false,
220 group: 'VideoModel.id',
221 limit: randomizedFactor,
222 order: getVideoSort('-trending'),
223 where: {
224 privacy: VideoPrivacy.PUBLIC
225 },
226 include: [
227 await VideoRedundancyModel.buildVideoFileForDuplication(),
228 VideoRedundancyModel.buildServerRedundancyInclude(),
229
230 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
231 ]
232 }
233
234 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
235 }
236
237 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
238 // On VideoModel!
239 const query = {
240 attributes: [ 'id', 'publishedAt' ],
241 limit: randomizedFactor,
242 order: getVideoSort('-publishedAt'),
243 where: {
244 privacy: VideoPrivacy.PUBLIC,
245 views: {
246 [ Sequelize.Op.gte ]: minViews
247 }
248 },
249 include: [
250 await VideoRedundancyModel.buildVideoFileForDuplication(),
251 VideoRedundancyModel.buildServerRedundancyInclude()
252 ]
253 }
254
255 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
256 }
257
258 static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
259 const expiredDate = new Date()
260 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
261
262 const actor = await getServerActor()
263
264 const query = {
265 where: {
266 actorId: actor.id,
267 strategy,
268 createdAt: {
269 [ Sequelize.Op.lt ]: expiredDate
270 }
271 }
272 }
273
274 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
275 }
276
277 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
278 const actor = await getServerActor()
279
280 const options = {
281 include: [
282 {
283 attributes: [],
284 model: VideoRedundancyModel,
285 required: true,
286 where: {
287 actorId: actor.id,
288 strategy
289 }
290 }
291 ]
292 }
293
294 return VideoFileModel.sum('size', options as any) // FIXME: typings
295 .then(v => {
296 if (!v || isNaN(v)) return 0
297
298 return v
299 })
300 }
301
302 static async listLocalExpired () {
303 const actor = await getServerActor()
304
305 const query = {
306 where: {
307 actorId: actor.id,
308 expiresOn: {
309 [ Sequelize.Op.lt ]: new Date()
310 }
311 }
312 }
313
314 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
315 }
316
317 static async listRemoteExpired () {
318 const actor = await getServerActor()
319
320 const query = {
321 where: {
322 actorId: {
323 [Sequelize.Op.ne]: actor.id
324 },
325 expiresOn: {
326 [ Sequelize.Op.lt ]: new Date()
327 }
328 }
329 }
330
331 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
332 }
333
334 static async listLocalOfServer (serverId: number) {
335 const actor = await getServerActor()
336
337 const query = {
338 where: {
339 actorId: actor.id
340 },
341 include: [
342 {
343 model: VideoFileModel,
344 required: true,
345 include: [
346 {
347 model: VideoModel,
348 required: true,
349 include: [
350 {
351 attributes: [],
352 model: VideoChannelModel.unscoped(),
353 required: true,
354 include: [
355 {
356 attributes: [],
357 model: ActorModel.unscoped(),
358 required: true,
359 where: {
360 serverId
361 }
362 }
363 ]
364 }
365 ]
366 }
367 ]
368 }
369 ]
370 }
371
372 return VideoRedundancyModel.findAll(query)
373 }
374
375 static async getStats (strategy: VideoRedundancyStrategy) {
376 const actor = await getServerActor()
377
378 const query = {
379 raw: true,
380 attributes: [
381 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
382 [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ],
383 [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ]
384 ],
385 where: {
386 strategy,
387 actorId: actor.id
388 },
389 include: [
390 {
391 attributes: [],
392 model: VideoFileModel,
393 required: true
394 }
395 ]
396 }
397
398 return VideoRedundancyModel.find(query as any) // FIXME: typings
399 .then((r: any) => ({
400 totalUsed: parseInt(r.totalUsed.toString(), 10),
401 totalVideos: r.totalVideos,
402 totalVideoFiles: r.totalVideoFiles
403 }))
404 }
405
406 isOwned () {
407 return !!this.strategy
408 }
409
410 toActivityPubObject (): CacheFileObject {
411 return {
412 id: this.url,
413 type: 'CacheFile' as 'CacheFile',
414 object: this.VideoFile.Video.url,
415 expires: this.expiresOn.toISOString(),
416 url: {
417 type: 'Link',
418 mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
419 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
420 href: this.fileUrl,
421 height: this.VideoFile.resolution,
422 size: this.VideoFile.size,
423 fps: this.VideoFile.fps
424 }
425 }
426 }
427
428 // Don't include video files we already duplicated
429 private static async buildVideoFileForDuplication () {
430 const actor = await getServerActor()
431
432 const notIn = Sequelize.literal(
433 '(' +
434 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
435 ')'
436 )
437
438 return {
439 attributes: [],
440 model: VideoFileModel.unscoped(),
441 required: true,
442 where: {
443 id: {
444 [ Sequelize.Op.notIn ]: notIn
445 }
446 }
447 }
448 }
449
450 private static buildServerRedundancyInclude () {
451 return {
452 attributes: [],
453 model: VideoChannelModel.unscoped(),
454 required: true,
455 include: [
456 {
457 attributes: [],
458 model: ActorModel.unscoped(),
459 required: true,
460 include: [
461 {
462 attributes: [],
463 model: ServerModel.unscoped(),
464 required: true,
465 where: {
466 redundancyAllowed: true
467 }
468 }
469 ]
470 }
471 ]
472 }
473 }
474 }