]>
Commit | Line | Data |
---|---|---|
1 | import { remove } from 'fs-extra' | |
2 | import memoizee from 'memoizee' | |
3 | import { join } from 'path' | |
4 | import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize' | |
5 | import { | |
6 | AllowNull, | |
7 | BelongsTo, | |
8 | Column, | |
9 | CreatedAt, | |
10 | DataType, | |
11 | Default, | |
12 | DefaultScope, | |
13 | ForeignKey, | |
14 | HasMany, | |
15 | Is, | |
16 | Model, | |
17 | Scopes, | |
18 | Table, | |
19 | UpdatedAt | |
20 | } from 'sequelize-typescript' | |
21 | import validator from 'validator' | |
22 | import { logger } from '@server/helpers/logger' | |
23 | import { extractVideo } from '@server/helpers/video' | |
24 | import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' | |
25 | import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' | |
26 | import { getFSTorrentFilePath } from '@server/lib/paths' | |
27 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' | |
28 | import { VideoResolution, VideoStorage } from '@shared/models' | |
29 | import { AttributesOnly } from '@shared/typescript-utils' | |
30 | import { | |
31 | isVideoFileExtnameValid, | |
32 | isVideoFileInfoHashValid, | |
33 | isVideoFileResolutionValid, | |
34 | isVideoFileSizeValid, | |
35 | isVideoFPSResolutionValid | |
36 | } from '../../helpers/custom-validators/videos' | |
37 | import { | |
38 | LAZY_STATIC_PATHS, | |
39 | MEMOIZE_LENGTH, | |
40 | MEMOIZE_TTL, | |
41 | STATIC_DOWNLOAD_PATHS, | |
42 | STATIC_PATHS, | |
43 | WEBSERVER | |
44 | } from '../../initializers/constants' | |
45 | import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file' | |
46 | import { VideoRedundancyModel } from '../redundancy/video-redundancy' | |
47 | import { doesExist } from '../shared' | |
48 | import { parseAggregateResult, throwIfNotValid } from '../utils' | |
49 | import { VideoModel } from './video' | |
50 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | |
51 | ||
52 | export enum ScopeNames { | |
53 | WITH_VIDEO = 'WITH_VIDEO', | |
54 | WITH_METADATA = 'WITH_METADATA', | |
55 | WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST' | |
56 | } | |
57 | ||
58 | @DefaultScope(() => ({ | |
59 | attributes: { | |
60 | exclude: [ 'metadata' ] | |
61 | } | |
62 | })) | |
63 | @Scopes(() => ({ | |
64 | [ScopeNames.WITH_VIDEO]: { | |
65 | include: [ | |
66 | { | |
67 | model: VideoModel.unscoped(), | |
68 | required: true | |
69 | } | |
70 | ] | |
71 | }, | |
72 | [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => { | |
73 | return { | |
74 | include: [ | |
75 | { | |
76 | model: VideoModel.unscoped(), | |
77 | required: false, | |
78 | where: options.whereVideo | |
79 | }, | |
80 | { | |
81 | model: VideoStreamingPlaylistModel.unscoped(), | |
82 | required: false, | |
83 | include: [ | |
84 | { | |
85 | model: VideoModel.unscoped(), | |
86 | required: true, | |
87 | where: options.whereVideo | |
88 | } | |
89 | ] | |
90 | } | |
91 | ] | |
92 | } | |
93 | }, | |
94 | [ScopeNames.WITH_METADATA]: { | |
95 | attributes: { | |
96 | include: [ 'metadata' ] | |
97 | } | |
98 | } | |
99 | })) | |
100 | @Table({ | |
101 | tableName: 'videoFile', | |
102 | indexes: [ | |
103 | { | |
104 | fields: [ 'videoId' ], | |
105 | where: { | |
106 | videoId: { | |
107 | [Op.ne]: null | |
108 | } | |
109 | } | |
110 | }, | |
111 | { | |
112 | fields: [ 'videoStreamingPlaylistId' ], | |
113 | where: { | |
114 | videoStreamingPlaylistId: { | |
115 | [Op.ne]: null | |
116 | } | |
117 | } | |
118 | }, | |
119 | ||
120 | { | |
121 | fields: [ 'infoHash' ] | |
122 | }, | |
123 | ||
124 | { | |
125 | fields: [ 'torrentFilename' ], | |
126 | unique: true | |
127 | }, | |
128 | ||
129 | { | |
130 | fields: [ 'filename' ], | |
131 | unique: true | |
132 | }, | |
133 | ||
134 | { | |
135 | fields: [ 'videoId', 'resolution', 'fps' ], | |
136 | unique: true, | |
137 | where: { | |
138 | videoId: { | |
139 | [Op.ne]: null | |
140 | } | |
141 | } | |
142 | }, | |
143 | { | |
144 | fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ], | |
145 | unique: true, | |
146 | where: { | |
147 | videoStreamingPlaylistId: { | |
148 | [Op.ne]: null | |
149 | } | |
150 | } | |
151 | } | |
152 | ] | |
153 | }) | |
154 | export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>>> { | |
155 | @CreatedAt | |
156 | createdAt: Date | |
157 | ||
158 | @UpdatedAt | |
159 | updatedAt: Date | |
160 | ||
161 | @AllowNull(false) | |
162 | @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution')) | |
163 | @Column | |
164 | resolution: number | |
165 | ||
166 | @AllowNull(false) | |
167 | @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size')) | |
168 | @Column(DataType.BIGINT) | |
169 | size: number | |
170 | ||
171 | @AllowNull(false) | |
172 | @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname')) | |
173 | @Column | |
174 | extname: string | |
175 | ||
176 | @AllowNull(true) | |
177 | @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true)) | |
178 | @Column | |
179 | infoHash: string | |
180 | ||
181 | @AllowNull(false) | |
182 | @Default(-1) | |
183 | @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps')) | |
184 | @Column | |
185 | fps: number | |
186 | ||
187 | @AllowNull(true) | |
188 | @Column(DataType.JSONB) | |
189 | metadata: any | |
190 | ||
191 | @AllowNull(true) | |
192 | @Column | |
193 | metadataUrl: string | |
194 | ||
195 | // Could be null for remote files | |
196 | @AllowNull(true) | |
197 | @Column | |
198 | fileUrl: string | |
199 | ||
200 | // Could be null for live files | |
201 | @AllowNull(true) | |
202 | @Column | |
203 | filename: string | |
204 | ||
205 | // Could be null for remote files | |
206 | @AllowNull(true) | |
207 | @Column | |
208 | torrentUrl: string | |
209 | ||
210 | // Could be null for live files | |
211 | @AllowNull(true) | |
212 | @Column | |
213 | torrentFilename: string | |
214 | ||
215 | @ForeignKey(() => VideoModel) | |
216 | @Column | |
217 | videoId: number | |
218 | ||
219 | @AllowNull(false) | |
220 | @Default(VideoStorage.FILE_SYSTEM) | |
221 | @Column | |
222 | storage: VideoStorage | |
223 | ||
224 | @BelongsTo(() => VideoModel, { | |
225 | foreignKey: { | |
226 | allowNull: true | |
227 | }, | |
228 | onDelete: 'CASCADE' | |
229 | }) | |
230 | Video: VideoModel | |
231 | ||
232 | @ForeignKey(() => VideoStreamingPlaylistModel) | |
233 | @Column | |
234 | videoStreamingPlaylistId: number | |
235 | ||
236 | @BelongsTo(() => VideoStreamingPlaylistModel, { | |
237 | foreignKey: { | |
238 | allowNull: true | |
239 | }, | |
240 | onDelete: 'CASCADE' | |
241 | }) | |
242 | VideoStreamingPlaylist: VideoStreamingPlaylistModel | |
243 | ||
244 | @HasMany(() => VideoRedundancyModel, { | |
245 | foreignKey: { | |
246 | allowNull: true | |
247 | }, | |
248 | onDelete: 'CASCADE', | |
249 | hooks: true | |
250 | }) | |
251 | RedundancyVideos: VideoRedundancyModel[] | |
252 | ||
253 | static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, { | |
254 | promise: true, | |
255 | max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, | |
256 | maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS | |
257 | }) | |
258 | ||
259 | static doesInfohashExist (infoHash: string) { | |
260 | const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' | |
261 | ||
262 | return doesExist(query, { infoHash }) | |
263 | } | |
264 | ||
265 | static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { | |
266 | const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID) | |
267 | ||
268 | return !!videoFile | |
269 | } | |
270 | ||
271 | static async doesOwnedTorrentFileExist (filename: string) { | |
272 | const query = 'SELECT 1 FROM "videoFile" ' + | |
273 | 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' + | |
274 | 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' + | |
275 | 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' + | |
276 | 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1' | |
277 | ||
278 | return doesExist(query, { filename }) | |
279 | } | |
280 | ||
281 | static async doesOwnedWebTorrentVideoFileExist (filename: string) { | |
282 | const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + | |
283 | `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` | |
284 | ||
285 | return doesExist(query, { filename }) | |
286 | } | |
287 | ||
288 | static loadByFilename (filename: string) { | |
289 | const query = { | |
290 | where: { | |
291 | filename | |
292 | } | |
293 | } | |
294 | ||
295 | return VideoFileModel.findOne(query) | |
296 | } | |
297 | ||
298 | static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { | |
299 | const query = { | |
300 | where: { | |
301 | torrentFilename: filename | |
302 | } | |
303 | } | |
304 | ||
305 | return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) | |
306 | } | |
307 | ||
308 | static loadWithMetadata (id: number) { | |
309 | return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) | |
310 | } | |
311 | ||
312 | static loadWithVideo (id: number) { | |
313 | return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id) | |
314 | } | |
315 | ||
316 | static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { | |
317 | const whereVideo = validator.isUUID(videoIdOrUUID + '') | |
318 | ? { uuid: videoIdOrUUID } | |
319 | : { id: videoIdOrUUID } | |
320 | ||
321 | const options = { | |
322 | where: { | |
323 | id | |
324 | } | |
325 | } | |
326 | ||
327 | return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] }) | |
328 | .findOne(options) | |
329 | .then(file => { | |
330 | // We used `required: false` so check we have at least a video or a streaming playlist | |
331 | if (!file.Video && !file.VideoStreamingPlaylist) return null | |
332 | ||
333 | return file | |
334 | }) | |
335 | } | |
336 | ||
337 | static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { | |
338 | const query = { | |
339 | include: [ | |
340 | { | |
341 | model: VideoModel.unscoped(), | |
342 | required: true, | |
343 | include: [ | |
344 | { | |
345 | model: VideoStreamingPlaylistModel.unscoped(), | |
346 | required: true, | |
347 | where: { | |
348 | id: streamingPlaylistId | |
349 | } | |
350 | } | |
351 | ] | |
352 | } | |
353 | ], | |
354 | transaction | |
355 | } | |
356 | ||
357 | return VideoFileModel.findAll(query) | |
358 | } | |
359 | ||
360 | static getStats () { | |
361 | const webtorrentFilesQuery: FindOptions = { | |
362 | include: [ | |
363 | { | |
364 | attributes: [], | |
365 | required: true, | |
366 | model: VideoModel.unscoped(), | |
367 | where: { | |
368 | remote: false | |
369 | } | |
370 | } | |
371 | ] | |
372 | } | |
373 | ||
374 | const hlsFilesQuery: FindOptions = { | |
375 | include: [ | |
376 | { | |
377 | attributes: [], | |
378 | required: true, | |
379 | model: VideoStreamingPlaylistModel.unscoped(), | |
380 | include: [ | |
381 | { | |
382 | attributes: [], | |
383 | model: VideoModel.unscoped(), | |
384 | required: true, | |
385 | where: { | |
386 | remote: false | |
387 | } | |
388 | } | |
389 | ] | |
390 | } | |
391 | ] | |
392 | } | |
393 | ||
394 | return Promise.all([ | |
395 | VideoFileModel.aggregate('size', 'SUM', webtorrentFilesQuery), | |
396 | VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery) | |
397 | ]).then(([ webtorrentResult, hlsResult ]) => ({ | |
398 | totalLocalVideoFilesSize: parseAggregateResult(webtorrentResult) + parseAggregateResult(hlsResult) | |
399 | })) | |
400 | } | |
401 | ||
402 | // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes | |
403 | static async customUpsert ( | |
404 | videoFile: MVideoFile, | |
405 | mode: 'streaming-playlist' | 'video', | |
406 | transaction: Transaction | |
407 | ) { | |
408 | const baseFind = { | |
409 | fps: videoFile.fps, | |
410 | resolution: videoFile.resolution, | |
411 | transaction | |
412 | } | |
413 | ||
414 | const element = mode === 'streaming-playlist' | |
415 | ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId }) | |
416 | : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId }) | |
417 | ||
418 | if (!element) return videoFile.save({ transaction }) | |
419 | ||
420 | for (const k of Object.keys(videoFile.toJSON())) { | |
421 | element[k] = videoFile[k] | |
422 | } | |
423 | ||
424 | return element.save({ transaction }) | |
425 | } | |
426 | ||
427 | static async loadWebTorrentFile (options: { | |
428 | videoId: number | |
429 | fps: number | |
430 | resolution: number | |
431 | transaction?: Transaction | |
432 | }) { | |
433 | const where = { | |
434 | fps: options.fps, | |
435 | resolution: options.resolution, | |
436 | videoId: options.videoId | |
437 | } | |
438 | ||
439 | return VideoFileModel.findOne({ where, transaction: options.transaction }) | |
440 | } | |
441 | ||
442 | static async loadHLSFile (options: { | |
443 | playlistId: number | |
444 | fps: number | |
445 | resolution: number | |
446 | transaction?: Transaction | |
447 | }) { | |
448 | const where = { | |
449 | fps: options.fps, | |
450 | resolution: options.resolution, | |
451 | videoStreamingPlaylistId: options.playlistId | |
452 | } | |
453 | ||
454 | return VideoFileModel.findOne({ where, transaction: options.transaction }) | |
455 | } | |
456 | ||
457 | static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) { | |
458 | const options = { | |
459 | where: { videoStreamingPlaylistId } | |
460 | } | |
461 | ||
462 | return VideoFileModel.destroy(options) | |
463 | } | |
464 | ||
465 | hasTorrent () { | |
466 | return this.infoHash && this.torrentFilename | |
467 | } | |
468 | ||
469 | getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { | |
470 | if (this.videoId) return (this as MVideoFileVideo).Video | |
471 | ||
472 | return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist | |
473 | } | |
474 | ||
475 | getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo { | |
476 | return extractVideo(this.getVideoOrStreamingPlaylist()) | |
477 | } | |
478 | ||
479 | isAudio () { | |
480 | return this.resolution === VideoResolution.H_NOVIDEO | |
481 | } | |
482 | ||
483 | isLive () { | |
484 | return this.size === -1 | |
485 | } | |
486 | ||
487 | isHLS () { | |
488 | return !!this.videoStreamingPlaylistId | |
489 | } | |
490 | ||
491 | getObjectStorageUrl () { | |
492 | if (this.isHLS()) { | |
493 | return getHLSPublicFileUrl(this.fileUrl) | |
494 | } | |
495 | ||
496 | return getWebTorrentPublicFileUrl(this.fileUrl) | |
497 | } | |
498 | ||
499 | getFileUrl (video: MVideo) { | |
500 | if (this.storage === VideoStorage.OBJECT_STORAGE) { | |
501 | return this.getObjectStorageUrl() | |
502 | } | |
503 | ||
504 | if (!this.Video) this.Video = video as VideoModel | |
505 | if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) | |
506 | ||
507 | return this.fileUrl | |
508 | } | |
509 | ||
510 | getFileStaticPath (video: MVideo) { | |
511 | if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) | |
512 | ||
513 | return join(STATIC_PATHS.WEBSEED, this.filename) | |
514 | } | |
515 | ||
516 | getFileDownloadUrl (video: MVideoWithHost) { | |
517 | const path = this.isHLS() | |
518 | ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) | |
519 | : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`) | |
520 | ||
521 | if (video.isOwned()) return WEBSERVER.URL + path | |
522 | ||
523 | // FIXME: don't guess remote URL | |
524 | return buildRemoteVideoBaseUrl(video, path) | |
525 | } | |
526 | ||
527 | getRemoteTorrentUrl (video: MVideo) { | |
528 | if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) | |
529 | ||
530 | return this.torrentUrl | |
531 | } | |
532 | ||
533 | // We proxify torrent requests so use a local URL | |
534 | getTorrentUrl () { | |
535 | if (!this.torrentFilename) return null | |
536 | ||
537 | return WEBSERVER.URL + this.getTorrentStaticPath() | |
538 | } | |
539 | ||
540 | getTorrentStaticPath () { | |
541 | if (!this.torrentFilename) return null | |
542 | ||
543 | return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename) | |
544 | } | |
545 | ||
546 | getTorrentDownloadUrl () { | |
547 | if (!this.torrentFilename) return null | |
548 | ||
549 | return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename) | |
550 | } | |
551 | ||
552 | removeTorrent () { | |
553 | if (!this.torrentFilename) return null | |
554 | ||
555 | const torrentPath = getFSTorrentFilePath(this) | |
556 | return remove(torrentPath) | |
557 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | |
558 | } | |
559 | ||
560 | hasSameUniqueKeysThan (other: MVideoFile) { | |
561 | return this.fps === other.fps && | |
562 | this.resolution === other.resolution && | |
563 | ( | |
564 | (this.videoId !== null && this.videoId === other.videoId) || | |
565 | (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) | |
566 | ) | |
567 | } | |
568 | ||
569 | withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { | |
570 | if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist }) | |
571 | ||
572 | return Object.assign(this, { Video: videoOrPlaylist }) | |
573 | } | |
574 | } |