From c48e82b5e0478434de30626d14594a97f2402e7c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Sep 2018 16:27:07 +0200 Subject: Basic video redundancy implementation --- server/models/activitypub/actor-follow.ts | 4 +- server/models/activitypub/actor.ts | 13 +- server/models/redundancy/video-redundancy.ts | 249 +++++++++++++++++++++++++++ server/models/server/server.ts | 17 +- server/models/video/video-file.ts | 25 ++- server/models/video/video.ts | 73 ++++---- 6 files changed, 344 insertions(+), 37 deletions(-) create mode 100644 server/models/redundancy/video-redundancy.ts (limited to 'server/models') diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 8bc095997..27bb43dae 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -19,7 +19,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { FollowState } from '../../../shared/models/actors' -import { AccountFollow } from '../../../shared/models/actors/follow.model' +import { ActorFollow } from '../../../shared/models/actors/follow.model' import { logger } from '../../helpers/logger' import { getServerActor } from '../../helpers/utils' import { ACTOR_FOLLOW_SCORE } from '../../initializers' @@ -529,7 +529,7 @@ export class ActorFollowModel extends Model { return ActorFollowModel.findAll(query) } - toFormattedJSON (): AccountFollow { + toFormattedJSON (): ActorFollow { const follower = this.ActorFollower.toFormattedJSON() const following = this.ActorFollowing.toFormattedJSON() diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 119d0c1da..ef8dd9f7c 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [ }, { model: () => VideoChannelModel.unscoped(), - required: false + required: false, + include: [ + { + model: () => AccountModel, + required: true + } + ] }, { model: () => ServerModel, @@ -337,6 +343,7 @@ export class ActorModel extends Model { uuid: this.uuid, name: this.preferredUsername, host: this.getHost(), + hostRedundancyAllowed: this.getRedundancyAllowed(), followingCount: this.followingCount, followersCount: this.followersCount, avatar, @@ -440,6 +447,10 @@ export class ActorModel extends Model { return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST } + getRedundancyAllowed () { + return this.Server ? this.Server.redundancyAllowed : false + } + getAvatarUrl () { if (!this.avatarId) return undefined diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts new file mode 100644 index 000000000..48ec77206 --- /dev/null +++ b/server/models/redundancy/video-redundancy.ts @@ -0,0 +1,249 @@ +import { + AfterDestroy, + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + Is, + Model, + Scopes, + Sequelize, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { ActorModel } from '../activitypub/actor' +import { throwIfNotValid } from '../utils' +import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' +import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { VideoFileModel } from '../video/video-file' +import { isDateValid } from '../../helpers/custom-validators/misc' +import { getServerActor } from '../../helpers/utils' +import { VideoModel } from '../video/video' +import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { logger } from '../../helpers/logger' +import { CacheFileObject } from '../../../shared' +import { VideoChannelModel } from '../video/video-channel' +import { ServerModel } from '../server/server' +import { sample } from 'lodash' +import { isTestInstance } from '../../helpers/core-utils' + +export enum ScopeNames { + WITH_VIDEO = 'WITH_VIDEO' +} + +@Scopes({ + [ ScopeNames.WITH_VIDEO ]: { + include: [ + { + model: () => VideoFileModel, + required: true, + include: [ + { + model: () => VideoModel, + required: true + } + ] + } + ] + } +}) + +@Table({ + tableName: 'videoRedundancy', + indexes: [ + { + fields: [ 'videoFileId' ] + }, + { + fields: [ 'actorId' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoRedundancyModel extends Model { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + expiresOn: Date + + @AllowNull(false) + @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) + fileUrl: string + + @AllowNull(false) + @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) + url: string + + @AllowNull(true) + @Column + strategy: string // Only used by us + + @ForeignKey(() => VideoFileModel) + @Column + videoFileId: number + + @BelongsTo(() => VideoFileModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + VideoFile: VideoFileModel + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: ActorModel + + @AfterDestroy + static removeFilesAndSendDelete (instance: VideoRedundancyModel) { + // Not us + if (!instance.strategy) return + + logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) + + return instance.VideoFile.Video.removeFile(instance.VideoFile) + } + + static loadByFileId (videoFileId: number) { + const query = { + where: { + videoFileId + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + + static loadByUrl (url: string) { + const query = { + where: { + url + } + } + + return VideoRedundancyModel.findOne(query) + } + + static async findMostViewToDuplicate (randomizedFactor: number) { + // On VideoModel! + const query = { + logging: !isTestInstance(), + limit: randomizedFactor, + order: [ [ 'views', 'DESC' ] ], + include: [ + { + model: VideoFileModel.unscoped(), + required: true, + where: { + id: { + [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn() + } + } + }, + { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ServerModel.unscoped(), + required: true, + where: { + redundancyAllowed: true + } + } + ] + } + ] + } + ] + } + + const rows = await VideoModel.unscoped().findAll(query) + + return sample(rows) + } + + static async getVideoFiles (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const queryVideoFiles = { + logging: !isTestInstance(), + where: { + actorId: actor.id, + strategy + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) + .findAll(queryVideoFiles) + } + + static listAllExpired () { + const query = { + logging: !isTestInstance(), + where: { + expiresOn: { + [Sequelize.Op.lt]: new Date() + } + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) + .findAll(query) + } + + toActivityPubObject (): CacheFileObject { + return { + id: this.url, + type: 'CacheFile' as 'CacheFile', + object: this.VideoFile.Video.url, + expires: this.expiresOn.toISOString(), + url: { + type: 'Link', + mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, + href: this.fileUrl, + height: this.VideoFile.resolution, + size: this.VideoFile.size, + fps: this.VideoFile.fps + } + } + } + + private static async buildExcludeIn () { + const actor = await getServerActor() + + return Sequelize.literal( + '(' + + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + + ')' + ) + } +} diff --git a/server/models/server/server.ts b/server/models/server/server.ts index 9749f503e..ca3b24d51 100644 --- a/server/models/server/server.ts +++ b/server/models/server/server.ts @@ -1,4 +1,4 @@ -import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { isHostValid } from '../../helpers/custom-validators/servers' import { ActorModel } from '../activitypub/actor' import { throwIfNotValid } from '../utils' @@ -19,6 +19,11 @@ export class ServerModel extends Model { @Column host: string + @AllowNull(false) + @Default(false) + @Column + redundancyAllowed: boolean + @CreatedAt createdAt: Date @@ -34,4 +39,14 @@ export class ServerModel extends Model { hooks: true }) Actors: ActorModel[] + + static loadByHost (host: string) { + const query = { + where: { + host + } + } + + return ServerModel.findOne(query) + } } diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 3bc4855f3..0907ea569 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -1,5 +1,18 @@ import { values } from 'lodash' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + HasMany, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' import { isVideoFileInfoHashValid, isVideoFileResolutionValid, @@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers' import { throwIfNotValid } from '../utils' import { VideoModel } from './video' import * as Sequelize from 'sequelize' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' @Table({ tableName: 'videoFile', @@ -70,6 +84,15 @@ export class VideoFileModel extends Model { }) Video: VideoModel + @HasMany(() => VideoRedundancyModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE', + hooks: true + }) + RedundancyVideos: VideoRedundancyModel[] + static isInfohashExists (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' const options = { diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 86316653f..27c631dcd 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -27,13 +27,13 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared' +import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoFilter } from '../../../shared/models/videos/video-query.type' import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { isBooleanValid } from '../../helpers/custom-validators/misc' +import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc' import { isVideoCategoryValid, isVideoDescriptionValid, @@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption' import { VideoBlacklistModel } from './video-blacklist' import { copy, remove, rename, stat, writeFile } from 'fs-extra' import { VideoViewModel } from './video-views' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -470,7 +471,13 @@ type AvailableForListIDsOptions = { include: [ { model: () => VideoFileModel.unscoped(), - required: false + required: false, + include: [ + { + model: () => VideoRedundancyModel.unscoped(), + required: false + } + ] } ] }, @@ -633,6 +640,7 @@ export class VideoModel extends Model { name: 'videoId', allowNull: false }, + hooks: true, onDelete: 'cascade' }) VideoFiles: VideoFileModel[] @@ -1325,9 +1333,7 @@ export class VideoModel extends Model { [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ], [ CONFIG.WEBSERVER.URL + '/tracker/announce' ] ], - urlList: [ - CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) - ] + urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ] } const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) @@ -1535,11 +1541,11 @@ export class VideoModel extends Model { } } - const url = [] + const url: ActivityUrlObject[] = [] for (const file of this.VideoFiles) { url.push({ type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ file.extname ], + mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, href: this.getVideoFileUrl(file, baseUrlHttp), height: file.resolution, size: file.size, @@ -1548,14 +1554,14 @@ export class VideoModel extends Model { url.push({ type: 'Link', - mimeType: 'application/x-bittorrent', + mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', href: this.getTorrentUrl(file, baseUrlHttp), height: file.resolution }) url.push({ type: 'Link', - mimeType: 'application/x-bittorrent;x-scheme-handler/magnet', + mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), height: file.resolution }) @@ -1796,7 +1802,7 @@ export class VideoModel extends Model { (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL } - private getBaseUrls () { + getBaseUrls () { let baseUrlHttp let baseUrlWs @@ -1811,39 +1817,42 @@ export class VideoModel extends Model { return { baseUrlHttp, baseUrlWs } } - private getThumbnailUrl (baseUrlHttp: string) { + generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { + const xs = this.getTorrentUrl(videoFile, baseUrlHttp) + const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] + + const redundancies = videoFile.RedundancyVideos + if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) + + const magnetHash = { + xs, + announce, + urlList, + infoHash: videoFile.infoHash, + name: this.name + } + + return magnetUtil.encode(magnetHash) + } + + getThumbnailUrl (baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() } - private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile) } - private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile) } - private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) } - private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } - - private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { - const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] - const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] - - const magnetHash = { - xs, - announce, - urlList, - infoHash: videoFile.infoHash, - name: this.name - } - - return magnetUtil.encode(magnetHash) - } } -- cgit v1.2.3