From d9a2a03196275065c28f4a0b7d4d7bc9992d77a1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 18 Feb 2021 10:15:11 +0100 Subject: Don't guess remote tracker URL --- server/models/server/tracker.ts | 73 +++++++++++++++++++++++++++++++ server/models/server/video-tracker.ts | 30 +++++++++++++ server/models/video/thumbnail.ts | 5 +-- server/models/video/video-caption.ts | 5 +-- server/models/video/video-file.ts | 9 +--- server/models/video/video-format-utils.ts | 49 ++++++++++----------- server/models/video/video.ts | 54 +++++++++++++---------- 7 files changed, 162 insertions(+), 63 deletions(-) create mode 100644 server/models/server/tracker.ts create mode 100644 server/models/server/video-tracker.ts (limited to 'server/models') diff --git a/server/models/server/tracker.ts b/server/models/server/tracker.ts new file mode 100644 index 000000000..d7c91faad --- /dev/null +++ b/server/models/server/tracker.ts @@ -0,0 +1,73 @@ +import { AllowNull, BelongsToMany, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { Transaction } from 'sequelize/types' +import { MTracker } from '@server/types/models/server/tracker' +import { VideoModel } from '../video/video' +import { VideoTrackerModel } from './video-tracker' + +@Table({ + tableName: 'tracker', + indexes: [ + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class TrackerModel extends Model { + + @AllowNull(false) + @Column + url: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @BelongsToMany(() => VideoModel, { + foreignKey: 'trackerId', + through: () => VideoTrackerModel, + onDelete: 'CASCADE' + }) + Videos: VideoModel[] + + static listUrlsByVideoId (videoId: number) { + const query = { + include: [ + { + attributes: [ 'id', 'trackerId' ], + model: VideoModel.unscoped(), + required: true, + where: { id: videoId } + } + ] + } + + return TrackerModel.findAll(query) + .then(rows => rows.map(rows => rows.url)) + } + + static findOrCreateTrackers (trackers: string[], transaction: Transaction): Promise { + if (trackers === null) return Promise.resolve([]) + + const tasks: Promise[] = [] + trackers.forEach(tracker => { + const query = { + where: { + url: tracker + }, + defaults: { + url: tracker + }, + transaction + } + + const promise = TrackerModel.findOrCreate(query) + .then(([ trackerInstance ]) => trackerInstance) + tasks.push(promise) + }) + + return Promise.all(tasks) + } +} diff --git a/server/models/server/video-tracker.ts b/server/models/server/video-tracker.ts new file mode 100644 index 000000000..367bf0117 --- /dev/null +++ b/server/models/server/video-tracker.ts @@ -0,0 +1,30 @@ +import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoModel } from '../video/video' +import { TrackerModel } from './tracker' + +@Table({ + tableName: 'videoTracker', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'trackerId' ] + } + ] +}) +export class VideoTrackerModel extends Model { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @ForeignKey(() => TrackerModel) + @Column + trackerId: number +} diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index 9533c8d19..319e1175a 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -15,7 +15,6 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' import { afterCommitIfTransaction } from '@server/helpers/database-utils' import { MThumbnail, MThumbnailVideo, MVideoWithHost } from '@server/types/models' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' @@ -168,10 +167,8 @@ export class ThumbnailModel extends Model { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename if (video.isOwned()) return WEBSERVER.URL + staticPath - if (this.fileUrl) return this.fileUrl - // Fallback if we don't have a file URL - return buildRemoteVideoBaseUrl(video, staticPath) + return this.fileUrl } getPath () { diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 71b067335..0bbe9b752 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -16,7 +16,6 @@ import { UpdatedAt } from 'sequelize-typescript' import { v4 as uuidv4 } from 'uuid' -import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' import { MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo, MVideoWithHost } from '@server/types/models' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' @@ -208,9 +207,7 @@ export class VideoCaptionModel extends Model { if (!this.Video) this.Video = video as VideoModel if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() - if (this.fileUrl) return this.fileUrl - // Fallback if we don't have a file URL - return buildRemoteVideoBaseUrl(video, this.getCaptionStaticPath()) + return this.fileUrl } } diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 57807cbfd..5a3706259 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -427,10 +427,8 @@ export class VideoFileModel extends Model { if (!this.Video) this.Video = video as VideoModel if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) - if (this.fileUrl) return this.fileUrl - // Fallback if we don't have a file URL - return buildRemoteVideoBaseUrl(video, this.getFileStaticPath(video)) + return this.fileUrl } getFileStaticPath (video: MVideo) { @@ -454,10 +452,7 @@ export class VideoFileModel extends Model { getRemoteTorrentUrl (video: MVideoWithHost) { if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) - if (this.torrentUrl) return this.torrentUrl - - // Fallback if we don't have a torrent URL - return buildRemoteVideoBaseUrl(video, this.getTorrentStaticPath()) + return this.torrentUrl } // We proxify torrent requests so use a local URL diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index adf460734..9dc3e7722 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -14,8 +14,6 @@ import { } from '../../lib/activitypub/url' import { MStreamingPlaylistRedundanciesOpt, - MStreamingPlaylistVideo, - MVideo, MVideoAP, MVideoFile, MVideoFormattable, @@ -127,8 +125,6 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid } }) - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const tags = video.Tags ? video.Tags.map(t => t.name) : [] const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) @@ -147,14 +143,14 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid label: VideoModel.getStateLabel(video.state) }, - trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs), + trackerUrls: video.getTrackerUrls(), files: [], streamingPlaylists } // Format and sort video files - detailsJson.files = videoFilesModelToFormattedJSON(video, video, baseUrlHttp, baseUrlWs, video.VideoFiles) + detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) return Object.assign(formattedJson, detailsJson) } @@ -165,17 +161,13 @@ function streamingPlaylistsModelToFormattedJSON ( ): VideoStreamingPlaylist[] { if (isArray(playlists) === false) return [] - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - return playlists .map(playlist => { - const playlistWithVideo = Object.assign(playlist, { Video: video }) - const redundancies = isArray(playlist.RedundancyVideos) ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) : [] - const files = videoFilesModelToFormattedJSON(playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles) + const files = videoFilesModelToFormattedJSON(video, playlist.VideoFiles) return { id: playlist.id, @@ -194,14 +186,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) { return -1 } -// FIXME: refactor/merge model and video arguments function videoFilesModelToFormattedJSON ( - model: MVideo | MStreamingPlaylistVideo, video: MVideoFormattableDetails, - baseUrlHttp: string, - baseUrlWs: string, videoFiles: MVideoFileRedundanciesOpt[] ): VideoFile[] { + const trackerUrls = video.getTrackerUrls() + return [ ...videoFiles ] .filter(f => !f.isLive()) .sort(sortByResolutionDesc) @@ -213,7 +203,7 @@ function videoFilesModelToFormattedJSON ( }, // FIXME: deprecated in 3.2 - magnetUri: generateMagnetUri(model, video, videoFile, baseUrlHttp, baseUrlWs), + magnetUri: generateMagnetUri(video, videoFile, trackerUrls), size: videoFile.size, fps: videoFile.fps, @@ -229,15 +219,13 @@ function videoFilesModelToFormattedJSON ( }) } -// FIXME: refactor/merge model and video arguments function addVideoFilesInAPAcc ( acc: ActivityUrlObject[] | ActivityTagObject[], - model: MVideoAP | MStreamingPlaylistVideo, video: MVideoWithHost, - baseUrlHttp: string, - baseUrlWs: string, files: MVideoFile[] ) { + const trackerUrls = video.getTrackerUrls() + const sortedFiles = [ ...files ] .filter(f => !f.isLive()) .sort(sortByResolutionDesc) @@ -271,14 +259,13 @@ function addVideoFilesInAPAcc ( acc.push({ type: 'Link', mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: generateMagnetUri(model, video, file, baseUrlHttp, baseUrlWs), + href: generateMagnetUri(video, file, trackerUrls), height: file.resolution }) } } function videoModelToActivityPubObject (video: MVideoAP): VideoObject { - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() if (!video.Tags) video.Tags = [] const tag = video.Tags.map(t => ({ @@ -319,7 +306,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { } ] - addVideoFilesInAPAcc(url, video, video, baseUrlHttp, baseUrlWs, video.VideoFiles || []) + addVideoFilesInAPAcc(url, video, video.VideoFiles || []) for (const playlist of (video.VideoStreamingPlaylists || [])) { const tag = playlist.p2pMediaLoaderInfohashes @@ -331,8 +318,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { href: playlist.segmentsSha256Url }) - const playlistWithVideo = Object.assign(playlist, { Video: video }) - addVideoFilesInAPAcc(tag, playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles || []) + addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) url.push({ type: 'Link', @@ -342,6 +328,19 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { }) } + for (const trackerUrl of video.getTrackerUrls()) { + const rel2 = trackerUrl.startsWith('http') + ? 'http' + : 'websocket' + + url.push({ + type: 'Link', + name: `tracker-${rel2}`, + rel: [ 'tracker', rel2 ], + href: trackerUrl + }) + } + const subtitleLanguage = [] for (const caption of video.VideoCaptions) { subtitleLanguage.push({ diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 2e6b6aeec..9e67ca0f4 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -60,7 +60,6 @@ import { API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, - REMOTE_SCHEME, STATIC_PATHS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, @@ -107,6 +106,8 @@ import { ActorModel } from '../activitypub/actor' import { AvatarModel } from '../avatar/avatar' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { ServerModel } from '../server/server' +import { TrackerModel } from '../server/tracker' +import { VideoTrackerModel } from '../server/video-tracker' import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils' import { ScheduleVideoUpdateModel } from './schedule-video-update' import { TagModel } from './tag' @@ -137,6 +138,7 @@ export enum ScopeNames { FOR_API = 'FOR_API', WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS', WITH_TAGS = 'WITH_TAGS', + WITH_TRACKERS = 'WITH_TRACKERS', WITH_WEBTORRENT_FILES = 'WITH_WEBTORRENT_FILES', WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', WITH_BLACKLISTED = 'WITH_BLACKLISTED', @@ -320,6 +322,14 @@ export type AvailableForListIDsOptions = { [ScopeNames.WITH_TAGS]: { include: [ TagModel ] }, + [ScopeNames.WITH_TRACKERS]: { + include: [ + { + attributes: [ 'id', 'url' ], + model: TrackerModel + } + ] + }, [ScopeNames.WITH_BLACKLISTED]: { include: [ { @@ -616,6 +626,13 @@ export class VideoModel extends Model { }) Tags: TagModel[] + @BelongsToMany(() => TrackerModel, { + foreignKey: 'videoId', + through: () => VideoTrackerModel, + onDelete: 'CASCADE' + }) + Trackers: TrackerModel[] + @HasMany(() => ThumbnailModel, { foreignKey: { name: 'videoId', @@ -1436,6 +1453,7 @@ export class VideoModel extends Model { ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_THUMBNAILS, ScopeNames.WITH_LIVE, + ScopeNames.WITH_TRACKERS, { method: [ ScopeNames.WITH_WEBTORRENT_FILES, true ] }, { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } ] @@ -1887,18 +1905,15 @@ export class VideoModel extends Model { } getFormattedVideoFilesJSON (): VideoFile[] { - const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() let files: VideoFile[] = [] if (Array.isArray(this.VideoFiles)) { - const result = videoFilesModelToFormattedJSON(this, this, baseUrlHttp, baseUrlWs, this.VideoFiles) + const result = videoFilesModelToFormattedJSON(this, this.VideoFiles) files = files.concat(result) } for (const p of (this.VideoStreamingPlaylists || [])) { - p.Video = this - - const result = videoFilesModelToFormattedJSON(p, this, baseUrlHttp, baseUrlWs, p.VideoFiles) + const result = videoFilesModelToFormattedJSON(this, p.VideoFiles) files = files.concat(result) } @@ -2030,25 +2045,18 @@ export class VideoModel extends Model { return false } - getBaseUrls () { - if (this.isOwned()) { - return { - baseUrlHttp: WEBSERVER.URL, - baseUrlWs: WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT - } - } - - return { - baseUrlHttp: REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host, - baseUrlWs: REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host - } + getBandwidthBits (videoFile: MVideoFile) { + return Math.ceil((videoFile.size * 8) / this.duration) } - getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { - return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] - } + getTrackerUrls () { + if (this.isOwned()) { + return [ + WEBSERVER.URL + '/tracker/announce', + WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' + ] + } - getBandwidthBits (videoFile: MVideoFile) { - return Math.ceil((videoFile.size * 8) / this.duration) + return this.Trackers.map(t => t.url) } } -- cgit v1.2.3