From d9a2a03196275065c28f4a0b7d4d7bc9992d77a1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 18 Feb 2021 10:15:11 +0100 Subject: [PATCH] Don't guess remote tracker URL --- server/helpers/activitypub.ts | 6 +- .../custom-validators/activitypub/videos.ts | 24 +++- server/helpers/webtorrent.ts | 7 +- server/initializers/constants.ts | 2 +- server/initializers/database.ts | 4 + .../initializers/migrations/0590-trackers.ts | 44 ++++++ .../migrations/0595-remote-url.ts | 130 ++++++++++++++++++ server/lib/activitypub/videos.ts | 59 +++++++- .../schedulers/videos-redundancy-scheduler.ts | 5 +- server/lib/thumbnail.ts | 16 ++- server/models/server/tracker.ts | 73 ++++++++++ server/models/server/video-tracker.ts | 30 ++++ server/models/video/thumbnail.ts | 5 +- server/models/video/video-caption.ts | 5 +- server/models/video/video-file.ts | 9 +- server/models/video/video-format-utils.ts | 49 ++++--- server/models/video/video.ts | 54 ++++---- server/types/models/server/tracker.ts | 7 + server/types/models/video/video.ts | 4 +- .../activitypub/objects/common-objects.ts | 14 +- .../objects/video-torrent-object.ts | 3 + 21 files changed, 457 insertions(+), 93 deletions(-) create mode 100644 server/initializers/migrations/0590-trackers.ts create mode 100644 server/initializers/migrations/0595-remote-url.ts create mode 100644 server/models/server/tracker.ts create mode 100644 server/models/server/video-tracker.ts create mode 100644 server/types/models/server/tracker.ts diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 02a9d4026..08aef2908 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -201,10 +201,12 @@ function checkUrlsSameHost (url1: string, url2: string) { return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() } -function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string) { +function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string, scheme?: string) { + if (!scheme) scheme = REMOTE_SCHEME.HTTP + const host = video.VideoChannel.Actor.Server.host - return REMOTE_SCHEME.HTTP + '://' + host + path + return scheme + '://' + host + path } // --------------------------------------------------------------------------- diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index a01429c83..a41d37810 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -1,4 +1,7 @@ import validator from 'validator' +import { logger } from '@server/helpers/logger' +import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@shared/models' +import { VideoState } from '../../../../shared/models/videos' import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' import { peertubeTruncate } from '../../core-utils' import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' @@ -11,9 +14,6 @@ import { isVideoViewsValid } from '../videos' import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' -import { VideoState } from '../../../../shared/models/videos' -import { logger } from '@server/helpers/logger' -import { ActivityVideoFileMetadataObject } from '@shared/models' function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && @@ -84,6 +84,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { function isRemoteVideoUrlValid (url: any) { return url.type === 'Link' && + // Video file link ( ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) && isActivityPubUrlValid(url.href) && @@ -91,31 +92,41 @@ function isRemoteVideoUrlValid (url: any) { validator.isInt(url.size + '', { min: 0 }) && (!url.fps || validator.isInt(url.fps + '', { min: -1 })) ) || + // Torrent link ( ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) && isActivityPubUrlValid(url.href) && validator.isInt(url.height + '', { min: 0 }) ) || + // Magnet link ( ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) && validator.isLength(url.href, { min: 5 }) && validator.isInt(url.height + '', { min: 0 }) ) || + // HLS playlist link ( (url.mediaType || url.mimeType) === 'application/x-mpegURL' && isActivityPubUrlValid(url.href) && isArray(url.tag) ) || - isAPVideoFileMetadataObject(url) + isAPVideoTrackerUrlObject(url) || + isAPVideoFileUrlMetadataObject(url) } -function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject { +function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { return url && url.type === 'Link' && url.mediaType === 'application/json' && isArray(url.rel) && url.rel.includes('metadata') } +function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { + return isArray(url.rel) && + url.rel.includes('tracker') && + isActivityPubUrlValid(url.href) +} + // --------------------------------------------------------------------------- export { @@ -123,7 +134,8 @@ export { isRemoteStringIdentifierValid, sanitizeAndCheckVideoTorrentObject, isRemoteVideoUrlValid, - isAPVideoFileMetadataObject + isAPVideoFileUrlMetadataObject, + isAPVideoTrackerUrlObject } // --------------------------------------------------------------------------- diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 73418aa0a..4e08c27c6 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -107,16 +107,13 @@ async function createTorrentAndSetInfoHash ( videoFile.torrentFilename = torrentFilename } -// FIXME: merge/refactor videoOrPlaylist and video arguments function generateMagnetUri ( - videoOrPlaylist: MVideo | MStreamingPlaylistVideo, video: MVideoWithHost, videoFile: MVideoFileRedundanciesOpt, - baseUrlHttp: string, - baseUrlWs: string + trackerUrls: string[] ) { const xs = videoFile.getTorrentUrl() - const announce = videoOrPlaylist.getTrackerUrls(baseUrlHttp, baseUrlWs) + const announce = trackerUrls let urlList = [ videoFile.getFileUrl(video) ] const redundancies = videoFile.RedundancyVideos diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c4c7ffdac..fbedc2164 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 585 +const LAST_MIGRATION_VERSION = 595 // --------------------------------------------------------------------------- diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 61768234f..1f2b6d521 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -1,3 +1,5 @@ +import { TrackerModel } from '@server/models/server/tracker' +import { VideoTrackerModel } from '@server/models/server/video-tracker' import { QueryTypes, Transaction } from 'sequelize' import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' import { isTestInstance } from '../helpers/core-utils' @@ -128,6 +130,8 @@ async function initDatabaseModels (silent: boolean) { VideoPlaylistModel, VideoPlaylistElementModel, ThumbnailModel, + TrackerModel, + VideoTrackerModel, PluginModel ]) diff --git a/server/initializers/migrations/0590-trackers.ts b/server/initializers/migrations/0590-trackers.ts new file mode 100644 index 000000000..47b9022a3 --- /dev/null +++ b/server/initializers/migrations/0590-trackers.ts @@ -0,0 +1,44 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + { + const query = `CREATE TABLE IF NOT EXISTS "tracker" ( + "id" serial, + "url" varchar(255) NOT NULL, + "createdAt" timestamp WITH time zone NOT NULL, + "updatedAt" timestamp WITH time zone NOT NULL, + PRIMARY KEY ("id") + );` + + await utils.sequelize.query(query) + } + + { + const query = `CREATE TABLE IF NOT EXISTS "videoTracker" ( + "videoId" integer REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "trackerId" integer REFERENCES "tracker" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp WITH time zone NOT NULL, + "updatedAt" timestamp WITH time zone NOT NULL, + UNIQUE ("videoId", "trackerId"), + PRIMARY KEY ("videoId", "trackerId") + );` + + await utils.sequelize.query(query) + } + + await utils.sequelize.query(`CREATE UNIQUE INDEX "tracker_url" ON "tracker" ("url")`) +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0595-remote-url.ts b/server/initializers/migrations/0595-remote-url.ts new file mode 100644 index 000000000..85b367555 --- /dev/null +++ b/server/initializers/migrations/0595-remote-url.ts @@ -0,0 +1,130 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + + // Torrent and file URLs + { + const fromQueryWebtorrent = `SELECT 'https://' || server.host AS "serverUrl", '/static/webseed/' AS "filePath", "videoFile".id ` + + `FROM video ` + + `INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` + + `INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` + + `INNER JOIN server ON server.id = actor."serverId" ` + + `INNER JOIN "videoFile" ON "videoFile"."videoId" = video.id ` + + `WHERE video.remote IS TRUE` + + const fromQueryHLS = `SELECT 'https://' || server.host AS "serverUrl", ` + + `'/static/streaming-playlists/hls/' || video.uuid || '/' AS "filePath", "videoFile".id ` + + `FROM video ` + + `INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` + + `INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` + + `INNER JOIN server ON server.id = actor."serverId" ` + + `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."videoId" = video.id ` + + `INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ` + + `WHERE video.remote IS TRUE` + + for (const fromQuery of [ fromQueryWebtorrent, fromQueryHLS ]) { + const query = `UPDATE "videoFile" ` + + `SET "torrentUrl" = t."serverUrl" || '/static/torrents/' || "videoFile"."torrentFilename", ` + + `"fileUrl" = t."serverUrl" || t."filePath" || "videoFile"."filename" ` + + `FROM (${fromQuery}) AS t WHERE t.id = "videoFile"."id" AND "videoFile"."fileUrl" IS NULL` + + await utils.sequelize.query(query) + } + } + + // Caption URLs + { + const fromQuery = `SELECT 'https://' || server.host AS "serverUrl", "video".uuid, "videoCaption".id ` + + `FROM video ` + + `INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` + + `INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` + + `INNER JOIN server ON server.id = actor."serverId" ` + + `INNER JOIN "videoCaption" ON "videoCaption"."videoId" = video.id ` + + `WHERE video.remote IS TRUE` + + const query = `UPDATE "videoCaption" ` + + `SET "fileUrl" = t."serverUrl" || '/lazy-static/video-captions/' || t.uuid || '-' || "videoCaption"."language" || '.vtt' ` + + `FROM (${fromQuery}) AS t WHERE t.id = "videoCaption"."id" AND "videoCaption"."fileUrl" IS NULL` + + await utils.sequelize.query(query) + } + + // Thumbnail URLs + { + const fromQuery = `SELECT 'https://' || server.host AS "serverUrl", "video".uuid, "thumbnail".id ` + + `FROM video ` + + `INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` + + `INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` + + `INNER JOIN server ON server.id = actor."serverId" ` + + `INNER JOIN "thumbnail" ON "thumbnail"."videoId" = video.id ` + + `WHERE video.remote IS TRUE` + + // Thumbnails + { + const query = `UPDATE "thumbnail" ` + + `SET "fileUrl" = t."serverUrl" || '/static/thumbnails/' || t.uuid || '.jpg' ` + + `FROM (${fromQuery}) AS t WHERE t.id = "thumbnail"."id" AND "thumbnail"."fileUrl" IS NULL AND thumbnail.type = 1` + + await utils.sequelize.query(query) + } + + { + // Previews + const query = `UPDATE "thumbnail" ` + + `SET "fileUrl" = t."serverUrl" || '/lazy-static/previews/' || t.uuid || '.jpg' ` + + `FROM (${fromQuery}) AS t WHERE t.id = "thumbnail"."id" AND "thumbnail"."fileUrl" IS NULL AND thumbnail.type = 2` + + await utils.sequelize.query(query) + } + } + + // Trackers + { + const trackerUrls = [ + `'https://' || server.host || '/tracker/announce'`, + `'wss://' || server.host || '/tracker/socket'` + ] + + for (const trackerUrl of trackerUrls) { + { + const query = `INSERT INTO "tracker" ("url", "createdAt", "updatedAt") ` + + `SELECT ${trackerUrl} AS "url", NOW(), NOW() ` + + `FROM video ` + + `INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` + + `INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` + + `INNER JOIN server ON server.id = actor."serverId" ` + + `WHERE video.remote IS TRUE ` + + `ON CONFLICT DO NOTHING` + + await utils.sequelize.query(query) + } + + { + const query = `INSERT INTO "videoTracker" ("videoId", "trackerId", "createdAt", "updatedAt") ` + + `SELECT video.id, (SELECT tracker.id FROM tracker WHERE url = ${trackerUrl}) AS "trackerId", NOW(), NOW()` + + `FROM video ` + + `INNER JOIN "videoChannel" ON "videoChannel".id = video."channelId" ` + + `INNER JOIN actor ON actor.id = "videoChannel"."actorId" ` + + `INNER JOIN server ON server.id = actor."serverId" ` + + `WHERE video.remote IS TRUE` + + await utils.sequelize.query(query) + } + } + } + +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index a5f6537eb..66330a964 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -3,7 +3,8 @@ import { maxBy, minBy } from 'lodash' import * as magnetUtil from 'magnet-uri' import { basename, join } from 'path' import * as request from 'request' -import * as sequelize from 'sequelize' +import { Transaction } from 'sequelize/types' +import { TrackerModel } from '@server/models/server/tracker' import { VideoLiveModel } from '@server/models/video/video-live' import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' import { @@ -16,12 +17,16 @@ import { ActivityUrlObject, ActivityVideoUrlObject } from '../../../shared/index' -import { ActivityIconObject, VideoObject } from '../../../shared/models/activitypub/objects' +import { ActivityIconObject, ActivityTrackerUrlObject, VideoObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub' -import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' +import { + isAPVideoFileUrlMetadataObject, + isAPVideoTrackerUrlObject, + sanitizeAndCheckVideoTorrentObject +} from '../../helpers/custom-validators/activitypub/videos' import { isArray } from '../../helpers/custom-validators/misc' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' @@ -83,7 +88,7 @@ import { addVideoShares, shareVideoByServerAndChannel } from './share' import { addVideoComments } from './video-comments' import { createRates } from './video-rates' -async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) { +async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) { const video = videoArg as MVideoAP if ( @@ -433,6 +438,12 @@ async function updateVideoFromAP (options: { await setVideoTags({ video: videoUpdated, tags, transaction: t, defaultValue: videoUpdated.Tags }) } + // Update trackers + { + const trackers = getTrackerUrls(videoObject, videoUpdated) + await setVideoTrackers({ video: videoUpdated, trackers, transaction: t }) + } + { // Update captions await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoUpdated.id, t) @@ -577,7 +588,7 @@ function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') } -function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { +function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { return url && url.mediaType === 'application/x-mpegURL' } @@ -671,6 +682,12 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi }) await Promise.all(videoCaptionsPromises) + // Process trackers + { + const trackers = getTrackerUrls(videoObject, videoCreated) + await setVideoTrackers({ video: videoCreated, trackers, transaction: t }) + } + videoCreated.VideoFiles = videoFiles if (videoCreated.isLive) { @@ -797,7 +814,7 @@ function videoFileActivityUrlToDBAttributes ( : parsed.xs // Fetch associated metadata url, if any - const metadata = urls.filter(isAPVideoFileMetadataObject) + const metadata = urls.filter(isAPVideoFileUrlMetadataObject) .find(u => { return u.height === fileUrl.height && u.fps === fileUrl.fps && @@ -889,3 +906,33 @@ function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoWithHost) ? previewIcon.url : buildRemoteVideoBaseUrl(video, join(LAZY_STATIC_PATHS.PREVIEWS, video.generatePreviewName())) } + +function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { + let wsFound = false + + const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) + .map((u: ActivityTrackerUrlObject) => { + if (u.rel.includes('websocket')) wsFound = true + + return u.href + }) + + if (wsFound) return trackers + + return [ + buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), + buildRemoteVideoBaseUrl(video, '/tracker/announce') + ] +} + +async function setVideoTrackers (options: { + video: MVideo + trackers: string[] + transaction?: Transaction +}) { + const { video, trackers, transaction } = options + + const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) + + await video.$set('Trackers', trackerInstances, { transaction }) +} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 60008e695..9e2667416 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,6 +1,7 @@ import { move } from 'fs-extra' import { join } from 'path' import { getServerActor } from '@server/models/application/application' +import { TrackerModel } from '@server/models/server/tracker' import { VideoModel } from '@server/models/video/video' import { MStreamingPlaylist, @@ -221,8 +222,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = generateMagnetUri(video, video, file, baseUrlHttp, baseUrlWs) + const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) + const magnetUri = generateMagnetUri(video, file, trackerUrls) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index 4bad8d6ca..49317df28 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts @@ -1,5 +1,6 @@ import { copy } from 'fs-extra' import { join } from 'path' +import { logger } from '@server/helpers/logger' import { ThumbnailType } from '../../shared/models/videos/thumbnail.type' import { generateImageFromVideoFile } from '../helpers/ffmpeg-utils' import { processImage } from '../helpers/image-utils' @@ -62,7 +63,7 @@ function createVideoMiniatureFromUrl (options: { size?: ImageSize }) { const { downloadUrl, video, type, size } = options - const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) + const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) // Only save the file URL if it is a remote video const fileUrl = video.isOwned() @@ -76,10 +77,16 @@ function createVideoMiniatureFromUrl (options: { // If the thumbnail URL did not change and has a unique filename (introduced in 3.2), avoid thumbnail processing const thumbnailUrlChanged = !existingUrl || existingUrl !== downloadUrl || downloadUrl.endsWith(`${video.uuid}.jpg`) + + // Do not change the thumbnail filename if the file did not change + const filename = thumbnailUrlChanged + ? updatedFilename + : existingThumbnail.filename + const thumbnailCreator = () => { if (thumbnailUrlChanged) return downloadImage(downloadUrl, basePath, filename, { width, height }) - return copy(existingThumbnail.getPath(), ThumbnailModel.buildPath(type, filename)) + return Promise.resolve() } return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) @@ -236,7 +243,7 @@ async function createThumbnailFromFunction (parameters: { fileUrl = null } = parameters - const oldFilename = existingThumbnail + const oldFilename = existingThumbnail && existingThumbnail.filename !== filename ? existingThumbnail.filename : undefined @@ -248,7 +255,8 @@ async function createThumbnailFromFunction (parameters: { thumbnail.type = type thumbnail.fileUrl = fileUrl thumbnail.automaticallyGenerated = automaticallyGenerated - thumbnail.previousThumbnailFilename = oldFilename + + if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename await thumbnailCreator() diff --git a/server/models/server/tracker.ts b/server/models/server/tracker.ts new file mode 100644 index 000000000..d7c91faad --- /dev/null +++ b/server/models/server/tracker.ts @@ -0,0 +1,73 @@ +import { AllowNull, BelongsToMany, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { Transaction } from 'sequelize/types' +import { MTracker } from '@server/types/models/server/tracker' +import { VideoModel } from '../video/video' +import { VideoTrackerModel } from './video-tracker' + +@Table({ + tableName: 'tracker', + indexes: [ + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class TrackerModel extends Model { + + @AllowNull(false) + @Column + url: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @BelongsToMany(() => VideoModel, { + foreignKey: 'trackerId', + through: () => VideoTrackerModel, + onDelete: 'CASCADE' + }) + Videos: VideoModel[] + + static listUrlsByVideoId (videoId: number) { + const query = { + include: [ + { + attributes: [ 'id', 'trackerId' ], + model: VideoModel.unscoped(), + required: true, + where: { id: videoId } + } + ] + } + + return TrackerModel.findAll(query) + .then(rows => rows.map(rows => rows.url)) + } + + static findOrCreateTrackers (trackers: string[], transaction: Transaction): Promise { + if (trackers === null) return Promise.resolve([]) + + const tasks: Promise[] = [] + trackers.forEach(tracker => { + const query = { + where: { + url: tracker + }, + defaults: { + url: tracker + }, + transaction + } + + const promise = TrackerModel.findOrCreate(query) + .then(([ trackerInstance ]) => trackerInstance) + tasks.push(promise) + }) + + return Promise.all(tasks) + } +} diff --git a/server/models/server/video-tracker.ts b/server/models/server/video-tracker.ts new file mode 100644 index 000000000..367bf0117 --- /dev/null +++ b/server/models/server/video-tracker.ts @@ -0,0 +1,30 @@ +import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoModel } from '../video/video' +import { TrackerModel } from './tracker' + +@Table({ + tableName: 'videoTracker', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'trackerId' ] + } + ] +}) +export class VideoTrackerModel extends Model { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @ForeignKey(() => TrackerModel) + @Column + trackerId: number +} diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index 9533c8d19..319e1175a 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -15,7 +15,6 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' import { afterCommitIfTransaction } from '@server/helpers/database-utils' import { MThumbnail, MThumbnailVideo, MVideoWithHost } from '@server/types/models' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' @@ -168,10 +167,8 @@ export class ThumbnailModel extends Model { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename if (video.isOwned()) return WEBSERVER.URL + staticPath - if (this.fileUrl) return this.fileUrl - // Fallback if we don't have a file URL - return buildRemoteVideoBaseUrl(video, staticPath) + return this.fileUrl } getPath () { diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 71b067335..0bbe9b752 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -16,7 +16,6 @@ import { UpdatedAt } from 'sequelize-typescript' import { v4 as uuidv4 } from 'uuid' -import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' import { MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo, MVideoWithHost } from '@server/types/models' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' @@ -208,9 +207,7 @@ export class VideoCaptionModel extends Model { if (!this.Video) this.Video = video as VideoModel if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() - if (this.fileUrl) return this.fileUrl - // Fallback if we don't have a file URL - return buildRemoteVideoBaseUrl(video, this.getCaptionStaticPath()) + return this.fileUrl } } diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 57807cbfd..5a3706259 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -427,10 +427,8 @@ export class VideoFileModel extends Model { if (!this.Video) this.Video = video as VideoModel if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) - if (this.fileUrl) return this.fileUrl - // Fallback if we don't have a file URL - return buildRemoteVideoBaseUrl(video, this.getFileStaticPath(video)) + return this.fileUrl } getFileStaticPath (video: MVideo) { @@ -454,10 +452,7 @@ export class VideoFileModel extends Model { getRemoteTorrentUrl (video: MVideoWithHost) { if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) - if (this.torrentUrl) return this.torrentUrl - - // Fallback if we don't have a torrent URL - return buildRemoteVideoBaseUrl(video, this.getTorrentStaticPath()) + return this.torrentUrl } // We proxify torrent requests so use a local URL diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index adf460734..9dc3e7722 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -14,8 +14,6 @@ import { } from '../../lib/activitypub/url' import { MStreamingPlaylistRedundanciesOpt, - MStreamingPlaylistVideo, - MVideo, MVideoAP, MVideoFile, MVideoFormattable, @@ -127,8 +125,6 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid } }) - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const tags = video.Tags ? video.Tags.map(t => t.name) : [] const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) @@ -147,14 +143,14 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid label: VideoModel.getStateLabel(video.state) }, - trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs), + trackerUrls: video.getTrackerUrls(), files: [], streamingPlaylists } // Format and sort video files - detailsJson.files = videoFilesModelToFormattedJSON(video, video, baseUrlHttp, baseUrlWs, video.VideoFiles) + detailsJson.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) return Object.assign(formattedJson, detailsJson) } @@ -165,17 +161,13 @@ function streamingPlaylistsModelToFormattedJSON ( ): VideoStreamingPlaylist[] { if (isArray(playlists) === false) return [] - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - return playlists .map(playlist => { - const playlistWithVideo = Object.assign(playlist, { Video: video }) - const redundancies = isArray(playlist.RedundancyVideos) ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) : [] - const files = videoFilesModelToFormattedJSON(playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles) + const files = videoFilesModelToFormattedJSON(video, playlist.VideoFiles) return { id: playlist.id, @@ -194,14 +186,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) { return -1 } -// FIXME: refactor/merge model and video arguments function videoFilesModelToFormattedJSON ( - model: MVideo | MStreamingPlaylistVideo, video: MVideoFormattableDetails, - baseUrlHttp: string, - baseUrlWs: string, videoFiles: MVideoFileRedundanciesOpt[] ): VideoFile[] { + const trackerUrls = video.getTrackerUrls() + return [ ...videoFiles ] .filter(f => !f.isLive()) .sort(sortByResolutionDesc) @@ -213,7 +203,7 @@ function videoFilesModelToFormattedJSON ( }, // FIXME: deprecated in 3.2 - magnetUri: generateMagnetUri(model, video, videoFile, baseUrlHttp, baseUrlWs), + magnetUri: generateMagnetUri(video, videoFile, trackerUrls), size: videoFile.size, fps: videoFile.fps, @@ -229,15 +219,13 @@ function videoFilesModelToFormattedJSON ( }) } -// FIXME: refactor/merge model and video arguments function addVideoFilesInAPAcc ( acc: ActivityUrlObject[] | ActivityTagObject[], - model: MVideoAP | MStreamingPlaylistVideo, video: MVideoWithHost, - baseUrlHttp: string, - baseUrlWs: string, files: MVideoFile[] ) { + const trackerUrls = video.getTrackerUrls() + const sortedFiles = [ ...files ] .filter(f => !f.isLive()) .sort(sortByResolutionDesc) @@ -271,14 +259,13 @@ function addVideoFilesInAPAcc ( acc.push({ type: 'Link', mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: generateMagnetUri(model, video, file, baseUrlHttp, baseUrlWs), + href: generateMagnetUri(video, file, trackerUrls), height: file.resolution }) } } function videoModelToActivityPubObject (video: MVideoAP): VideoObject { - const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() if (!video.Tags) video.Tags = [] const tag = video.Tags.map(t => ({ @@ -319,7 +306,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { } ] - addVideoFilesInAPAcc(url, video, video, baseUrlHttp, baseUrlWs, video.VideoFiles || []) + addVideoFilesInAPAcc(url, video, video.VideoFiles || []) for (const playlist of (video.VideoStreamingPlaylists || [])) { const tag = playlist.p2pMediaLoaderInfohashes @@ -331,8 +318,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { href: playlist.segmentsSha256Url }) - const playlistWithVideo = Object.assign(playlist, { Video: video }) - addVideoFilesInAPAcc(tag, playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles || []) + addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) url.push({ type: 'Link', @@ -342,6 +328,19 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { }) } + for (const trackerUrl of video.getTrackerUrls()) { + const rel2 = trackerUrl.startsWith('http') + ? 'http' + : 'websocket' + + url.push({ + type: 'Link', + name: `tracker-${rel2}`, + rel: [ 'tracker', rel2 ], + href: trackerUrl + }) + } + const subtitleLanguage = [] for (const caption of video.VideoCaptions) { subtitleLanguage.push({ diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 2e6b6aeec..9e67ca0f4 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -60,7 +60,6 @@ import { API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, - REMOTE_SCHEME, STATIC_PATHS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, @@ -107,6 +106,8 @@ import { ActorModel } from '../activitypub/actor' import { AvatarModel } from '../avatar/avatar' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { ServerModel } from '../server/server' +import { TrackerModel } from '../server/tracker' +import { VideoTrackerModel } from '../server/video-tracker' import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils' import { ScheduleVideoUpdateModel } from './schedule-video-update' import { TagModel } from './tag' @@ -137,6 +138,7 @@ export enum ScopeNames { FOR_API = 'FOR_API', WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS', WITH_TAGS = 'WITH_TAGS', + WITH_TRACKERS = 'WITH_TRACKERS', WITH_WEBTORRENT_FILES = 'WITH_WEBTORRENT_FILES', WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', WITH_BLACKLISTED = 'WITH_BLACKLISTED', @@ -320,6 +322,14 @@ export type AvailableForListIDsOptions = { [ScopeNames.WITH_TAGS]: { include: [ TagModel ] }, + [ScopeNames.WITH_TRACKERS]: { + include: [ + { + attributes: [ 'id', 'url' ], + model: TrackerModel + } + ] + }, [ScopeNames.WITH_BLACKLISTED]: { include: [ { @@ -616,6 +626,13 @@ export class VideoModel extends Model { }) Tags: TagModel[] + @BelongsToMany(() => TrackerModel, { + foreignKey: 'videoId', + through: () => VideoTrackerModel, + onDelete: 'CASCADE' + }) + Trackers: TrackerModel[] + @HasMany(() => ThumbnailModel, { foreignKey: { name: 'videoId', @@ -1436,6 +1453,7 @@ export class VideoModel extends Model { ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_THUMBNAILS, ScopeNames.WITH_LIVE, + ScopeNames.WITH_TRACKERS, { method: [ ScopeNames.WITH_WEBTORRENT_FILES, true ] }, { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } ] @@ -1887,18 +1905,15 @@ export class VideoModel extends Model { } getFormattedVideoFilesJSON (): VideoFile[] { - const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() let files: VideoFile[] = [] if (Array.isArray(this.VideoFiles)) { - const result = videoFilesModelToFormattedJSON(this, this, baseUrlHttp, baseUrlWs, this.VideoFiles) + const result = videoFilesModelToFormattedJSON(this, this.VideoFiles) files = files.concat(result) } for (const p of (this.VideoStreamingPlaylists || [])) { - p.Video = this - - const result = videoFilesModelToFormattedJSON(p, this, baseUrlHttp, baseUrlWs, p.VideoFiles) + const result = videoFilesModelToFormattedJSON(this, p.VideoFiles) files = files.concat(result) } @@ -2030,25 +2045,18 @@ export class VideoModel extends Model { return false } - getBaseUrls () { - if (this.isOwned()) { - return { - baseUrlHttp: WEBSERVER.URL, - baseUrlWs: WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT - } - } - - return { - baseUrlHttp: REMOTE_SCHEME.HTTP + '://' + this.VideoChannel.Account.Actor.Server.host, - baseUrlWs: REMOTE_SCHEME.WS + '://' + this.VideoChannel.Account.Actor.Server.host - } + getBandwidthBits (videoFile: MVideoFile) { + return Math.ceil((videoFile.size * 8) / this.duration) } - getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { - return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] - } + getTrackerUrls () { + if (this.isOwned()) { + return [ + WEBSERVER.URL + '/tracker/announce', + WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' + ] + } - getBandwidthBits (videoFile: MVideoFile) { - return Math.ceil((videoFile.size * 8) / this.duration) + return this.Trackers.map(t => t.url) } } diff --git a/server/types/models/server/tracker.ts b/server/types/models/server/tracker.ts new file mode 100644 index 000000000..5fe03f8c0 --- /dev/null +++ b/server/types/models/server/tracker.ts @@ -0,0 +1,7 @@ +import { TrackerModel } from '../../../models/server/tracker' + +export type MTracker = Omit + +// ############################################################################ + +export type MTrackerUrl = Pick diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts index 92dcbaf59..692490230 100644 --- a/server/types/models/video/video.ts +++ b/server/types/models/video/video.ts @@ -1,5 +1,6 @@ import { PickWith, PickWithOpt } from '@shared/core-utils' import { VideoModel } from '../../../models/video/video' +import { MTrackerUrl } from '../server/tracker' import { MUserVideoHistoryTime } from '../user/user-video-history' import { MScheduleVideoUpdate } from './schedule-video-update' import { MTag } from './tag' @@ -216,4 +217,5 @@ export type MVideoFormattableDetails = Use<'VideoChannel', MChannelFormattable> & Use<'Tags', MTag[]> & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> + Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'Trackers', MTrackerUrl[]> diff --git a/shared/models/activitypub/objects/common-objects.ts b/shared/models/activitypub/objects/common-objects.ts index 43e8ea067..76f0e3bcf 100644 --- a/shared/models/activitypub/objects/common-objects.ts +++ b/shared/models/activitypub/objects/common-objects.ts @@ -30,7 +30,7 @@ export type ActivityPlaylistSegmentHashesObject = { href: string } -export type ActivityVideoFileMetadataObject = { +export type ActivityVideoFileMetadataUrlObject = { type: 'Link' rel: [ 'metadata', any ] mediaType: 'application/json' @@ -39,6 +39,13 @@ export type ActivityVideoFileMetadataObject = { fps: number } +export type ActivityTrackerUrlObject = { + type: 'Link' + rel: [ 'tracker', 'websocket' | 'http' ] + name: string + href: string +} + export type ActivityPlaylistInfohashesObject = { type: 'Infohash' name: string @@ -96,7 +103,7 @@ export type ActivityTagObject = | ActivityMentionObject | ActivityBitTorrentUrlObject | ActivityMagnetUrlObject - | ActivityVideoFileMetadataObject + | ActivityVideoFileMetadataUrlObject export type ActivityUrlObject = ActivityVideoUrlObject @@ -104,7 +111,8 @@ export type ActivityUrlObject = | ActivityBitTorrentUrlObject | ActivityMagnetUrlObject | ActivityHtmlUrlObject - | ActivityVideoFileMetadataObject + | ActivityVideoFileMetadataUrlObject + | ActivityTrackerUrlObject export interface ActivityPubAttributedTo { type: 'Group' | 'Person' diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-torrent-object.ts index 6d18e93d5..bfbcfb1a5 100644 --- a/shared/models/activitypub/objects/video-torrent-object.ts +++ b/shared/models/activitypub/objects/video-torrent-object.ts @@ -40,11 +40,14 @@ export interface VideoObject { icon: ActivityIconObject[] url: ActivityUrlObject[] + likes: string dislikes: string shares: string comments: string + attributedTo: ActivityPubAttributedTo[] + to?: string[] cc?: string[] } -- 2.41.0