From ae9bbed46dbc8d9870c9feb66bbada484c1c7582 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Apr 2019 11:13:49 +0200 Subject: Update P2P media loader peer version --- server.ts | 4 +++ server/helpers/core-utils.ts | 2 +- server/helpers/video.ts | 3 -- server/initializers/constants.ts | 5 ++- .../migrations/0355-p2p-peer-version.ts | 41 ++++++++++++++++++++++ server/lib/activitypub/videos.ts | 22 +++++------- server/lib/hls.ts | 22 ++++++++++-- server/models/video/video-file.ts | 24 +++++++++++++ server/models/video/video-streaming-playlist.ts | 22 ++++++++++-- 9 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 server/initializers/migrations/0355-p2p-peer-version.ts diff --git a/server.ts b/server.ts index df56bcd82..dfaa8ad3d 100644 --- a/server.ts +++ b/server.ts @@ -103,6 +103,7 @@ import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-upd import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' import { PeerTubeSocket } from './server/lib/peertube-socket' +import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' // ----------- Command line ----------- @@ -233,6 +234,9 @@ async function startApplication () { PeerTubeSocket.Instance.init(server) + updateStreamingPlaylistsInfohashesIfNeeded() + .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err })) + // Make server listening server.listen(port, hostname, () => { logger.info('Server listening on %s:%d', hostname, port) diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index f38b82d97..3f737c1d6 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -58,7 +58,7 @@ export function parseDuration (duration: number | string): number { } } - throw new Error('Duration could not be properly parsed') + throw new Error(`Duration ${duration} could not be properly parsed`) } export function parseBytes (value: string | number): number { diff --git a/server/helpers/video.ts b/server/helpers/video.ts index f6f51a297..c90fe06c7 100644 --- a/server/helpers/video.ts +++ b/server/helpers/video.ts @@ -1,7 +1,4 @@ -import { CONFIG } from '../initializers' import { VideoModel } from '../models/video/video' -import { UserRight } from '../../shared' -import { UserModel } from '../models/account/user' type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f59d3ef7a..ac19231d0 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -18,7 +18,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 350 +const LAST_MIGRATION_VERSION = 355 // --------------------------------------------------------------------------- @@ -726,6 +726,8 @@ const TRACKER_RATE_LIMITS = { ANNOUNCES_PER_IP: 30 // maximum announces for all our torrents in the interval } +const P2P_MEDIA_LOADER_PEER_VERSION = 2 + // --------------------------------------------------------------------------- // Special constants for a test instance @@ -772,6 +774,7 @@ updateWebserverUrls() export { API_VERSION, HLS_REDUNDANCY_DIRECTORY, + P2P_MEDIA_LOADER_PEER_VERSION, AVATARS_SIZE, ACCEPT_HEADERS, BCRYPT_SALT_SIZE, diff --git a/server/initializers/migrations/0355-p2p-peer-version.ts b/server/initializers/migrations/0355-p2p-peer-version.ts new file mode 100644 index 000000000..18f23d9b7 --- /dev/null +++ b/server/initializers/migrations/0355-p2p-peer-version.ts @@ -0,0 +1,41 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + + { + const data = { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: null + } + await utils.queryInterface.addColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data) + } + + { + const query = `UPDATE "videoStreamingPlaylist" SET "p2pMediaLoaderPeerVersion" = 0;` + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: null + } + await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index d935e3f90..339f8e797 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -290,7 +290,11 @@ async function updateVideoFromAP (options: { } { - const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) + const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes( + options.video, + options.videoObject, + options.video.VideoFiles + ) const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) // Remove video files that do not exist anymore @@ -449,9 +453,9 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor } const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) - await Promise.all(videoFilePromises) + const videoFiles = await Promise.all(videoFilePromises) - const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) + const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) await Promise.all(playlistPromises) @@ -575,20 +579,12 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid return attributes } -function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { +function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) { const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] if (playlistUrls.length === 0) return [] const attributes: FilteredModelAttributes[] = [] for (const playlistUrlObject of playlistUrls) { - const p2pMediaLoaderInfohashes = playlistUrlObject.tag - .filter(t => t.type === 'Infohash') - .map(t => t.name) - if (p2pMediaLoaderInfohashes.length === 0) { - logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject }) - continue - } - const segmentsSha256UrlObject = playlistUrlObject.tag .find(t => { return isAPPlaylistSegmentHashesUrlObject(t) @@ -602,7 +598,7 @@ function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObj type: VideoStreamingPlaylistType.HLS, playlistUrl: playlistUrlObject.href, segmentsSha256Url: segmentsSha256UrlObject.href, - p2pMediaLoaderInfohashes, + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles), videoId: video.id } diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 74ed25183..5a7d61dee 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,6 +1,6 @@ import { VideoModel } from '../models/video/video' -import { basename, join, dirname } from 'path' -import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY } from '../initializers' +import { basename, dirname, join } from 'path' +import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, sequelizeTypescript } from '../initializers' import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' import { getVideoFileSize } from '../helpers/ffmpeg-utils' import { sha256 } from '../helpers/core-utils' @@ -9,6 +9,21 @@ import { logger } from '../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' import { generateRandomString } from '../helpers/utils' import { flatten, uniq } from 'lodash' +import { VideoFileModel } from '../models/video/video-file' + +async function updateStreamingPlaylistsInfohashesIfNeeded () { + const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() + + // Use separate SQL queries, because we could have many videos to update + for (const playlist of playlistsToUpdate) { + await sequelizeTypescript.transaction(async t => { + const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) + + playlist.p2pMediaLoaderInfohashes = await VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) + await playlist.save({ transaction: t }) + }) + } +} async function updateMasterHLSPlaylist (video: VideoModel) { const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) @@ -159,7 +174,8 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, export { updateMasterHLSPlaylist, updateSha256Segments, - downloadPlaylistSegments + downloadPlaylistSegments, + updateStreamingPlaylistsInfohashesIfNeeded } // --------------------------------------------------------------------------- diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index b861b0704..c14d96bc5 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -23,6 +23,7 @@ import { throwIfNotValid } from '../utils' import { VideoModel } from './video' import * as Sequelize from 'sequelize' import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist' @Table({ tableName: 'videoFile', @@ -120,6 +121,29 @@ export class VideoFileModel extends Model { return VideoFileModel.findByPk(id, options) } + static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Sequelize.Transaction) { + const query = { + include: [ + { + model: VideoModel.unscoped(), + required: true, + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + required: true, + where: { + id: streamingPlaylistId + } + } + ] + } + ], + transaction + } + + return VideoFileModel.findAll(query) + } + static async getStats () { let totalLocalVideoFilesSize = await VideoFileModel.sum('size', { include: [ diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index b147aca36..0333755c5 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -6,7 +6,7 @@ import * as Sequelize from 'sequelize' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers' +import { CONSTRAINTS_FIELDS, STATIC_PATHS, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers' import { VideoFileModel } from './video-file' import { join } from 'path' import { sha1 } from '../../helpers/core-utils' @@ -49,6 +49,10 @@ export class VideoStreamingPlaylistModel extends Model throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) @Column @@ -92,14 +96,26 @@ export class VideoStreamingPlaylistModel extends Model