aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server.ts4
-rw-r--r--server/helpers/core-utils.ts2
-rw-r--r--server/helpers/video.ts3
-rw-r--r--server/initializers/constants.ts5
-rw-r--r--server/initializers/migrations/0355-p2p-peer-version.ts41
-rw-r--r--server/lib/activitypub/videos.ts22
-rw-r--r--server/lib/hls.ts22
-rw-r--r--server/models/video/video-file.ts24
-rw-r--r--server/models/video/video-streaming-playlist.ts22
9 files changed, 121 insertions, 24 deletions
diff --git a/server.ts b/server.ts
index df56bcd82..dfaa8ad3d 100644
--- a/server.ts
+++ b/server.ts
@@ -103,6 +103,7 @@ import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-upd
103import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' 103import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
104import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' 104import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
105import { PeerTubeSocket } from './server/lib/peertube-socket' 105import { PeerTubeSocket } from './server/lib/peertube-socket'
106import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
106 107
107// ----------- Command line ----------- 108// ----------- Command line -----------
108 109
@@ -233,6 +234,9 @@ async function startApplication () {
233 234
234 PeerTubeSocket.Instance.init(server) 235 PeerTubeSocket.Instance.init(server)
235 236
237 updateStreamingPlaylistsInfohashesIfNeeded()
238 .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
239
236 // Make server listening 240 // Make server listening
237 server.listen(port, hostname, () => { 241 server.listen(port, hostname, () => {
238 logger.info('Server listening on %s:%d', hostname, port) 242 logger.info('Server listening on %s:%d', hostname, port)
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index f38b82d97..3f737c1d6 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -58,7 +58,7 @@ export function parseDuration (duration: number | string): number {
58 } 58 }
59 } 59 }
60 60
61 throw new Error('Duration could not be properly parsed') 61 throw new Error(`Duration ${duration} could not be properly parsed`)
62} 62}
63 63
64export function parseBytes (value: string | number): number { 64export function parseBytes (value: string | number): number {
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index f6f51a297..c90fe06c7 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,7 +1,4 @@
1import { CONFIG } from '../initializers'
2import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
3import { UserRight } from '../../shared'
4import { UserModel } from '../models/account/user'
5 2
6type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' 3type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'
7 4
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f59d3ef7a..ac19231d0 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -18,7 +18,7 @@ let config: IConfig = require('config')
18 18
19// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
20 20
21const LAST_MIGRATION_VERSION = 350 21const LAST_MIGRATION_VERSION = 355
22 22
23// --------------------------------------------------------------------------- 23// ---------------------------------------------------------------------------
24 24
@@ -726,6 +726,8 @@ const TRACKER_RATE_LIMITS = {
726 ANNOUNCES_PER_IP: 30 // maximum announces for all our torrents in the interval 726 ANNOUNCES_PER_IP: 30 // maximum announces for all our torrents in the interval
727} 727}
728 728
729const P2P_MEDIA_LOADER_PEER_VERSION = 2
730
729// --------------------------------------------------------------------------- 731// ---------------------------------------------------------------------------
730 732
731// Special constants for a test instance 733// Special constants for a test instance
@@ -772,6 +774,7 @@ updateWebserverUrls()
772export { 774export {
773 API_VERSION, 775 API_VERSION,
774 HLS_REDUNDANCY_DIRECTORY, 776 HLS_REDUNDANCY_DIRECTORY,
777 P2P_MEDIA_LOADER_PEER_VERSION,
775 AVATARS_SIZE, 778 AVATARS_SIZE,
776 ACCEPT_HEADERS, 779 ACCEPT_HEADERS,
777 BCRYPT_SALT_SIZE, 780 BCRYPT_SALT_SIZE,
diff --git a/server/initializers/migrations/0355-p2p-peer-version.ts b/server/initializers/migrations/0355-p2p-peer-version.ts
new file mode 100644
index 000000000..18f23d9b7
--- /dev/null
+++ b/server/initializers/migrations/0355-p2p-peer-version.ts
@@ -0,0 +1,41 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize,
7 db: any
8}): Promise<void> {
9
10 {
11 const data = {
12 type: Sequelize.INTEGER,
13 allowNull: true,
14 defaultValue: null
15 }
16 await utils.queryInterface.addColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data)
17 }
18
19 {
20 const query = `UPDATE "videoStreamingPlaylist" SET "p2pMediaLoaderPeerVersion" = 0;`
21 await utils.sequelize.query(query)
22 }
23
24 {
25 const data = {
26 type: Sequelize.INTEGER,
27 allowNull: false,
28 defaultValue: null
29 }
30 await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data)
31 }
32}
33
34function down (options) {
35 throw new Error('Not implemented.')
36}
37
38export {
39 up,
40 down
41}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index d935e3f90..339f8e797 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -290,7 +290,11 @@ async function updateVideoFromAP (options: {
290 } 290 }
291 291
292 { 292 {
293 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject) 293 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
294 options.video,
295 options.videoObject,
296 options.video.VideoFiles
297 )
294 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) 298 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
295 299
296 // Remove video files that do not exist anymore 300 // Remove video files that do not exist anymore
@@ -449,9 +453,9 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
449 } 453 }
450 454
451 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 455 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
452 await Promise.all(videoFilePromises) 456 const videoFiles = await Promise.all(videoFilePromises)
453 457
454 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject) 458 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
455 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) 459 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
456 await Promise.all(playlistPromises) 460 await Promise.all(playlistPromises)
457 461
@@ -575,20 +579,12 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
575 return attributes 579 return attributes
576} 580}
577 581
578function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { 582function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
579 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] 583 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
580 if (playlistUrls.length === 0) return [] 584 if (playlistUrls.length === 0) return []
581 585
582 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] 586 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
583 for (const playlistUrlObject of playlistUrls) { 587 for (const playlistUrlObject of playlistUrls) {
584 const p2pMediaLoaderInfohashes = playlistUrlObject.tag
585 .filter(t => t.type === 'Infohash')
586 .map(t => t.name)
587 if (p2pMediaLoaderInfohashes.length === 0) {
588 logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
589 continue
590 }
591
592 const segmentsSha256UrlObject = playlistUrlObject.tag 588 const segmentsSha256UrlObject = playlistUrlObject.tag
593 .find(t => { 589 .find(t => {
594 return isAPPlaylistSegmentHashesUrlObject(t) 590 return isAPPlaylistSegmentHashesUrlObject(t)
@@ -602,7 +598,7 @@ function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObj
602 type: VideoStreamingPlaylistType.HLS, 598 type: VideoStreamingPlaylistType.HLS,
603 playlistUrl: playlistUrlObject.href, 599 playlistUrl: playlistUrlObject.href,
604 segmentsSha256Url: segmentsSha256UrlObject.href, 600 segmentsSha256Url: segmentsSha256UrlObject.href,
605 p2pMediaLoaderInfohashes, 601 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
606 videoId: video.id 602 videoId: video.id
607 } 603 }
608 604
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 74ed25183..5a7d61dee 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,6 +1,6 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2import { basename, join, dirname } from 'path' 2import { basename, dirname, join } from 'path'
3import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY } from '../initializers' 3import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, sequelizeTypescript } from '../initializers'
4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' 4import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils' 5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils' 6import { sha256 } from '../helpers/core-utils'
@@ -9,6 +9,21 @@ import { logger } from '../helpers/logger'
9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' 9import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
10import { generateRandomString } from '../helpers/utils' 10import { generateRandomString } from '../helpers/utils'
11import { flatten, uniq } from 'lodash' 11import { flatten, uniq } from 'lodash'
12import { VideoFileModel } from '../models/video/video-file'
13
14async function updateStreamingPlaylistsInfohashesIfNeeded () {
15 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
16
17 // Use separate SQL queries, because we could have many videos to update
18 for (const playlist of playlistsToUpdate) {
19 await sequelizeTypescript.transaction(async t => {
20 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
21
22 playlist.p2pMediaLoaderInfohashes = await VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
23 await playlist.save({ transaction: t })
24 })
25 }
26}
12 27
13async function updateMasterHLSPlaylist (video: VideoModel) { 28async function updateMasterHLSPlaylist (video: VideoModel) {
14 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 29 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
@@ -159,7 +174,8 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
159export { 174export {
160 updateMasterHLSPlaylist, 175 updateMasterHLSPlaylist,
161 updateSha256Segments, 176 updateSha256Segments,
162 downloadPlaylistSegments 177 downloadPlaylistSegments,
178 updateStreamingPlaylistsInfohashesIfNeeded
163} 179}
164 180
165// --------------------------------------------------------------------------- 181// ---------------------------------------------------------------------------
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index b861b0704..c14d96bc5 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -23,6 +23,7 @@ import { throwIfNotValid } from '../utils'
23import { VideoModel } from './video' 23import { VideoModel } from './video'
24import * as Sequelize from 'sequelize' 24import * as Sequelize from 'sequelize'
25import { VideoRedundancyModel } from '../redundancy/video-redundancy' 25import { VideoRedundancyModel } from '../redundancy/video-redundancy'
26import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
26 27
27@Table({ 28@Table({
28 tableName: 'videoFile', 29 tableName: 'videoFile',
@@ -120,6 +121,29 @@ export class VideoFileModel extends Model<VideoFileModel> {
120 return VideoFileModel.findByPk(id, options) 121 return VideoFileModel.findByPk(id, options)
121 } 122 }
122 123
124 static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Sequelize.Transaction) {
125 const query = {
126 include: [
127 {
128 model: VideoModel.unscoped(),
129 required: true,
130 include: [
131 {
132 model: VideoStreamingPlaylistModel.unscoped(),
133 required: true,
134 where: {
135 id: streamingPlaylistId
136 }
137 }
138 ]
139 }
140 ],
141 transaction
142 }
143
144 return VideoFileModel.findAll(query)
145 }
146
123 static async getStats () { 147 static async getStats () {
124 let totalLocalVideoFilesSize = await VideoFileModel.sum('size', { 148 let totalLocalVideoFilesSize = await VideoFileModel.sum('size', {
125 include: [ 149 include: [
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index b147aca36..0333755c5 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -6,7 +6,7 @@ import * as Sequelize from 'sequelize'
6import { VideoRedundancyModel } from '../redundancy/video-redundancy' 6import { VideoRedundancyModel } from '../redundancy/video-redundancy'
7import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 7import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
8import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 8import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
9import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers' 9import { CONSTRAINTS_FIELDS, STATIC_PATHS, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers'
10import { VideoFileModel } from './video-file' 10import { VideoFileModel } from './video-file'
11import { join } from 'path' 11import { join } from 'path'
12import { sha1 } from '../../helpers/core-utils' 12import { sha1 } from '../../helpers/core-utils'
@@ -50,6 +50,10 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
50 p2pMediaLoaderInfohashes: string[] 50 p2pMediaLoaderInfohashes: string[]
51 51
52 @AllowNull(false) 52 @AllowNull(false)
53 @Column
54 p2pMediaLoaderPeerVersion: number
55
56 @AllowNull(false)
53 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) 57 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
54 @Column 58 @Column
55 segmentsSha256Url: string 59 segmentsSha256Url: string
@@ -92,14 +96,26 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
92 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) { 96 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
93 const hashes: string[] = [] 97 const hashes: string[] = []
94 98
95 // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97 99 // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115
96 for (let i = 0; i < videoFiles.length; i++) { 100 for (let i = 0; i < videoFiles.length; i++) {
97 hashes.push(sha1(`1${playlistUrl}+V${i}`)) 101 hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`))
98 } 102 }
99 103
100 return hashes 104 return hashes
101 } 105 }
102 106
107 static listByIncorrectPeerVersion () {
108 const query = {
109 where: {
110 p2pMediaLoaderPeerVersion: {
111 [Sequelize.Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
112 }
113 }
114 }
115
116 return VideoStreamingPlaylistModel.findAll(query)
117 }
118
103 static loadWithVideo (id: number) { 119 static loadWithVideo (id: number) {
104 const options = { 120 const options = {
105 include: [ 121 include: [